snappy-prerender 2.7.0 → 2.8.1

package/src/core/render.js

@@ -1,24 +1,13 @@
+import { installClock, installClockScripts } from './clock.js';
 import { captureFormState } from './form-state.js';
 import { inlineStylesheets } from './inline-css.js';
-import { isPageEmpty } from './page-content.js';
+import { localeContextOptions, localeStorageState, seedLocaleContext } from './locale-context.js';
+import { readPageState } from './page-content.js';
 import { createResourceCollector } from './resources.js';
 import { normaliseRoute, toAppRoute, toServerUrl } from './routes.js';
 import { captureRuntimeStyles } from './runtime-styles.js';
 import { freezeAnimations, scrollThroughPage, settlePage, trackNetwork } from './settle.js';
-import { separateTextNodes } from './text-separators.js';
-
-const SCRIPT_ESCAPES = {
-  '<': '\\u003C',
-  '>': '\\u003E',
-  '/': '\\u002F',
-  '\u2028': '\\u2028',
-  '\u2029': '\\u2029',
-};
-
-// Serialises a value for embedding in a script element without allowing tag breakout.
-function escapeForScript(value) {
-  return JSON.stringify(value).replace(/[<>/\u2028\u2029]/g, (char) => SCRIPT_ESCAPES[char]);
-}
+import { readRenderSnapshot } from './text-separators.js';
 
 /**
  * Renders one route in a fresh browser context and returns the serialised DOM plus
@@ -30,17 +19,20 @@ export async function renderRoute({
   browser,
   origin,
   route,
+  locale = null,
   config,
   screenshot = false,
   traceFile = null,
+  caches = null,
 }) {
   const context = await browser.newContext({
     viewport: config.viewport,
-    storageState: config.storageState ?? undefined,
+    storageState: localeStorageState(config, locale),
     userAgent: config.userAgent ?? undefined,
     ignoreHTTPSErrors: config.ignoreHTTPSErrors,
     reducedMotion: config.freezeAnimations ? 'reduce' : 'no-preference',
     serviceWorkers: 'block',
+    ...localeContextOptions(config, locale),
   });
   let tracing = false;
   if (traceFile) {
@@ -48,6 +40,11 @@ export async function renderRoute({
     tracing = true;
   }
   const page = await context.newPage();
+  await installClock(page, config.fixedTime ?? null);
+  await installClockScripts(page, {
+    fixedTime: config.fixedTime ?? null,
+    randomSeed: config.randomSeed ?? null,
+  });
   const pageErrors = [];
   const consoleErrors = [];
   const failedRequests = [];
@@ -75,6 +72,15 @@ export async function renderRoute({
   const metadataDefaults = config.metadata
     ? { base: config.base, flatOutput: config.flatOutput, ...config.metadata }
     : null;
+  await seedLocaleContext({
+    page,
+    context,
+    origin,
+    config,
+    route,
+    locale,
+    phase: 'render',
+  });
   if (metadataDefaults) {
     await page.addInitScript((defaults) => {
       window.__SNAPPY_META__ = defaults;
@@ -127,20 +133,21 @@ export async function renderRoute({
       }
     }
     await settlePage(page, config, tracker);
-    const empty = await isPageEmpty(page);
-    const statusCode = await page.evaluate(() => {
-      const marker = document.querySelector('meta[name="prerender-status-code"]');
-      if (!marker) return null;
-      const value = Number.parseInt(marker.getAttribute('content') ?? '', 10);
-      return Number.isFinite(value) ? value : null;
-    });
+    const { empty, statusCode } = await readPageState(page);
     const finalUrl = page.url();
 
     if (config.captureRuntimeStyles) await captureRuntimeStyles(page);
     if (config.captureFormState) await captureFormState(page);
     if (config.inlineCss === true || config.inlineCss === 'inline') {
-      const inlined = await inlineStylesheets({ page, minifyCssOptions: config.minifyCss });
+      const inlined = await inlineStylesheets({
+        page,
+        minifyCssOptions: config.minifyCss,
+        cache: caches?.inlineCss ?? null,
+      });
       inlineCssSkipped = inlined.skipped;
+      if (caches?.inlineCss) {
+        for (const sheet of inlined.fetched ?? []) caches.inlineCss.set(sheet.href, sheet.css);
+      }
     }
     await collector.settled();
     const stateScript = await injectCapturedState(page, collector.json, {
@@ -150,7 +157,7 @@ export async function renderRoute({
     if (metadataDefaults) await injectMetadataDefaults(page, metadataDefaults);
     if (config.freezeAnimations) await freezeAnimations(page);
     if (config.scrollToBottom) await scrollThroughPage(page, config);
-    const lcpUrl = await page.evaluate(() => window.__snappyLcp?.at(-1)?.url ?? null);
+    const { lcpUrl, hrefs } = await readRenderSnapshot(page);
     let lcpImage = null;
     if (lcpUrl) {
       try {
@@ -160,9 +167,7 @@ export async function renderRoute({
         // Ignore malformed performance entries.
       }
     }
-    await separateTextNodes(page);
-
-    const links = await collectLinks(page, config.base, origin);
+    const links = collectLinks(hrefs, config.base, origin);
 
     if (screenshot) {
       const buffer = await page.screenshot({
@@ -311,16 +316,21 @@ async function injectCapturedState(page, cache, { externalScripts = false, base
  * executable script, so a strict script-src applies to nothing here.
  */
 async function injectMetadataDefaults(page, defaults) {
-  const payload = await page.evaluate((fallback) => {
+  await page.evaluate((fallback) => {
     document.querySelectorAll('script[data-snappy-meta]').forEach((script) => script.remove());
     const current = typeof window === 'undefined' ? null : window.__SNAPPY_META__;
     const recorded = typeof window === 'undefined' ? null : window.__snappyHeadOriginals;
     const merged = { ...(current ?? fallback) };
     if (recorded) merged.originals = recorded;
-    return Object.keys(merged).length > 0 ? merged : fallback;
-  }, defaults);
-  const element = escapeForScript(payload);
-  await page.evaluate((text) => {
+    const payload = Object.keys(merged).length > 0 ? merged : fallback;
+    const escapes = {
+      '<': '\\u003C',
+      '>': '\\u003E',
+      '/': '\\u002F',
+      '\u2028': '\\u2028',
+      '\u2029': '\\u2029',
+    };
+    const text = JSON.stringify(payload).replace(/[<>/\u2028\u2029]/g, (char) => escapes[char]);
     const script = document.createElement('script');
     script.setAttribute('type', 'application/json');
     script.setAttribute('data-snappy-meta', '');
@@ -328,7 +338,7 @@ async function injectMetadataDefaults(page, defaults) {
     const first = document.scripts[0];
     if (first?.parentNode) first.parentNode.insertBefore(script, first);
     else document.head.appendChild(script);
-  }, element);
+  }, defaults);
 }
 
 /**
@@ -336,8 +346,7 @@ async function injectMetadataDefaults(page, defaults) {
  * app-relative routes for the crawler. Off-origin links are ignored so external
  * URLs can never be mistaken for routes and written into the output directory.
  */
-async function collectLinks(page, base, origin) {
-  const hrefs = await page.$$eval('a[href]', (anchors) => anchors.map((anchor) => anchor.href));
+function collectLinks(hrefs, base, origin) {
   const routes = [];
   for (const href of hrefs) {
     let parsed;
@@ -350,7 +359,7 @@ async function collectLinks(page, base, origin) {
     const route = toAppRoute(href, base);
     if (route) routes.push(route);
   }
-  return routes;
+  return [...new Set(routes)];
 }
 
 // Parses an HTTP URL, returning null for data, blob and other non-network URLs.

package/src/core/report.js

@@ -34,7 +34,11 @@ export function buildReport(report, config = null) {
     version: 1,
     generatedAt: new Date().toISOString(),
     durationMs: report.durationMs,
+    timings: report.timings,
     ok: report.ok,
+    clock: report.clock ?? null,
+    locales: report.locales ?? null,
+    seoLocale: report.seoLocale ?? null,
     summary: {
       routes: report.routeDetails.length,
       files: report.files.filter((file) => file.status !== 'dry-run').length,
@@ -48,6 +52,9 @@ export function buildReport(report, config = null) {
           modes: report.verification.modes,
           frameworks: report.verification.frameworks,
           skipped: report.verification.skipped ?? 0,
+          policySkipped: report.verification.policySkipped ?? 0,
+          allowed: report.verification.allowed ?? 0,
+          allowedRerender: report.verification.allowedRerender ?? 0,
           errors: report.verification.errors,
         }
       : null,
@@ -85,9 +92,10 @@ function sanitiseValue(value, config) {
   let text = value;
   if (config?.storageState) {
     for (const secret of [config.storageState, path.resolve(config.storageState)]) {
-      text = text.split(secret).join('[redacted]');
+      if (text.includes(secret)) text = text.split(secret).join('[redacted]');
     }
   }
+  if (!/https?:\/\//i.test(text)) return text;
   return text.replace(/https?:\/\/[^\s"'<>]+/gi, redactUrl);
 }
 

package/src/core/route-sources.js

@@ -1,9 +1,11 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { isSafeRoute, normaliseRoute, toAppRoute } from './routes.js';
+import { mapLimit } from './scheduler.js';
 
 const MAX_SOURCE_ROUTES = 100000;
 const FETCH_TIMEOUT = 10000;
+const SITEMAP_FETCH_CONCURRENCY = 4;
 const XML_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
 
 /**
@@ -80,9 +82,18 @@ async function readSitemapLevel(location, pageUrls, depth) {
     throw new Error(`${location} is not a sitemap: no urlset or sitemapindex element found`);
   }
   if (/<sitemapindex\b/i.test(text)) {
-    for (const child of extractLocs(text)) {
-      await readSitemapLevel(resolveChild(child, base), pageUrls, depth + 1);
-    }
+    const children = extractLocs(text);
+    const { results, errors } = await mapLimit(
+      children,
+      SITEMAP_FETCH_CONCURRENCY,
+      async (child) => {
+        const routes = [];
+        await readSitemapLevel(resolveChild(child, base), routes, depth + 1);
+        return routes;
+      },
+    );
+    if (errors.length > 0) throw errors[0].error;
+    for (const routes of results) pageUrls.push(...routes);
     return;
   }
   pageUrls.push(...extractLocs(text));

package/src/core/run.js

@@ -3,9 +3,11 @@ import path from 'node:path';
 import { writePreloadManifest, writeSitemap } from './artifacts.js';
 import { auditRun } from './audit/index.js';
 import { launchBrowser } from './browser.js';
+import { resolveFixedTime } from './clock.js';
 import { resolveConfig } from './config.js';
 import { renderAllRoutes } from './crawl.js';
 import { prepareDestination } from './destination.js';
+import { buildLocalePlan } from './locales.js';
 import { createLogger } from './log.js';
 import { routeToFile } from './output.js';
 import { pruneStaleFiles, readPruneManifest, writePruneManifest } from './prune.js';
@@ -29,12 +31,17 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
   for (const warning of config.warnings ?? []) log.warn(warning);
   const sourceDir = path.resolve(config.sourceDir);
   const startedAt = Date.now();
+  const fixedTime = resolveFixedTime(config, startedAt);
+  config.fixedTime = fixedTime;
+  const localePlan = buildLocalePlan(config);
 
   await assertSourceDir(sourceDir, config);
-  const outputDir = await resolveOutputDir({ sourceDir, config, log });
+  const [outputDir, routeSources] = await Promise.all([
+    resolveOutputDir({ sourceDir, config, log }),
+    loadRouteSeeds(config),
+  ]);
   const previousManifest =
     config.pruneStaleRoutes && !config.dryRun ? await readPruneManifest(outputDir) : null;
-  const routeSources = await loadRouteSeeds(config);
   for (const source of routeSources.sources) {
     const skipped = source.skipped > 0 ? `, ${source.skipped} skipped` : '';
     log.info(`Route source ${source.source}: ${source.routes} route(s)${skipped}`);
@@ -44,14 +51,21 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
     log.warn(`saveAs: '${config.saveAs}' writes images, so hydration verification is skipped`);
   }
 
-  const server = config.serveCmd
-    ? await startCommandServer({
-        command: config.serveCmd,
-        url: config.url,
-        timeout: config.serveCmdTimeout,
-        shutdownTimeout: config.shutdownTimeout,
-      })
-    : await startStaticServer({ dir: outputDir, base: config.base });
+  const browserPromise = launchBrowser(config, log);
+  let server;
+  try {
+    server = config.serveCmd
+      ? await startCommandServer({
+          command: config.serveCmd,
+          url: config.url,
+          timeout: config.serveCmdTimeout,
+          shutdownTimeout: config.shutdownTimeout,
+        })
+      : await startStaticServer({ dir: outputDir, base: config.base });
+  } catch (error) {
+    browserPromise.then(({ browser }) => browser.close()).catch(() => {});
+    throw error;
+  }
 
   const report = {
     routes: [],
@@ -67,6 +81,17 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
     notFoundRoutes: [],
     routeSources: routeSources.sources,
     routeDetails: [],
+    clock: fixedTime === null ? null : { fixedTime, timezone: config.timezone ?? null },
+    locales: localePlan
+      ? { mode: localePlan.mode, default: localePlan.default, active: localePlan.active ?? null }
+      : null,
+    seoLocale: config.seoLocale
+      ? {
+          code: config.seoLocale.code ?? config.locales?.active ?? null,
+          source: config.seoLocale.source,
+          inject: config.seoLocale.inject,
+        }
+      : null,
     cspSkipped: [],
     traces: [],
     verification: null,
@@ -77,16 +102,19 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
     truncated: null,
     ok: false,
     durationMs: 0,
+    timings: { render: 0, write: 0, artifacts: 0, verify: 0, audit: 0 },
   };
   const manifestEntries = [];
   const state = { warnedBuildOriginMetadata: false, warnedCriticalCssSkipped: false };
   let browser = null;
 
   try {
-    const launched = await launchBrowser(config, log);
+    const launched = await browserPromise;
     browser = launched.browser;
     log.info(`Browser: ${launched.source}`);
+    const renderCaches = { inlineCss: new Map(), criticalCss: new Map() };
 
+    let phaseStartedAt = Date.now();
     const rendered = await renderAllRoutes({
       browser,
       server,
@@ -95,7 +123,19 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
       report,
       outputDir,
       sourceSeeds: routeSources.routes,
+      caches: renderCaches,
+      localePlan,
     });
+    report.timings.render = Date.now() - phaseStartedAt;
+    const { rendered: renderedRoutes, routeLocales } = rendered;
+    if (localePlan) {
+      const renderedByLocale = {};
+      for (const route of report.routes) {
+        const code = routeLocales.get(route)?.locale?.code ?? localePlan.default;
+        renderedByLocale[code] = (renderedByLocale[code] ?? 0) + 1;
+      }
+      report.locales.rendered = renderedByLocale;
+    }
     if (report.routes.length === 0) {
       report.errors.push({ route: null, message: 'no routes were rendered' });
       log.error('No routes were rendered; check include, exclude and sourceDir');
@@ -110,19 +150,32 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
       );
     }
 
+    phaseStartedAt = Date.now();
     const verifiable = await writeAllRoutes({
-      rendered,
+      rendered: renderedRoutes,
+      routeLocales,
+      localePlan,
       outputDir,
       config,
       log,
       report,
       state,
       manifestEntries,
+      caches: renderCaches,
     });
+    report.timings.write = Date.now() - phaseStartedAt;
+    const localeScripts = report.files
+      .filter((file) => file.file?.startsWith('snappy-seo-locale-'))
+      .map((file) => file.file);
+    if (report.seoLocale) {
+      report.seoLocale.script = localeScripts[0] ?? null;
+      report.seoLocale.scripts = [...new Set(localeScripts)];
+    }
     logFileSummary(report, log);
     logNotFoundNotice(report, config, log);
     logDuplicateTitles(report, log);
 
+    phaseStartedAt = Date.now();
     if (config.sitemap) {
       try {
         await writeSitemap({ outputDir, config, log, report });
@@ -155,6 +208,7 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
         log.error(`preload manifest write failed: ${error.message}`);
       }
     }
+    report.timings.artifacts = Date.now() - phaseStartedAt;
 
     const canVerify = config.verify && !config.dryRun && config.saveAs === 'html';
     if (canVerify && config.serveCmd) {
@@ -164,13 +218,13 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
     }
     // With verifyChangedOnly, a route whose HTML file was left untouched is not reloaded: the HTML
     // and the bundle are the same as the last time it was verified.
+    const fileStatus = new Map(
+      report.files.map((file) => [`${file.route ?? ''}|${file.file ?? ''}`, file.status]),
+    );
     const toVerify = config.verifyChangedOnly
       ? verifiable.filter((route) => {
           const file = routeToFile(route, config);
-          const entry = report.files.find(
-            (candidate) => candidate.route === route && candidate.file === file,
-          );
-          return entry?.status !== 'unchanged';
+          return fileStatus.get(`${route}|${file}`) !== 'unchanged';
         })
       : verifiable;
     const skipped = verifiable.length - toVerify.length;
@@ -178,14 +232,18 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
       log.info(`Verification skipped for ${skipped} unchanged route(s), as verifyChangedOnly asks`);
     }
     if (canVerify && !config.serveCmd && toVerify.length > 0) {
+      phaseStartedAt = Date.now();
       report.verification = await verifyRoutes({
         browser,
         origin: server.origin,
         routes: toVerify.sort(),
         config,
         outputDir,
+        routeLocales,
       });
+      report.timings.verify = Date.now() - phaseStartedAt;
       report.verification.skipped = skipped;
+      report.verification.policySkipped = report.verification.skippedRoutes.length;
       logVerification(report.verification, config, log);
       mergeVerification(report);
     }
@@ -224,6 +282,7 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
 
   report.durationMs = Date.now() - startedAt;
   if (config.audit) {
+    const auditStartedAt = Date.now();
     try {
       report.audit = await auditRun({ report, config, outputDir });
       logAudit(report.audit, log);
@@ -231,8 +290,11 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
       report.errors.push({ route: null, message: `audit failed: ${error.message}` });
       log.error(`audit failed: ${error.message}`);
     }
+    report.timings.audit = Date.now() - auditStartedAt;
   }
   const reRendered = report.verification?.modes['re-rendered'] ?? 0;
+  const allowedRerender = report.verification?.allowedRerender ?? 0;
+  const notAllowedRerender = Math.max(0, reRendered - allowedRerender);
   const undetected = report.verification
     ? (report.verification.modes.undetected ?? 0) + (report.verification.modes.unknown ?? 0)
     : 0;
@@ -247,7 +309,7 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
     !(config.failOnHydrationError && report.verification && !report.verification.ok) &&
     !(config.failOnVerificationError && report.verification?.errors.length > 0) &&
     !(config.failOnUndetected && undetected > 0) &&
-    !(config.failOnRerender && reRendered > 0);
+    !(config.failOnRerender && notAllowedRerender > 0);
   try {
     report.reportFile = await writeReport({ report, config, log });
   } catch (error) {
@@ -300,12 +362,16 @@ async function resolveOutputDir({ sourceDir, config, log }) {
 
 // Adds the verification result to each route record for the JSON report.
 function mergeVerification(report) {
+  const details = new Map(report.routeDetails.map((record) => [record.route, record]));
   for (const entry of report.verification?.routes ?? []) {
-    const detail = report.routeDetails.find((record) => record.route === entry.route);
+    const detail = details.get(entry.route);
     if (!detail) continue;
     detail.mode = entry.mode;
     detail.framework = entry.framework ?? null;
     detail.hydrationErrors = entry.hydrationErrors;
+    detail.ignoredHydrationErrors = entry.ignoredHydrationErrors ?? [];
+    detail.hydrationPolicy = entry.hydrationPolicy ?? null;
+    detail.hydrationAllowed = entry.allowed === true;
     detail.pageErrors = [...new Set([...(detail.pageErrors ?? []), ...entry.pageErrors])];
     detail.consoleErrors = [...new Set([...(detail.consoleErrors ?? []), ...entry.consoleErrors])];
     detail.verificationStatus = entry.status;
@@ -372,8 +438,10 @@ function logFileSummary(report, log) {
 // were reported but not enforced.
 function logVerification(verification, config, log) {
   const modes = verification.modes ?? {};
+  const allowed = verification.allowed ?? 0;
+  const policySkipped = verification.policySkipped ?? 0;
   log.info(
-    `Hydration modes: ${modes.hydrated ?? 0} hydrated, ${modes['re-rendered'] ?? 0} re-rendered, ${modes.undetected ?? 0} undetected, ${modes.unknown ?? 0} unknown`,
+    `Hydration modes: ${modes.hydrated ?? 0} hydrated, ${modes['re-rendered'] ?? 0} re-rendered, ${modes.undetected ?? 0} undetected, ${modes.unknown ?? 0} unknown${allowed > 0 ? `, ${allowed} allowed` : ''}${policySkipped > 0 ? `, ${policySkipped} skipped by policy` : ''}`,
   );
   const notDetected = (modes.undetected ?? 0) + (modes.unknown ?? 0);
   if (notDetected > 0) {
@@ -415,7 +483,9 @@ function logVerification(verification, config, log) {
     }
   }
 
-  const reRendered = verification.routes.filter((entry) => entry.mode === 're-rendered');
+  const reRendered = verification.routes.filter(
+    (entry) => entry.mode === 're-rendered' && !entry.allowed,
+  );
   if (reRendered.length > 0) {
     const sample = reRendered
       .slice(0, 5)