snappy-prerender 2.7.0 → 2.8.1

package/src/core/crawl.js

@@ -1,5 +1,12 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import {
+  activeLocale,
+  isLocalePrefixedRoute,
+  localeByCode,
+  localeRoute,
+  stripLocalePrefix,
+} from './locales.js';
 import { renderRoute } from './render.js';
 import { resolveRouteConfig } from './route-config.js';
 import {
@@ -24,12 +31,37 @@ export async function renderAllRoutes({
   report,
   outputDir,
   sourceSeeds = [],
+  caches = null,
+  localePlan = null,
 }) {
   const visited = new Set();
   const scheduled = new Set();
   const rendered = new Map();
+  const routeLocales = new Map();
   const cap = config.maxRoutes ?? Number.POSITIVE_INFINITY;
   let nextOrder = 0;
+  const defaultLocale = localePlan ? localeByCode(localePlan, localePlan.default) : null;
+  const notFoundRoute = config.notFoundRoute ? normaliseRoute(config.notFoundRoute) : null;
+
+  // Expands a base route into every locale variant it publishes. The not-found route is
+  // deliberately not localized: a host needs one 404 document.
+  const variantsFor = (route) => {
+    if (!localePlan) return [{ route, locale: null, baseRoute: route }];
+    if (localePlan.mode === 'single') {
+      return [{ route, locale: activeLocale(localePlan), baseRoute: route }];
+    }
+    const baseRoute = defaultLocale ? stripLocalePrefix(route, defaultLocale) : route;
+    const isNotFound = notFoundRoute !== null && normaliseRoute(baseRoute) === notFoundRoute;
+    const variants = [];
+    for (const locale of localePlan.list) {
+      if (isNotFound && locale !== defaultLocale) continue;
+      const localized = isNotFound
+        ? normaliseRoute(config.notFoundRoute)
+        : localeRoute(locale, baseRoute);
+      variants.push({ route: localized, locale, baseRoute });
+    }
+    return variants;
+  };
 
   // Queues routes up to maxRoutes, recording exactly which candidates did not fit.
   // A seed used only to reach wildcard routes is given output: false and is rendered
@@ -38,14 +70,24 @@ export async function renderAllRoutes({
     const accepted = [];
     const dropped = [];
     for (const route of routes) {
-      if (scheduled.has(route)) continue;
-      if (scheduled.size >= cap) {
-        dropped.push(route);
-        continue;
+      if (isLocalePrefixedRoute(route, localePlan)) continue;
+      for (const variant of variantsFor(route)) {
+        if (scheduled.has(variant.route)) continue;
+        if (scheduled.size >= cap) {
+          dropped.push(variant.route);
+          continue;
+        }
+        scheduled.add(variant.route);
+        routeLocales.set(variant.route, variant);
+        accepted.push({
+          route: variant.route,
+          output: outputFor(variant.baseRoute),
+          order: nextOrder,
+          locale: variant.locale,
+          baseRoute: variant.baseRoute,
+        });
+        nextOrder += 1;
       }
-      scheduled.add(route);
-      accepted.push({ route, output: outputFor(route), order: nextOrder });
-      nextOrder += 1;
     }
     if (dropped.length > 0) {
       const previous = report.truncated?.droppedRoutes ?? [];
@@ -58,7 +100,14 @@ export async function renderAllRoutes({
     return accepted;
   };
 
-  const renderOne = async ({ route, output, depth = 0, order = null }) => {
+  const renderOne = async ({
+    route,
+    output,
+    depth = 0,
+    order = null,
+    locale = null,
+    baseRoute = route,
+  }) => {
     if (visited.has(route)) return null;
     visited.add(route);
     log.debug(`rendering ${route}${output ? '' : ' as a crawl seed'}`);
@@ -73,13 +122,15 @@ export async function renderAllRoutes({
       report,
       outputDir,
       screenshot: config.saveAs !== 'html',
+      caches,
+      locale,
     });
     if (!result) return null;
     result.depth = depth;
     result.order = order;
     try {
       const notFoundRoute = config.notFoundRoute ? normaliseRoute(config.notFoundRoute) : null;
-      const isNotFoundRoute = notFoundRoute !== null && route === notFoundRoute;
+      const isNotFoundRoute = notFoundRoute !== null && normaliseRoute(baseRoute) === notFoundRoute;
       if (result.statusCode === 404 && !isNotFoundRoute) {
         report.errors.push({ route, message: 'returned 404' });
         log.error(`  route ${route} returned 404`);
@@ -147,7 +198,12 @@ export async function renderAllRoutes({
   if (sourceSeeds.length === 0 || config.includeProvided) {
     candidates.push(...crawlRoots(config));
   }
-  const roots = accept([...new Set(sortRoutes(candidates, config))], outputFor);
+  const roots = accept(
+    [...new Set(sortRoutes(candidates, config))].filter(
+      (route) => !isLocalePrefixedRoute(route, localePlan),
+    ),
+    outputFor,
+  );
 
   const deterministic = config.maxRoutes !== null || config.routePriority.length > 0;
   if (deterministic) {
@@ -155,15 +211,17 @@ export async function renderAllRoutes({
   } else {
     const pool = createPool({
       concurrency: config.concurrency,
-      handler: async ({ route, depth, output, order }) => {
-        const renderedRoute = await renderOne({ route, output, depth, order });
+      handler: async ({ route, depth, output, order, locale, baseRoute }) => {
+        const renderedRoute = await renderOne({ route, output, depth, order, locale, baseRoute });
         if (!renderedRoute) return;
         const { result, routeConfig } = renderedRoute;
         if (routeConfig.crawl && withinDepth(depth + 1, routeConfig.maxDepth)) {
           const discovered = filterRoutes(result.links, {
             include: config.includeProvided ? config.include : [],
             exclude: config.exclude,
-          }).filter((candidate) => !scheduled.has(candidate));
+          })
+            .filter((candidate) => !isLocalePrefixedRoute(candidate, localePlan))
+            .filter((candidate) => !scheduled.has(candidate));
           const accepted = accept(sortRoutes(discovered, config));
           pool.push(accepted.map((item) => ({ ...item, depth: depth + 1 })));
         }
@@ -176,7 +234,7 @@ export async function renderAllRoutes({
       log.error(`  pool error on ${item.route}: ${error.message}`);
     }
   }
-  return rendered;
+  return { rendered, routeLocales };
 }
 
 /**
@@ -194,6 +252,8 @@ async function renderWithRetries({
   report,
   outputDir,
   screenshot,
+  caches,
+  locale,
 }) {
   const attempts = config.retries + 1;
   for (let attempt = 1; attempt <= attempts; attempt += 1) {
@@ -206,9 +266,11 @@ async function renderWithRetries({
         browser,
         origin: server.origin,
         route,
+        locale,
         config: routeConfig,
         screenshot,
         traceFile,
+        caches,
       });
       if (attempt > 1) log.info(`  ${route}: succeeded on attempt ${attempt}`);
       result.attempts = attempt;

package/src/core/csp.js

@@ -20,11 +20,23 @@ const CSP_MARKER = 'data-snappy-csp';
  */
 export function applyCspHashes(html) {
   const document = parse(html);
+  const result = applyCspHashesToDocument(document);
+  return {
+    html: result.existingPolicy ? html : serialize(document),
+    ...result,
+  };
+}
+
+/**
+ * Applies the hash policy to an already parsed document. Returns counts and whether a
+ * user-owned policy was left untouched.
+ */
+export function applyCspHashesToDocument(document) {
   const head = findHead(document);
-  if (!head) return { html, hashedScripts: 0, hashedStyles: 0, existingPolicy: false };
+  if (!head) return { hashedScripts: 0, hashedStyles: 0, existingPolicy: false };
   removeGeneratedPolicy(head);
   const existing = findPolicyMeta(head);
-  if (existing) return { html, hashedScripts: 0, hashedStyles: 0, existingPolicy: true };
+  if (existing) return { hashedScripts: 0, hashedStyles: 0, existingPolicy: true };
 
   const scriptHashes = [];
   const styleHashes = [];
@@ -43,7 +55,6 @@ export function applyCspHashes(html) {
   ];
   head.childNodes.unshift(metaNode(directives.join('; ')));
   return {
-    html: serialize(document),
     hashedScripts: scriptHashes.length,
     hashedStyles: styleHashes.length,
     existingPolicy: false,

package/src/core/destination.js

@@ -1,5 +1,8 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import { mapLimit } from './scheduler.js';
+
+const COPY_CONCURRENCY = 8;
 
 /**
  * Prepares the directory the run writes into. When a destination is configured the
@@ -58,13 +61,26 @@ export async function prepareDestination({
 async function copyTree(source, target) {
   await fs.mkdir(target, { recursive: true });
   const entries = await fs.readdir(source, { withFileTypes: true });
-  let skipped = 0;
-  for (const entry of entries) {
+  const { results, errors } = await mapLimit(entries, COPY_CONCURRENCY, async (entry) => {
     const from = path.join(source, entry.name);
     const to = path.join(target, entry.name);
-    if (entry.isDirectory()) skipped += await copyTree(from, to);
-    else if (entry.isFile()) await fs.copyFile(from, to);
-    else skipped += 1;
-  }
-  return skipped;
+    if (entry.isDirectory()) return copyTree(from, to);
+    if (!entry.isFile()) return 1;
+    const [sourceStats, targetStats] = await Promise.all([
+      fs.stat(from),
+      fs.stat(to).catch(() => null),
+    ]);
+    if (
+      targetStats?.isFile() &&
+      targetStats.size === sourceStats.size &&
+      targetStats.mtimeMs === sourceStats.mtimeMs
+    ) {
+      return 0;
+    }
+    await fs.copyFile(from, to);
+    await fs.utimes(to, sourceStats.atime, sourceStats.mtime);
+    return 0;
+  });
+  if (errors.length > 0) throw errors[0].error;
+  return results.reduce((total, skipped) => total + (skipped ?? 0), 0);
 }

package/src/core/hydration-policy.js

@@ -0,0 +1,38 @@
+import { matchesPattern } from './routes.js';
+
+/**
+ * Resolves the hydration policy for one route. Global lists provide the default and a
+ * matching route override wins, so a broad allow-list can be narrowed for one route.
+ * @param {object} config
+ * @param {string} route
+ * @returns {'strict' | 'allow-rerender' | 'skip'}
+ */
+export function resolveHydrationPolicy(config, route) {
+  let policy = 'strict';
+  if ((config.hydration?.skip ?? []).some((pattern) => matchesPattern(route, pattern))) {
+    policy = 'skip';
+  } else if (
+    (config.hydration?.allowRerender ?? []).some((pattern) => matchesPattern(route, pattern))
+  ) {
+    policy = 'allow-rerender';
+  }
+  for (const override of config.routeOverrides ?? []) {
+    if (!matchesPattern(route, override.match)) continue;
+    if (override.hydration !== undefined) policy = override.hydration;
+  }
+  return policy;
+}
+
+/**
+ * Reports whether a verification result is allowed to differ from the prerendered markup.
+ * A route only becomes allowed after it actually re-rendered or logged a hydration error.
+ * @param {'strict' | 'allow-rerender' | 'skip'} policy
+ * @param {{ mode: string, hydrationErrors: string[] }} result
+ * @returns {boolean}
+ */
+export function isHydrationAllowed(policy, result) {
+  return (
+    policy === 'allow-rerender' &&
+    (result.mode === 're-rendered' || (result.hydrationErrors?.length ?? 0) > 0)
+  );
+}

package/src/core/inline-css.js

@@ -1,3 +1,4 @@
+import path from 'node:path';
 import { hasImportRule, rewriteCssUrls } from './css-url.js';
 import { minifyCss } from './minify.js';
 
@@ -8,8 +9,9 @@ import { minifyCss } from './minify.js';
  * place. Relative url() values are rebased against each sheet before joining so fonts
  * and images keep working on nested routes.
  */
-export async function inlineStylesheets({ page, minifyCssOptions }) {
-  const collected = await page.evaluate(async () => {
+export async function inlineStylesheets({ page, minifyCssOptions, cache = null }) {
+  const cachedCss = cache ? Object.fromEntries(cache) : {};
+  const collected = await page.evaluate(async (knownCss) => {
     // Only media that applies on screen can be inlined without changing how the sheet
     // applies, so a print stylesheet is left as a link instead of becoming active.
     const INLINEABLE_MEDIA = new Set(['', 'all', 'screen']);
@@ -22,24 +24,30 @@ export async function inlineStylesheets({ page, minifyCssOptions }) {
         skipped.push(link.href);
         continue;
       }
+      if (Object.hasOwn(knownCss, link.href)) {
+        sheets.push({ css: knownCss[link.href], href: link.href, fetched: false });
+        continue;
+      }
       try {
         const response = await fetch(link.href);
         if (!response.ok) throw new Error(String(response.status));
-        sheets.push({ css: await response.text(), href: link.href });
+        sheets.push({ css: await response.text(), href: link.href, fetched: true });
       } catch {
         skipped.push(link.href);
       }
     }
     return { sheets, skipped };
-  });
+  }, cachedCss);
 
   const inlineable = [];
   const skipped = [...collected.skipped];
+  const fetched = [];
   for (const sheet of collected.sheets) {
+    if (sheet.fetched) fetched.push({ href: sheet.href, css: sheet.css });
     if (hasImportRule(sheet.css)) skipped.push(sheet.href);
     else inlineable.push(sheet);
   }
-  if (inlineable.length === 0) return { skipped };
+  if (inlineable.length === 0) return { skipped, fetched };
 
   const pageOrigin = new URL(page.url()).origin;
   const joined = inlineable
@@ -59,7 +67,7 @@ export async function inlineStylesheets({ page, minifyCssOptions }) {
     },
     { text: css, hrefs: inlineable.map((sheet) => sheet.href) },
   );
-  return { skipped };
+  return { skipped, fetched };
 }
 
 /**
@@ -68,7 +76,43 @@ export async function inlineStylesheets({ page, minifyCssOptions }) {
  * beasties is an optional peer dependency because most projects never need it.
  * `preload` is passed through so a strict script-src can turn the swap handlers off.
  */
-export async function inlineCriticalCss({ html, outputDir, base, preload = 'media' }) {
+export async function inlineCriticalCss({
+  html,
+  outputDir,
+  base,
+  preload = 'media',
+  cache = null,
+}) {
+  const key = `${path.resolve(outputDir)}|${base}|${preload}`;
+  let state = cache?.get(key);
+  if (!state) {
+    state = {
+      processor: createCriticalCssProcessor({ outputDir, base, preload }),
+      tail: Promise.resolve(),
+    };
+    cache?.set(key, state);
+  }
+  // Beasties keeps per-instance selector state and is not safe for concurrent process
+  // calls, so this queue serialises routes while sharing the instance.
+  const run = state.tail.then(async () => {
+    const beasties = await state.processor;
+    return beasties.process(html);
+  });
+  state.tail = run.then(
+    () => undefined,
+    () => undefined,
+  );
+  try {
+    return await run;
+  } catch (error) {
+    if (cache?.get(key) === state) cache.delete(key);
+    throw error;
+  }
+}
+
+// Creates the Beasties processor once per output configuration. The selector cache is
+// reused across routes; stylesheet text is still read per page by the library.
+async function createCriticalCssProcessor({ outputDir, base, preload }) {
   let Beasties;
   try {
     ({ default: Beasties } = await import('beasties'));
@@ -77,12 +121,11 @@ export async function inlineCriticalCss({ html, outputDir, base, preload = 'medi
       "inlineCss: 'critical' needs the optional beasties dependency. Install it with: npm install beasties",
     );
   }
-  const beasties = new Beasties({
+  return new Beasties({
     path: outputDir,
     publicPath: base,
     logLevel: 'silent',
     preload,
     noscriptFallback: true,
   });
-  return beasties.process(html);
 }

package/src/core/locale-context.js

@@ -0,0 +1,63 @@
+import {
+  resolveSeoLocale,
+  seedContextCookie,
+  seedCookies,
+  seedLocalStorage,
+  seedPageStorage,
+} from './seo-locale.js';
+
+/**
+ * Builds the Playwright context options that make one locale deterministic: browser
+ * locale, timezone and the base storage state. The caller keeps auth storage separate from
+ * locale cookies and local storage.
+ * @param {object} config
+ * @param {object | null} locale
+ * @returns {{ locale?: string, timezoneId?: string }}
+ */
+export function localeContextOptions(config, locale) {
+  const options = {};
+  const browserLocale = locale?.locale ?? null;
+  if (browserLocale) options.locale = browserLocale;
+  const timezone = locale?.timezone ?? config.timezone ?? null;
+  if (timezone) options.timezoneId = timezone;
+  return options;
+}
+
+/**
+ * Returns the storage state a locale should use, preferring its own file over the global
+ * authenticated state.
+ * @param {object} config
+ * @param {object | null} locale
+ * @returns {string | undefined}
+ */
+export function localeStorageState(config, locale) {
+  return locale?.storageState ?? config.storageState ?? undefined;
+}
+
+/**
+ * Seeds the extra cookies and local storage declared on a locale record. These are
+ * independent of the SEO locale recipe and let a locale carry app-specific keys.
+ * @param {{ page: import('playwright-core').Page, context: import('playwright-core').BrowserContext, origin: string, locale: object | null }} input
+ */
+export async function seedLocaleExtras({ page, context, origin, locale }) {
+  if (!locale) return;
+  await seedCookies(context, origin, locale.cookies ?? []);
+  await seedLocalStorage(page, locale.localStorage ?? {}, true);
+}
+
+/**
+ * Seeds the cookie and local storage the app reads before it boots. Render and verification
+ * use the same recipe, so the selected locale is identical in both passes.
+ * @param {{ page: import('playwright-core').Page, context: import('playwright-core').BrowserContext, origin: string, config: object, route: string, locale: object | null, phase: 'render' | 'verify' }} input
+ * @returns {Promise<object | null>} the resolved SEO locale, when one applied
+ */
+export async function seedLocaleContext({ page, context, origin, config, route, locale, phase }) {
+  await seedLocaleExtras({ page, context, origin, locale });
+  const seo = resolveSeoLocale(config, locale, route);
+  if (!seo) return null;
+  if (phase === 'render' && !seo.seedRender) return null;
+  if (phase === 'verify' && !seo.seedVerification) return null;
+  await seedPageStorage(page, seo);
+  await seedContextCookie(context, origin, seo);
+  return seo;
+}