snappy-prerender 2.6.0 → 2.8.0

package/src/core/write.js

@@ -1,17 +1,34 @@
 import { createHash } from 'node:crypto';
+import { parse, serialize } from 'parse5';
 import { findBuildOriginUrls } from './build-origin.js';
+import { applyCspHashesToDocument } from './csp.js';
+import { resolveHydrationPolicy } from './hydration-policy.js';
 import { inlineCriticalCss } from './inline-css.js';
+import { applyLocaleMetadata, buildLocaleLinks, localePageUrl } from './locales.js';
 import { minifyHtml } from './minify.js';
 import { normaliseHtml } from './normalize.js';
 import {
   findOutputCollisions,
+  routeToFile,
   routeToScreenshotFile,
   writeBufferIfChanged,
   writeFileIfChanged,
   writeRouteHtml,
 } from './output.js';
-import { collectPageResources } from './preload-manifest.js';
+import { collectPageMetaFromDocument } from './page-meta.js';
+import { collectScriptSourcesFromDocument, injectPreloadLinksInto } from './preload-links.js';
+import { collectPageResourcesFromDocument } from './preload-manifest.js';
+import { resolveRouteConfig } from './route-config.js';
 import { normaliseRoute, toPublicPath } from './routes.js';
+import { mapLimit } from './scheduler.js';
+import {
+  buildSeoLocaleScript,
+  injectSeoLocaleScript,
+  resolveSeoLocale,
+  seoLocaleMetadataOptions,
+} from './seo-locale.js';
+
+const WRITE_CONCURRENCY = 4;
 
 /**
  * Writes every rendered route and returns the routes that can be verified. A write
@@ -19,12 +36,15 @@ import { normaliseRoute, toPublicPath } from './routes.js';
  */
 export async function writeAllRoutes({
   rendered,
+  routeLocales = new Map(),
+  localePlan = null,
   outputDir,
   config,
   log,
   report,
   state,
   manifestEntries,
+  caches = null,
 }) {
   const sorted = [...report.routes].sort();
   const conflicted = reportOutputCollisions({ routes: sorted, config, log, report });
@@ -33,30 +53,92 @@ export async function writeAllRoutes({
 
   const verifiable = [];
   const hashes = new Map();
-  for (const route of sorted) {
-    if (conflicted.has(route)) continue;
-    try {
-      const { file, resources } = await writeRouteOutput({
-        route,
-        result: rendered.get(route),
-        outputDir,
-        config,
-        log,
-        state,
-        report,
-      });
-      report.files.push(file);
-      if (resources) manifestEntries.push({ route, ...resources });
-      if (file.hash) {
-        const group = hashes.get(file.hash) ?? [];
-        group.push(route);
-        hashes.set(file.hash, group);
+  const localeScripts = new Map();
+  const { results } = await mapLimit(
+    sorted,
+    Math.min(WRITE_CONCURRENCY, config.concurrency),
+    async (route) => {
+      if (conflicted.has(route)) return null;
+      const routeReport = { errors: [], duplicateTitles: [], cspSkipped: [], files: [] };
+      try {
+        const routeConfig = resolveRouteConfig(config, route);
+        const result = rendered.get(route);
+        const { file, resources } = await writeRouteOutput({
+          route,
+          result,
+          outputDir,
+          config: routeConfig,
+          log,
+          state,
+          report: routeReport,
+          caches,
+          locale: routeLocales.get(route)?.locale ?? null,
+          baseRoute: routeLocales.get(route)?.baseRoute ?? route,
+          localePlan,
+          localeScripts,
+        });
+        return {
+          route,
+          result,
+          file,
+          resources,
+          hash: file.hash,
+          report: routeReport,
+        };
+      } catch (error) {
+        routeReport.errors.push({ route, message: `write failed: ${error.message}` });
+        log.error(`  write failed for ${route}: ${error.message}`);
+        return { route, report: routeReport };
       }
-      verifiable.push(route);
-    } catch (error) {
-      report.errors.push({ route, message: `write failed: ${error.message}` });
-      log.error(`  write failed for ${route}: ${error.message}`);
+    },
+  );
+
+  for (const entry of results) {
+    if (!entry) continue;
+    for (const error of entry.report.errors) report.errors.push(error);
+    for (const route of entry.report.duplicateTitles) report.duplicateTitles.push(route);
+    for (const route of entry.report.cspSkipped) report.cspSkipped.push(route);
+    for (const file of entry.report.files ?? []) report.files.push(file);
+    if (!entry.file) continue;
+    const { route, result, file, resources } = entry;
+    report.files.push(file);
+    report.routeDetails.push({
+      route,
+      locale: routeLocales.get(route)?.locale?.code ?? null,
+      order: result.order ?? null,
+      file: file.file,
+      status: file.status,
+      bytes: file.bytes,
+      depth: result.depth ?? null,
+      attempts: result.attempts ?? 1,
+      httpStatus: result.httpStatus ?? null,
+      finalUrl: result.finalUrl ?? null,
+      empty: result.empty === true,
+      notFound: file.notFound === true,
+      noindex: file.noindex === true,
+      meta: file.meta ?? null,
+      images: file.images ?? [],
+      alternates: file.alternates ?? [],
+      localeUrl: file.localeUrl ?? null,
+      mode: null,
+      framework: null,
+      hydrationErrors: [],
+      ignoredHydrationErrors: [],
+      hydrationPolicy: resolveHydrationPolicy(config, route),
+      hydrationAllowed: false,
+      pageErrors: result.pageErrors ?? [],
+      consoleErrors: result.consoleErrors ?? [],
+      failedRequests: result.failedRequests ?? [],
+      firstMismatch: null,
+      buildOriginUrls: file.buildOriginUrls ?? [],
+    });
+    if (resources) manifestEntries.push({ route, ...resources });
+    if (entry.hash) {
+      const group = hashes.get(entry.hash) ?? [];
+      group.push(route);
+      hashes.set(entry.hash, group);
     }
+    verifiable.push(route);
   }
   reportDuplicatePages({ groups: [...hashes.values()], config, log, report });
   return verifiable;
@@ -144,7 +226,20 @@ function reportOutputCollisions({ routes, config, log, report }) {
  * Applies the configured post-processing to one rendered route and writes it: DOM
  * cleanups and link hints, critical CSS extraction, then optional minification.
  */
-async function writeRouteOutput({ route, result, outputDir, config, log, state, report }) {
+async function writeRouteOutput({
+  route,
+  result,
+  outputDir,
+  config,
+  log,
+  state,
+  report,
+  caches,
+  locale = null,
+  baseRoute = route,
+  localePlan = null,
+  localeScripts = new Map(),
+}) {
   if (result.screenshot) {
     const file = routeToScreenshotFile(route, config);
     if (config.dryRun) {
@@ -166,23 +261,89 @@ async function writeRouteOutput({ route, result, outputDir, config, log, state,
     return { file: { route, ...written }, resources: null };
   }
 
-  const { html, stats } = normaliseHtml(result.html, {
-    removeStyleTags: config.removeStyleTags,
-    removeScriptTags: config.removeScriptTags,
-    asyncScriptTags: config.asyncScriptTags,
-    removeBlobs: config.removeBlobs,
-    removeAttributes: config.removeAttributes,
-    stripBuildOriginMetadata: config.stripBuildOriginMetadata,
-    noindex:
-      config.notFoundRoute !== null &&
-      normaliseRoute(route) === normaliseRoute(config.notFoundRoute),
-    preconnectOrigins: config.preconnectThirdParty
-      ? [...result.collector.thirdPartyOrigins].sort()
-      : [],
-    preloadImages: config.preloadImages
-      ? [...result.collector.images].sort().map((image) => toPublicPath(image, config.base))
-      : [],
-  });
+  const needsString =
+    config.inlineCss === 'critical' || Boolean(config.minifyHtml) || Boolean(config.transformHtml);
+  const normalised = normaliseHtml(
+    result.html,
+    {
+      removeStyleTags: config.removeStyleTags,
+      removeScriptTags: config.removeScriptTags,
+      asyncScriptTags: config.asyncScriptTags,
+      removeBlobs: config.removeBlobs,
+      removeAttributes: config.removeAttributes,
+      stripBuildOriginMetadata: config.stripBuildOriginMetadata,
+      noindex:
+        config.notFoundRoute !== null &&
+        normaliseRoute(route) === normaliseRoute(config.notFoundRoute),
+      preconnectOrigins: config.preconnectThirdParty
+        ? [...result.collector.thirdPartyOrigins].sort()
+        : [],
+      preloadImages: config.preloadImages
+        ? [...result.collector.images].sort().map((image) => toPublicPath(image, config.base))
+        : [],
+    },
+    { serialize: needsString },
+  );
+  const { stats } = normalised;
+  let document = normalised.document;
+  let preTransformScripts = new Set();
+  let transformedHtml = null;
+  let localeChanged = false;
+  if (needsString) {
+    let output = normalised.html;
+    if (config.inlineCss === 'critical') {
+      output = await applyCriticalCss({ output, outputDir, config, log, state, caches });
+    }
+    if (config.minifyHtml) {
+      output = await minifyHtml(output, config.minifyHtml, config.minifyCss);
+    }
+    if (config.transformHtml) {
+      preTransformScripts = collectScriptSourcesFromDocument(document);
+      const transformed = await config.transformHtml(output, {
+        route,
+        file: routeToFile(route, config),
+        stats,
+        config,
+      });
+      if (transformed !== null && transformed !== undefined && typeof transformed !== 'string') {
+        throw new TypeError('transformHtml must return a string, null or undefined');
+      }
+      output = transformed ?? output;
+    }
+    transformedHtml = output;
+    document = parse(output);
+  }
+  const noindex =
+    documentHasNoindex(document) ||
+    (config.notFoundRoute !== null &&
+      normaliseRoute(baseRoute) === normaliseRoute(config.notFoundRoute));
+  const localeLinks =
+    locale && localePlan ? buildLocaleLinks({ locale, baseRoute, plan: localePlan, config }) : null;
+  if (localeLinks) {
+    localeChanged = true;
+    const seo = resolveSeoLocale(config, locale, route);
+    applyLocaleMetadata(document, localeLinks, {
+      skipAlternates: noindex,
+      ...seoLocaleMetadataOptions(seo),
+    });
+    if (seo?.inject) {
+      const script = buildSeoLocaleScript(seo);
+      const external = config.csp === 'strict' || (config.externalScripts && config.csp !== 'hash');
+      if (external) {
+        const externalHref = await ensureLocaleScript({
+          script,
+          outputDir,
+          config,
+          report,
+          log,
+          cache: localeScripts,
+        });
+        if (externalHref) injectSeoLocaleScript(document, script, externalHref);
+      } else {
+        injectSeoLocaleScript(document, script, null);
+      }
+    }
+  }
   if (
     stats.removedElements > 0 ||
     stats.removedAttributes > 0 ||
@@ -198,11 +359,49 @@ async function writeRouteOutput({ route, result, outputDir, config, log, state,
     log.debug(`  inline css: left ${result.inlineCssSkipped.length} stylesheet(s) as links`);
   }
 
-  let output = html;
-  if (config.inlineCss === 'critical') {
-    output = await applyCriticalCss({ output, outputDir, config, log, state });
+  let preloadAdded = 0;
+  if (config.preloadLinks) {
+    const basePrefix = config.base === '/' ? '' : config.base.slice(0, -1);
+    const skipScripts = new Set(
+      [...preTransformScripts].map((src) => {
+        if (!src.startsWith('/')) return src;
+        const appRelative =
+          basePrefix && (src === basePrefix || src.startsWith(`${basePrefix}/`))
+            ? src.slice(basePrefix.length)
+            : src;
+        return toPublicPath(appRelative, config.base);
+      }),
+    );
+    preloadAdded = injectPreloadLinksInto(document, {
+      scripts: config.removeScriptTags ? [] : [...result.collector.scripts].sort(),
+      fonts: new Map(
+        [...result.collector.fonts].sort(([left], [right]) => left.localeCompare(right)),
+      ),
+      lcpImage: result.lcpImage,
+      base: config.base,
+      ignoreForPreload: config.ignoreForPreload,
+      skipScripts,
+    });
   }
-  if (config.minifyHtml) {
+  let cspChanged = false;
+  if (config.csp === 'hash') {
+    const policy = applyCspHashesToDocument(document);
+    cspChanged = !policy.existingPolicy;
+    if (policy.existingPolicy) {
+      log.warn(`  ${route}: kept the existing CSP meta and injected no hashes`);
+      report.cspSkipped.push(route);
+    } else {
+      log.debug(
+        `  csp ${route}: hashed ${policy.hashedScripts} script(s) and ${policy.hashedStyles} style(s)`,
+      );
+    }
+  }
+  const meta = collectPageMetaFromDocument(document);
+  let output =
+    transformedHtml !== null && preloadAdded === 0 && !cspChanged && !localeChanged
+      ? transformedHtml
+      : serialize(document);
+  if (localeChanged && config.minifyHtml) {
     output = await minifyHtml(output, config.minifyHtml, config.minifyCss);
   }
   warnOnBuildOriginMetadata({
@@ -212,18 +411,18 @@ async function writeRouteOutput({ route, result, outputDir, config, log, state,
     log,
     state,
   });
-  if (stats.titleElements > 1) report.duplicateTitles.push(route);
+  if (meta.titleCount > 1) report.duplicateTitles.push(route);
+  const buildOrigins = findBuildOriginUrls(output);
   if (config.failOnBuildOriginMetadata) {
-    const origins = findBuildOriginUrls(output);
-    if (origins.length > 0) {
-      const message = `metadata still points at the build server: ${origins.join(', ')}`;
+    if (buildOrigins.length > 0) {
+      const message = `metadata still points at the build server: ${buildOrigins.join(', ')}`;
       report.errors.push({ route, message });
       log.error(`  ${route}: ${message}`);
     }
   }
   const file = await writeRouteHtml({ dir: outputDir, route, html: output, config });
   const resources = config.preloadManifest
-    ? collectPageResources(output, { base: config.base, route })
+    ? collectPageResourcesFromDocument(document, { base: config.base, route })
     : null;
   return {
     file: {
@@ -233,7 +432,16 @@ async function writeRouteOutput({ route, result, outputDir, config, log, state,
       hash: createHash('sha256')
         .update(result.html ?? output)
         .digest('hex'),
-      noindex: stats.noindex,
+      noindex: meta.noindex,
+      notFound: result.notFound === true,
+      meta,
+      images: meta.images,
+      alternates: meta.alternates,
+      localeUrl: locale
+        ? localePageUrl(locale, baseRoute, config, localePlan?.mode ?? 'path')
+        : null,
+      inlineStyleAttributes: meta.inlineStyleAttributes,
+      buildOriginUrls: buildOrigins,
     },
     resources,
   };
@@ -243,13 +451,14 @@ async function writeRouteOutput({ route, result, outputDir, config, log, state,
  * Inlines critical CSS, or explains once why it cannot: the strategy reads stylesheets from
  * the output directory, which a command server does not have to serve.
  */
-async function applyCriticalCss({ output, outputDir, config, log, state }) {
+async function applyCriticalCss({ output, outputDir, config, log, state, caches }) {
   if (!config.serveCmd) {
     return inlineCriticalCss({
       html: output,
       outputDir,
       base: config.base,
       preload: config.criticalCssPreload,
+      cache: caches?.criticalCss ?? null,
     });
   }
   if (!state.warnedCriticalCssSkipped) {
@@ -261,6 +470,59 @@ async function applyCriticalCss({ output, outputDir, config, log, state }) {
   return output;
 }
 
+/**
+ * Reports whether the document already declares a noindex policy. Locale alternates are
+ * skipped on those pages, while language attributes are still applied.
+ * @param {object} document
+ * @returns {boolean}
+ */
+function documentHasNoindex(document) {
+  let noindex = false;
+  const walk = (node) => {
+    if (noindex) return;
+    if (
+      node.tagName === 'meta' &&
+      node.attrs?.some((entry) => entry.name === 'name' && entry.value.toLowerCase() === 'robots')
+    ) {
+      const content = node.attrs.find((entry) => entry.name === 'content')?.value ?? '';
+      if (/\b(noindex|none)\b/i.test(content)) noindex = true;
+    }
+    for (const child of node.childNodes ?? []) walk(child);
+  };
+  walk(document);
+  return noindex;
+}
+
+/**
+ * Writes the external SEO locale bootstrap once per unique script. The file is
+ * content-addressed, so reruns and locales that produce the same script share it.
+ * @returns {Promise<string | null>} public href, or null when the write failed
+ */
+async function ensureLocaleScript({ script, outputDir, config, report, log, cache }) {
+  const hash = createHash('sha256').update(script).digest('hex').slice(0, 16);
+  const file = `snappy-seo-locale-${hash}.js`;
+  if (cache.has(file)) return cache.get(file);
+  const pending = (async () => {
+    const bytes = Buffer.byteLength(script);
+    if (config.dryRun) {
+      report.files.push({ route: null, file, status: 'dry-run', bytes });
+      return toPublicPath(`/${file}`, config.base);
+    }
+    try {
+      const written = await writeFileIfChanged({ dir: outputDir, file, contents: script });
+      report.files.push({ route: null, ...written });
+      log.debug(`  wrote ${file}: ${written.status}`);
+      return toPublicPath(`/${file}`, config.base);
+    } catch (error) {
+      report.errors.push({ route: null, message: `locale script write failed: ${error.message}` });
+      log.error(`  write failed for ${file}: ${error.message}`);
+      return null;
+    }
+  })();
+  cache.set(file, pending);
+  return pending;
+}
+
 /**
  * Warns once when metadata contains the build server's origin, which means the page was
  * composed without a public siteUrl and og:url, og:image or canonical would ship wrong.

package/src/head/index.js

@@ -6,13 +6,103 @@ const useIsomorphicLayoutEffect = typeof window === 'undefined' ? useEffect : us
 
 let sequence = 0;
 
+/**
+ * Attribute names and values accepted by html, body, title and arbitrary head tags.
+ * @typedef {Object<string, string | number | boolean | null | undefined>} HeadAttributes
+ */
+
+/**
+ * One arbitrary head element.
+ * @typedef {Object} HeadTag
+ * @property {'script' | 'style' | 'noscript' | 'base'} tag
+ * @property {HeadAttributes} [attrs]
+ * @property {string | number} [text]
+ * @property {string} [html] Trusted inner HTML for noscript only.
+ * @property {string} [key]
+ */
+
+/**
+ * One hreflang alternate.
+ * @typedef {Object} HeadAlternate
+ * @property {string} href
+ * @property {string} [hreflang]
+ * @property {string} [lang]
+ * @property {string} [type]
+ */
+
+/**
+ * Extra meta or link attributes. keys and duplicates are consumed by the head layer.
+ * @typedef {Object<string, string | number | boolean | null | undefined>} HeadExtra
+ */
+
+/**
+ * One merged head entry reported to onChangeClientState.
+ * @typedef {Object} HeadEntry
+ * @property {string} kind
+ * @property {string} key
+ * @property {Record<string, string>} [attrs]
+ * @property {string} [text]
+ * @property {string} [html]
+ * @property {string} [value]
+ */
+
+/**
+ * Entries grouped by kind for onChangeClientState.
+ * @typedef {Object} HeadChangeState
+ * @property {HeadEntry | null} title
+ * @property {HeadEntry[]} meta
+ * @property {HeadEntry[]} link
+ * @property {HeadEntry[]} script
+ * @property {HeadEntry[]} style
+ * @property {HeadEntry[]} noscript
+ * @property {HeadEntry[]} base
+ * @property {HeadEntry[]} htmlAttributes
+ * @property {HeadEntry[]} bodyAttributes
+ * @property {HeadEntry[]} titleAttributes
+ */
+
+/**
+ * @typedef {(state: HeadChangeState, added: HeadEntry[], removed: HeadEntry[]) => void} HeadChange
+ */
+
+/**
+ * Props accepted by Head and setHead.
+ * @typedef {Object} HeadProps
+ * @property {string} [title]
+ * @property {string} [defaultTitle]
+ * @property {string} [titleTemplate]
+ * @property {string} [siteName]
+ * @property {string} [description]
+ * @property {string} [robots]
+ * @property {string} [type]
+ * @property {string} [locale]
+ * @property {string | false} [canonical]
+ * @property {string} [image]
+ * @property {string[]} [images]
+ * @property {string[]} [alternateLocales]
+ * @property {string} [lang]
+ * @property {'ltr' | 'rtl' | 'auto'} [dir]
+ * @property {boolean} [openGraph]
+ * @property {boolean} [twitter]
+ * @property {HeadAlternate[]} [alternates]
+ * @property {unknown} [structuredData]
+ * @property {HeadExtra[]} [extra]
+ * @property {HeadTag[]} [tags]
+ * @property {HeadAttributes} [htmlAttributes]
+ * @property {HeadAttributes} [bodyAttributes]
+ * @property {HeadAttributes} [titleAttributes]
+ * @property {HeadChange} [onChangeClientState]
+ */
+
 /**
  * Declares document metadata for the route being rendered: title, description, robots,
- * canonical, Open Graph and Twitter card tags. The head is reconciled on every render while
- * the component is mounted, so client-side navigation gives each route its own head exactly
- * like a site that renders a page per request.
+ * canonical, Open Graph and Twitter card tags, plus arbitrary head tags and html, body and
+ * title attributes. The head is reconciled on every render while the component is mounted,
+ * so client-side navigation gives each route its own head exactly like a site that renders a
+ * page per request.
  *
  * Renders nothing, so it never affects hydration of the page content.
+ * @param {HeadProps} props
  */
 export function Head(props) {
   const idRef = useRef(null);
@@ -26,12 +116,12 @@ export function Head(props) {
     sequenceRef.current = sequence;
   }
 
-  const entries = buildHeadEntries(props, readMetadataDefaults());
+  const entries = buildHeadEntries(props, readMetadataDefaults(), idRef.current);
 
   // A layout effect so a route change updates the head before the browser paints, and a
   // plain effect on the server, where layout effects never run.
   useIsomorphicLayoutEffect(() => {
-    registerHead(idRef.current, sequenceRef.current, entries);
+    registerHead(idRef.current, sequenceRef.current, entries, props.onChangeClientState ?? null);
   });
 
   useEffect(() => () => unregisterHead(idRef.current), []);
@@ -42,14 +132,22 @@ export function Head(props) {
 /**
  * Imperative equivalent for code that is not a React component, for example a vanilla
  * router or an analytics callback.
+ * @param {string} id
+ * @param {HeadProps} props
  */
 export function setHead(id, props) {
   sequence += 1;
-  registerHead(id, sequence, buildHeadEntries(props, readMetadataDefaults()));
+  registerHead(
+    id,
+    sequence,
+    buildHeadEntries(props, readMetadataDefaults(), id),
+    props.onChangeClientState ?? null,
+  );
 }
 
 /**
  * Removes metadata previously set with setHead.
+ * @param {string} id
  */
 export function clearHead(id) {
   unregisterHead(id);