snappy-prerender 2.6.0-rc.2

package/src/core/routes.js

@@ -0,0 +1,150 @@
+/**
+ * Canonicalises a route to an app-relative path without query, hash or trailing
+ * slash. Dot segments are resolved so crawled links can never escape the output
+ * directory through path traversal.
+ */
+export function normaliseRoute(input) {
+  let route = String(input).trim();
+  const hashIndex = route.indexOf('#');
+  if (hashIndex !== -1) route = route.slice(0, hashIndex);
+  const queryIndex = route.indexOf('?');
+  if (queryIndex !== -1) route = route.slice(0, queryIndex);
+  if (!route.startsWith('/')) route = `/${route}`;
+
+  const segments = [];
+  for (const segment of route.split('/')) {
+    if (segment === '' || segment === '.') continue;
+    if (segment === '..') {
+      segments.pop();
+      continue;
+    }
+    segments.push(segment);
+  }
+  return `/${segments.join('/')}`;
+}
+
+/**
+ * Matches a route against one include/exclude pattern. Strings match exactly,
+ * strings ending in `*` match by prefix, and RegExp patterns are tested directly.
+ */
+export function matchesPattern(route, pattern) {
+  // A global or sticky RegExp keeps lastIndex between calls, so it is reset around each test.
+  if (pattern instanceof RegExp) {
+    pattern.lastIndex = 0;
+    const matched = pattern.test(route);
+    pattern.lastIndex = 0;
+    return matched;
+  }
+  if (pattern.endsWith('*')) return route.startsWith(pattern.slice(0, -1));
+  return route === pattern;
+}
+
+// Reports whether any pattern in a list matches the route.
+function matchesAny(route, patterns) {
+  return patterns.some((pattern) => matchesPattern(route, pattern));
+}
+/**
+ * Reports whether a route is safe to map onto the output directory. Literal dot
+ * segments are already resolved, so this catches the encoded forms: percent-encoded
+ * traversal and backslashes, which would otherwise create junk directories and act
+ * as path separators on Windows.
+ */
+export function isSafeRoute(route) {
+  if (route.includes('\\')) return false;
+  const lower = route.toLowerCase();
+  const suspicious = lower.includes('%2e') || lower.includes('%2f') || lower.includes('%5c');
+  if (!suspicious) return true;
+
+  let decoded;
+  try {
+    decoded = decodeURIComponent(route);
+  } catch {
+    return false;
+  }
+  if (decoded.includes('\\')) return false;
+  return !decoded.split('/').some((segment) => segment === '..' || segment === '.');
+}
+
+/**
+ * Applies include/exclude filtering and de-duplication to a list of candidate
+ * routes. Include only filters when the user supplied patterns beyond the default.
+ */
+export function filterRoutes(routes, { include, exclude }) {
+  const seen = new Set();
+  const kept = [];
+  for (const candidate of routes) {
+    const route = normaliseRoute(candidate);
+    if (seen.has(route)) continue;
+    if (!isSafeRoute(route)) continue;
+    if (include.length > 0 && !matchesAny(route, include)) continue;
+    if (exclude.length > 0 && matchesAny(route, exclude)) continue;
+    seen.add(route);
+    kept.push(route);
+  }
+  return kept;
+}
+
+/**
+ * Extracts the app-relative route from an absolute link found in the page. Returns
+ * null for non-http links or links outside the configured base path.
+ */
+export function toAppRoute(href, base) {
+  let url;
+  try {
+    url = new URL(href);
+  } catch {
+    return null;
+  }
+  if (url.protocol !== 'http:' && url.protocol !== 'https:') return null;
+
+  let pathname = url.pathname;
+  if (base !== '/') {
+    const prefix = base.slice(0, -1);
+    if (pathname === prefix) pathname = '/';
+    else if (pathname.startsWith(`${prefix}/`)) pathname = pathname.slice(prefix.length);
+    else return null;
+  }
+  return normaliseRoute(pathname);
+}
+
+/**
+ * Builds the absolute URL the browser should visit for an app-relative route,
+ * re-applying the configured base path.
+ */
+export function toServerUrl(route, base, origin) {
+  return `${origin}${toPublicPath(route, base)}`;
+}
+
+/**
+ * Re-applies the base path to an app-relative path, for hints that the browser
+ * resolves against the served origin rather than the app root.
+ */
+export function toPublicPath(pathname, base) {
+  const prefix = base === '/' ? '' : base.slice(0, -1);
+  return `${prefix}${pathname}`;
+}
+
+/**
+ * Reports whether a pattern can be requested as a literal route. Wildcard and
+ * RegExp patterns only ever act as allowlist filters, never as crawl seeds.
+ */
+export function isConcreteRoute(pattern) {
+  return typeof pattern === 'string' && !pattern.includes('*');
+}
+
+/**
+ * Resolves the crawl roots. Concrete include entries are seeded directly; when the
+ * include list only holds patterns, crawling starts from the root route. Excluded
+ * routes are dropped here too, so a seed cannot bypass the exclude list.
+ */
+export function crawlRoots({ include, exclude = [] }) {
+  const concrete = include.filter(isConcreteRoute);
+  return filterRoutes(concrete.length > 0 ? concrete : ['/'], { include: [], exclude });
+}
+
+/**
+ * Reports whether a route still needs visiting given the crawl depth limit.
+ */
+export function withinDepth(depth, maxDepth) {
+  return maxDepth === null || depth <= maxDepth;
+}

package/src/core/run.js

@@ -0,0 +1,523 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { launchBrowser } from './browser.js';
+import { resolveConfig } from './config.js';
+import { prepareDestination } from './destination.js';
+import { inlineCriticalCss } from './inline-css.js';
+import { createLogger } from './log.js';
+import { minifyHtml } from './minify.js';
+import { normaliseHtml } from './normalize.js';
+import {
+  findOutputCollisions,
+  routeToScreenshotFile,
+  writeFileIfChanged,
+  writeRouteHtml,
+} from './output.js';
+import { buildPreloadManifest } from './preload-manifest.js';
+import { renderRoute } from './render.js';
+import { crawlRoots, filterRoutes, normaliseRoute, toPublicPath, withinDepth } from './routes.js';
+import { createPool } from './scheduler.js';
+import { startCommandServer, startStaticServer } from './server.js';
+import { verifyRoutes } from './verify.js';
+
+/**
+ * Entry point of the prerenderer: serves the built output, crawls and renders
+ * routes in a real browser, post-processes and writes one file per route, then
+ * verifies hydration. Returns a report instead of throwing so callers decide how to
+ * fail.
+ */
+export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
+  const config = resolveConfig(userOptions);
+  const log = injectedLog ?? createLogger(config.logLevel);
+  for (const warning of config.warnings ?? []) log.warn(warning);
+  const sourceDir = path.resolve(config.sourceDir);
+  const startedAt = Date.now();
+
+  await assertSourceDir(sourceDir, config);
+  const outputDir = await resolveOutputDir({ sourceDir, config, log });
+
+  if (config.saveAs !== 'html' && config.verify) {
+    log.warn(`saveAs: '${config.saveAs}' writes images, so hydration verification is skipped`);
+  }
+
+  const server = config.serveCmd
+    ? await startCommandServer({
+        command: config.serveCmd,
+        url: config.url,
+        timeout: config.serveCmdTimeout,
+        shutdownTimeout: config.shutdownTimeout,
+      })
+    : await startStaticServer({ dir: outputDir, base: config.base });
+
+  const report = {
+    routes: [],
+    files: [],
+    errors: [],
+    pageErrors: [],
+    duplicateTitles: [],
+    verification: null,
+    preloadManifest: null,
+    truncated: null,
+    ok: false,
+    durationMs: 0,
+  };
+  const manifestEntries = [];
+  const state = { warnedBuildOriginMetadata: false, warnedCriticalCssSkipped: false };
+  let browser = null;
+
+  try {
+    const launched = await launchBrowser(config, log);
+    browser = launched.browser;
+    log.info(`Browser: ${launched.source}`);
+
+    const rendered = await renderAllRoutes({
+      browser,
+      server,
+      config,
+      log,
+      report,
+      manifestEntries,
+      outputDir,
+    });
+    const pageErrors = report.pageErrors.length;
+    log.info(
+      `Rendered ${report.routes.length} route(s)${pageErrors > 0 ? `, ${pageErrors} page error(s)` : ''}`,
+    );
+    if (pageErrors > 0 && !config.failOnPageError) {
+      log.warn(
+        'Page errors are reported, not enforced, so the generated HTML still shipped. Set failOnPageError to fail the build instead.',
+      );
+    }
+
+    const verifiable = await writeAllRoutes({ rendered, outputDir, config, log, report, state });
+    logFileSummary(report, log);
+    logNotFoundNotice(report, config, log);
+    logDuplicateTitles(report, log);
+
+    if (report.truncated) {
+      log.error(
+        `maxRoutes (${report.truncated.limit}) reached after ${report.truncated.routes} route(s); output is incomplete`,
+      );
+    }
+
+    if (config.preloadManifest && !config.dryRun) {
+      report.preloadManifest = await writePreloadManifest({
+        manifestEntries,
+        outputDir,
+        config,
+        log,
+      });
+    }
+
+    const canVerify = config.verify && !config.dryRun && config.saveAs === 'html';
+    if (canVerify && config.serveCmd) {
+      log.warn(
+        'hydration verification is skipped with serveCmd: the command server decides which document it serves, so the written files cannot be checked',
+      );
+    }
+    if (canVerify && !config.serveCmd && verifiable.length > 0) {
+      report.verification = await verifyRoutes({
+        browser,
+        origin: server.origin,
+        routes: verifiable.sort(),
+        config,
+      });
+      logVerification(report.verification, config, log);
+    }
+  } finally {
+    if (browser) await browser.close();
+    const stopped = await server.close();
+    if (stopped === false) {
+      log.warn('serveCmd process did not exit after SIGKILL and may still be running');
+    }
+  }
+
+  report.durationMs = Date.now() - startedAt;
+  const reRendered = report.verification?.modes['re-rendered'] ?? 0;
+  report.ok =
+    !(config.failOnError && (report.errors.length > 0 || report.truncated !== null)) &&
+    !(config.failOnPageError && report.pageErrors.length > 0) &&
+    !(config.failOnHydrationError && report.verification && !report.verification.ok) &&
+    !(config.failOnRerender && reRendered > 0);
+  return report;
+}
+
+/**
+ * Fails with a clear message when the output directory is missing, instead of
+ * letting every route fail later with an opaque browser navigation error.
+ */
+async function assertSourceDir(sourceDir, config) {
+  if (config.serveCmd) return;
+  const stats = await fs.stat(sourceDir).catch(() => null);
+  if (!stats?.isDirectory()) {
+    throw new Error(`sourceDir does not exist or is not a directory: ${sourceDir}`);
+  }
+}
+
+/**
+ * Resolves where generated files go. A destination is copied from the source first
+ * so assets sit beside the generated HTML; with serveCmd there is nothing to serve
+ * from disk, so the destination is only used as the write target.
+ */
+async function resolveOutputDir({ sourceDir, config, log }) {
+  if (config.serveCmd) return path.resolve(config.destination ?? config.sourceDir);
+  return prepareDestination({
+    sourceDir,
+    destination: config.destination ? path.resolve(config.destination) : null,
+    log,
+  });
+}
+
+/**
+ * Runs the crawl and render pool. Each rendered page contributes its output, any new
+ * same-origin routes, and the resources recorded for the link-hint features.
+ */
+async function renderAllRoutes({
+  browser,
+  server,
+  config,
+  log,
+  report,
+  manifestEntries,
+  outputDir,
+}) {
+  const visited = new Set();
+  const scheduled = new Set();
+  const rendered = new Map();
+  const cap = config.maxRoutes ?? Number.POSITIVE_INFINITY;
+
+  // Queues routes up to maxRoutes, flagging the report the first time the cap bites.
+  const accept = (routes) => {
+    const accepted = [];
+    for (const route of routes) {
+      if (scheduled.has(route)) continue;
+      if (scheduled.size >= cap) {
+        report.truncated = { limit: config.maxRoutes, routes: scheduled.size };
+        break;
+      }
+      scheduled.add(route);
+      accepted.push(route);
+    }
+    return accepted;
+  };
+
+  const pool = createPool({
+    concurrency: config.concurrency,
+    handler: async ({ route, depth }) => {
+      if (visited.has(route)) return;
+      visited.add(route);
+      log.debug(`rendering ${route}`);
+      try {
+        const screenshotPath =
+          config.saveAs !== 'html' && !config.dryRun
+            ? path.join(outputDir, routeToScreenshotFile(route, config))
+            : null;
+        const result = await renderRoute({
+          browser,
+          origin: server.origin,
+          route,
+          config,
+          screenshotPath,
+        });
+        rendered.set(route, result);
+        report.routes.push(route);
+        manifestEntries.push({
+          route,
+          scripts: [...result.collector.scripts],
+          styles: [...result.collector.styles],
+        });
+        for (const message of result.pageErrors) {
+          report.pageErrors.push({ route, message });
+          log.warn(`  page error on ${route}: ${message}`);
+        }
+        if (config.crawl && withinDepth(depth + 1, config.maxDepth)) {
+          const discovered = filterRoutes(result.links, {
+            include: config.includeProvided ? config.include : [],
+            exclude: config.exclude,
+          }).filter((candidate) => !scheduled.has(candidate));
+          pool.push(
+            accept(discovered).map((candidate) => ({ route: candidate, depth: depth + 1 })),
+          );
+        }
+      } catch (error) {
+        report.errors.push({ route, message: error.message });
+        log.error(`  failed ${route}: ${error.message}`);
+      }
+    },
+  });
+
+  const roots = accept(crawlRoots(config));
+  pool.push(roots.map((route) => ({ route, depth: 0 })));
+  const { errors } = await pool.run();
+  for (const { item, error } of errors) {
+    report.errors.push({ route: item.route, message: error.message });
+    log.error(`  pool error on ${item.route}: ${error.message}`);
+  }
+  return rendered;
+}
+
+/**
+ * Writes every rendered route and returns the routes that can be verified. A write
+ * failure is recorded against its route instead of aborting the remaining writes.
+ */
+async function writeAllRoutes({ rendered, outputDir, config, log, report, state }) {
+  const sorted = [...report.routes].sort();
+  const conflicted = reportOutputCollisions({ routes: sorted, config, log, report });
+
+  await writeStateScripts({ rendered, outputDir, config, log, report });
+
+  const verifiable = [];
+  for (const route of sorted) {
+    if (conflicted.has(route)) continue;
+    try {
+      const file = await writeRouteOutput({
+        route,
+        result: rendered.get(route),
+        outputDir,
+        config,
+        log,
+        state,
+        report,
+      });
+      report.files.push(file);
+      verifiable.push(route);
+    } catch (error) {
+      report.errors.push({ route, message: `write failed: ${error.message}` });
+      log.error(`  write failed for ${route}: ${error.message}`);
+    }
+  }
+  return verifiable;
+}
+
+/**
+ * Writes the captured-state scripts routes reference when externalScripts is on. The payload
+ * is content-addressed, so routes that inject the same state share one file and a rerun
+ * leaves it untouched.
+ */
+async function writeStateScripts({ rendered, outputDir, config, log, report }) {
+  const seen = new Set();
+  for (const [route, result] of rendered) {
+    const stateScript = result?.stateScript;
+    if (!stateScript || seen.has(stateScript.file)) continue;
+    seen.add(stateScript.file);
+    if (config.dryRun) {
+      report.files.push({
+        route,
+        file: stateScript.file,
+        status: 'dry-run',
+        bytes: Buffer.byteLength(stateScript.contents),
+      });
+      continue;
+    }
+    try {
+      const written = await writeFileIfChanged({
+        dir: outputDir,
+        file: stateScript.file,
+        contents: stateScript.contents,
+      });
+      report.files.push({ route, ...written });
+      log.debug(`  wrote ${stateScript.file}: ${written.status}`);
+    } catch (error) {
+      report.errors.push({ route, message: `write failed: ${error.message}` });
+      log.error(`  write failed for ${stateScript.file}: ${error.message}`);
+    }
+  }
+}
+
+/**
+ * Records an error for every route involved in an output file collision and returns
+ * the routes to skip, so an ambiguous write can never overwrite another route.
+ */
+function reportOutputCollisions({ routes, config, log, report }) {
+  const conflicted = new Set();
+  for (const [file, colliding] of findOutputCollisions(routes, config)) {
+    for (const route of colliding) {
+      conflicted.add(route);
+      const others = colliding.filter((other) => other !== route).join(', ');
+      report.errors.push({
+        route,
+        message: `output file collision: ${file} is also written by ${others}`,
+      });
+    }
+    log.error(`  output file collision on ${file}: ${colliding.join(', ')}`);
+  }
+  return conflicted;
+}
+
+/**
+ * Applies the configured post-processing to one rendered route and writes it: DOM
+ * cleanups and link hints, critical CSS extraction, then optional minification.
+ */
+async function writeRouteOutput({ route, result, outputDir, config, log, state, report }) {
+  if (result.screenshot) {
+    const file = routeToScreenshotFile(route, config);
+    const stats = await fs.stat(path.join(outputDir, file));
+    return { route, file, status: 'written', bytes: stats.size };
+  }
+
+  const { html, stats } = normaliseHtml(result.html, {
+    removeStyleTags: config.removeStyleTags,
+    removeScriptTags: config.removeScriptTags,
+    asyncScriptTags: config.asyncScriptTags,
+    removeBlobs: config.removeBlobs,
+    preconnectOrigins: config.preconnectThirdParty
+      ? [...result.collector.thirdPartyOrigins].sort()
+      : [],
+    preloadImages: config.preloadImages
+      ? [...result.collector.images].sort().map((image) => toPublicPath(image, config.base))
+      : [],
+  });
+  if (
+    stats.removedElements > 0 ||
+    stats.dedupedStyles > 0 ||
+    stats.hints > 0 ||
+    stats.textSeparators > 0
+  ) {
+    log.debug(
+      `  cleaned ${route}: ${stats.removedElements} element(s) removed, ${stats.dedupedStyles} duplicate style(s), ${stats.hints} hint(s), ${stats.textSeparators} text separator(s)`,
+    );
+  }
+  if ((result.inlineCssSkipped ?? []).length > 0) {
+    log.debug(`  inline css: left ${result.inlineCssSkipped.length} stylesheet(s) as links`);
+  }
+
+  let output = html;
+  if (config.inlineCss === 'critical') {
+    output = await applyCriticalCss({ output, outputDir, config, log, state });
+  }
+  if (config.minifyHtml) {
+    output = await minifyHtml(output, config.minifyHtml, config.minifyCss);
+  }
+  warnOnBuildOriginMetadata({ origins: stats.metadataOrigins, route, log, state });
+  if (stats.titleElements > 1) report.duplicateTitles.push(route);
+  return writeRouteHtml({ dir: outputDir, route, html: output, config });
+}
+
+/**
+ * Inlines critical CSS, or explains once why it cannot: the strategy reads stylesheets from
+ * the output directory, which a command server does not have to serve.
+ */
+async function applyCriticalCss({ output, outputDir, config, log, state }) {
+  if (!config.serveCmd) {
+    return inlineCriticalCss({
+      html: output,
+      outputDir,
+      base: config.base,
+      preload: config.criticalCssPreload,
+    });
+  }
+  if (!state.warnedCriticalCssSkipped) {
+    state.warnedCriticalCssSkipped = true;
+    log.warn(
+      "inlineCss: 'critical' is skipped with serveCmd: stylesheets are read from the output directory, which the command server does not have to serve",
+    );
+  }
+  return output;
+}
+
+/**
+ * Warns once when metadata contains the build server's origin, which means the page was
+ * composed without a public siteUrl and og:url, og:image or canonical would ship wrong.
+ */
+function warnOnBuildOriginMetadata({ origins, route, log, state }) {
+  if (state.warnedBuildOriginMetadata || origins.length === 0) return;
+  state.warnedBuildOriginMetadata = true;
+  log.warn(
+    `metadata on ${route} points at the build server (${origins.join(', ')}); set metadata.siteUrl so og:url, og:image and canonical match the deployed site`,
+  );
+}
+
+// Notes routes whose output carries more than one title, which is what React 19 native
+// metadata and the head layer together produce.
+function logDuplicateTitles(report, log) {
+  if (report.duplicateTitles.length === 0) return;
+  const sample = report.duplicateTitles.slice(0, 5).join(', ');
+  log.warn(
+    `${report.duplicateTitles.length} route(s) contain more than one <title> element: ${sample}${report.duplicateTitles.length > 5 ? ', …' : ''}. React 19 native metadata and the head layer can each write one, so use one system.`,
+  );
+}
+
+/**
+ * Writes the preload manifest: per route, the scripts and stylesheets it loaded,
+ * as Link header values a host can serve for Early Hints or HTTP headers.
+ */
+async function writePreloadManifest({ manifestEntries, outputDir, config, log }) {
+  const manifest = buildPreloadManifest(manifestEntries, {
+    ignoreForPreload: config.ignoreForPreload,
+    base: config.base,
+  });
+  const file = 'preload-manifest.json';
+  await fs.writeFile(path.join(outputDir, file), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');
+  log.info(`Wrote preload manifest covering ${manifest.length} route(s)`);
+  return { file, routes: manifest.length };
+}
+
+// Notes when a not-found route is configured but was never rendered, so no 404.html
+// was written and a static host would serve its own default page instead.
+function logNotFoundNotice(report, config, log) {
+  if (!config.notFoundRoute || config.dryRun || config.saveAs !== 'html') return;
+  const notFoundRoute = normaliseRoute(config.notFoundRoute);
+  if (report.routes.includes(notFoundRoute)) return;
+  log.warn(
+    `no 404.html was written: the notFoundRoute (${notFoundRoute}) was not prerendered. Add it to include, or point notFoundRoute at a route you do prerender.`,
+  );
+}
+
+// Prints the write outcome counts, including a dedicated dry-run wording.
+function logFileSummary(report, log) {
+  const written = report.files.filter((file) => file.status === 'written').length;
+  const unchanged = report.files.filter((file) => file.status === 'unchanged').length;
+  const dryRun = report.files.filter((file) => file.status === 'dry-run').length;
+  const failed = report.errors.length > 0 ? `, ${report.errors.length} failed route(s)` : '';
+
+  if (dryRun > 0 && written === 0 && unchanged === 0) {
+    log.info(`Dry run: rendered ${dryRun} file(s), nothing written${failed}`);
+    return;
+  }
+  const parts = [`Wrote ${written} file(s)`];
+  if (unchanged > 0) parts.push(`${unchanged} unchanged`);
+  log.info(parts.join(', ') + failed);
+}
+
+// Reports verification results per route, warns when routes discarded their prerendered
+// markup, surfaces errors the client logged while booting, and notes when hydration failures
+// were reported but not enforced.
+function logVerification(verification, config, log) {
+  if (verification.ok) {
+    log.success(`Hydration verified for ${verification.routes.length} route(s)`);
+  } else {
+    const failed = verification.routes.filter((route) => !route.ok);
+    for (const entry of failed) {
+      log.warn(`Hydration failed on ${entry.route}:`);
+      for (const message of entry.hydrationErrors) log.warn(`  ${message.split('\n')[0]}`);
+    }
+    for (const error of verification.errors) {
+      log.warn(`Verification could not run on ${error.route}: ${error.message}`);
+    }
+    if (failed.length > 0 && !config.failOnHydrationError) {
+      log.warn(
+        `Hydration failures are reported, not enforced, so the prerendered HTML still shipped. Set failOnHydrationError to fail the build instead.`,
+      );
+    }
+  }
+
+  for (const entry of verification.routes) {
+    for (const message of entry.pageErrors) {
+      log.warn(`  page error on ${entry.route} while verifying: ${message.split('\n')[0]}`);
+    }
+    for (const message of entry.consoleErrors.slice(0, 3)) {
+      log.debug(`  console error on ${entry.route}: ${message.split('\n')[0]}`);
+    }
+  }
+
+  const reRendered = verification.routes.filter((entry) => entry.mode === 're-rendered');
+  if (reRendered.length > 0) {
+    const sample = reRendered
+      .slice(0, 5)
+      .map((entry) => entry.route)
+      .join(', ');
+    log.warn(
+      `${reRendered.length} route(s) re-rendered instead of hydrating, so the prerendered markup was discarded on boot: ${sample}${reRendered.length > 5 ? ', …' : ''}`,
+    );
+  }
+}