snappy-prerender 2.6.0-rc.2 → 2.6.0

package/src/cli.js

@@ -30,16 +30,33 @@ Options:
       --max-depth <n>         Crawl depth limit, 0 renders roots only (default: unlimited)
       --max-routes <n>        Stop after this many routes and fail the run (default: unlimited)
       --concurrency <n>       Parallel browser pages (default: derived from CPU count)
-      --timeout <ms>          Per-route timeout (default: 30000)
+      --timeout <ms>          Per-route navigation/settle timeout (default: 30000)
+      --quiet-period <ms>     Network must be idle this long before capture (default: 500)
+      --scroll-step-delay <ms>  Delay between scroll steps (default: 100)
+      --viewport <WxH>        Browser viewport (default: 1280x720)
+      --ready-selector <sel>  Selector awaited when set
+      --ready-flag <name>     Window flag awaited when defined (default: __prerenderReady)
+      --no-headless           Run the browser visibly
       --wait-for <selector>   Wait for a selector on every route before capture
       --browser <choice>      auto | chrome | msedge | chromium | path (default: auto)
       --no-browser-download   Refuse the Chrome for Testing fallback download
       --browser-download-hash <sha256>  Pin the fallback download to a SHA-256 digest
+      --browser-download-version <build>  Pin the Chrome for Testing build id
       --storage-state <file>  Playwright storage state for authenticated routes
       --no-verify             Skip the hydration verification pass
+      --verify-changed-only   Skip verification for routes whose HTML did not change
       --fail-on-hydration-error  Fail the build when hydration errors are found
       --fail-on-rerender      Fail when a route re-renders instead of hydrating
       --fail-on-page-error    Fail when a page throws while it is being rendered
+      --no-fail-on-error      Report route errors without failing the run
+      --fail-on-empty-page    Fail when a route renders no app content (default: on)
+      --no-fail-on-empty-page Allow an empty page and report it instead
+      --fail-on-console-error Fail when the browser logs a console error while rendering
+      --fail-on-failed-request  Fail when a same-origin request fails while rendering
+      --fail-on-duplicate-content  Fail when two routes produce identical HTML
+      --fail-on-build-origin-metadata  Fail when metadata still points at the prerender server
+      --fail-on-undetected    Fail when verification cannot find a framework root
+      --fail-on-verification-error  Fail when the verification pass cannot run
       --block-third-party     Block third-party requests while rendering
       --no-block-third-party  Allow third-party requests (the default)
       --allowed-hosts <hosts>  Third-party hosts to allow while blocking the rest, comma separated
@@ -50,6 +67,7 @@ Options:
       --flat                  Write about.html instead of about/index.html
       --not-found <route>     Route emitted as 404.html (default: /404)
       --destination <dir>     Write output to another directory (default: in place)
+      --clean-destination     Empty the destination before copying the build output
       --save-as <format>      html | png | jpeg (default: html)
       --user-agent <value>    User agent used while rendering (default: SnappyPrerender)
       --metadata-site-url <url>       Public site URL for canonical and og:url
@@ -57,6 +75,7 @@ Options:
       --metadata-title-template <t>   Title template, for example '%s | Example'
       --metadata-default-image <url>  Share image for routes without one
       --metadata-trailing-slash <mode>  preserve | always | never
+      --metadata-base <path>  Base path used for absolute metadata URLs
       --inline-css <strategy> inline | critical (critical needs the beasties package)
       --critical-css-preload <mode>  Swap for deferred CSS: none | body | media | swap | swap-high | swap-low | js | js-lazy
       --minify-html           Minify the generated HTML
@@ -65,18 +84,25 @@ Options:
       --no-preconnect-third-party  Disable preconnect hints
       --preload-images        Add preload hints for images on the page
       --preload-manifest      Write preload-manifest.json with Link header hints
+      --sitemap               Write sitemap.xml from the rendered routes (needs metadata.siteUrl)
       --ignore-for-preload <names>  File names excluded from the manifest, comma separated
       --cache-ajax-requests   Expose captured JSON responses as window.snapStore
+      --max-cached-bytes <n>  Largest JSON response cached per request (default: 5242880)
       --remove-scripts        Strip every script tag from the output
       --remove-styles         Strip every style tag from the output
       --async-scripts         Mark external scripts async
       --external-scripts      Write injected state scripts as same-origin files, for a strict CSP
       --csp <mode>            off | strict: strict makes the output work under script-src 'self'
       --no-remove-blobs       Keep blob stylesheet links instead of dropping them
+      --remove-attributes <names>  Drop these attributes from the output, comma separated
+      --no-strip-build-origin-metadata  Keep tags that point at the prerender server
+      --prune-stale-routes    Delete pages listed by the previous run's manifest
       --ignore-https-errors   Ignore TLS errors while rendering
       --browser-args <args>   Extra browser launch arguments, comma separated
       --url <url>             App URL to connect to when using --serve-cmd
       --serve-cmd <command>   Command that starts the app server instead of static serving
+      --serve-cmd-timeout <ms>  How long to wait for the command server (default: 30000)
+      --shutdown-timeout <ms>  Grace period before force-killing the server (default: 5000)
       --dry-run               Render and report without writing files
       --log-level <level>     silent | error | warn | info | debug (default: info)
       --yes, -y               Accept every default, for a shell without a terminal
@@ -102,15 +128,34 @@ async function main() {
       'max-routes': { type: 'string' },
       concurrency: { type: 'string' },
       timeout: { type: 'string' },
+      'quiet-period': { type: 'string' },
+      'scroll-step-delay': { type: 'string' },
+      'max-cached-bytes': { type: 'string' },
+      'serve-cmd-timeout': { type: 'string' },
+      'shutdown-timeout': { type: 'string' },
+      viewport: { type: 'string' },
+      'ready-flag': { type: 'string' },
+      'ready-selector': { type: 'string' },
+      headless: { type: 'boolean' },
       'wait-for': { type: 'string' },
       browser: { type: 'string' },
       'no-browser-download': { type: 'boolean' },
       'browser-download-hash': { type: 'string' },
+      'browser-download-version': { type: 'string' },
       'storage-state': { type: 'string' },
       'no-verify': { type: 'boolean' },
+      'verify-changed-only': { type: 'boolean' },
       'fail-on-hydration-error': { type: 'boolean' },
+      'fail-on-error': { type: 'boolean' },
       'fail-on-rerender': { type: 'boolean' },
       'fail-on-page-error': { type: 'boolean' },
+      'fail-on-empty-page': { type: 'boolean' },
+      'fail-on-console-error': { type: 'boolean' },
+      'fail-on-failed-request': { type: 'boolean' },
+      'fail-on-duplicate-content': { type: 'boolean' },
+      'fail-on-build-origin-metadata': { type: 'boolean' },
+      'fail-on-undetected': { type: 'boolean' },
+      'fail-on-verification-error': { type: 'boolean' },
       'no-block-third-party': { type: 'boolean' },
       'block-third-party': { type: 'boolean' },
       'allowed-hosts': { type: 'string', multiple: true },
@@ -121,6 +166,7 @@ async function main() {
       flat: { type: 'boolean' },
       'not-found': { type: 'string' },
       destination: { type: 'string' },
+      'clean-destination': { type: 'boolean' },
       'save-as': { type: 'string' },
       'user-agent': { type: 'string' },
       'metadata-site-url': { type: 'string' },
@@ -128,14 +174,17 @@ async function main() {
       'metadata-title-template': { type: 'string' },
       'metadata-default-image': { type: 'string' },
       'metadata-trailing-slash': { type: 'string' },
+      'metadata-base': { type: 'string' },
       'inline-css': { type: 'string' },
       'critical-css-preload': { type: 'string' },
       csp: { type: 'string' },
       'minify-html': { type: 'boolean' },
       'minify-css': { type: 'boolean' },
       'no-preconnect-third-party': { type: 'boolean' },
+      'preconnect-third-party': { type: 'boolean' },
       'preload-images': { type: 'boolean' },
       'preload-manifest': { type: 'boolean' },
+      sitemap: { type: 'boolean' },
       'ignore-for-preload': { type: 'string', multiple: true },
       'cache-ajax-requests': { type: 'boolean' },
       'remove-scripts': { type: 'boolean' },
@@ -143,6 +192,9 @@ async function main() {
       'async-scripts': { type: 'boolean' },
       'external-scripts': { type: 'boolean' },
       'no-remove-blobs': { type: 'boolean' },
+      'remove-attributes': { type: 'string', multiple: true },
+      'no-strip-build-origin-metadata': { type: 'boolean' },
+      'prune-stale-routes': { type: 'boolean' },
       'ignore-https-errors': { type: 'boolean' },
       'browser-args': { type: 'string', multiple: true },
       url: { type: 'string' },
@@ -199,17 +251,52 @@ function cliOptions({ values, positionals }) {
   if (values['max-routes'] !== undefined) options.maxRoutes = Number(values['max-routes']);
   if (values.concurrency !== undefined) options.concurrency = Number(values.concurrency);
   if (values.timeout !== undefined) options.timeout = Number(values.timeout);
+  if (values['quiet-period'] !== undefined) options.quietPeriod = Number(values['quiet-period']);
+  if (values['scroll-step-delay'] !== undefined) {
+    options.scrollStepDelay = Number(values['scroll-step-delay']);
+  }
+  if (values['max-cached-bytes'] !== undefined) {
+    options.maxCachedBytes = Number(values['max-cached-bytes']);
+  }
+  if (values['serve-cmd-timeout'] !== undefined) {
+    options.serveCmdTimeout = Number(values['serve-cmd-timeout']);
+  }
+  if (values['shutdown-timeout'] !== undefined) {
+    options.shutdownTimeout = Number(values['shutdown-timeout']);
+  }
+  if (values.viewport !== undefined) {
+    const match = /^(\d+)x(\d+)$/i.exec(values.viewport);
+    if (!match) throw new Error('viewport must look like 1280x720');
+    options.viewport = { width: Number(match[1]), height: Number(match[2]) };
+  }
+  if (values['ready-selector'] !== undefined) options.readySelector = values['ready-selector'];
+  if (values['ready-flag'] !== undefined) options.readyFlag = values['ready-flag'];
+  if (values.headless !== undefined) options.headless = values.headless;
   if (values['wait-for'] !== undefined) options.waitFor = values['wait-for'];
   if (values.browser !== undefined) options.browser = values.browser;
   if (values['no-browser-download']) options.browserDownload = false;
   if (values['browser-download-hash'] !== undefined) {
     options.browserDownloadHash = values['browser-download-hash'];
   }
+  if (values['browser-download-version'] !== undefined) {
+    options.browserDownloadVersion = values['browser-download-version'];
+  }
   if (values['storage-state'] !== undefined) options.storageState = values['storage-state'];
   if (values['no-verify']) options.verify = false;
+  if (values['verify-changed-only']) options.verifyChangedOnly = true;
   if (values['fail-on-hydration-error']) options.failOnHydrationError = true;
   if (values['fail-on-rerender']) options.failOnRerender = true;
   if (values['fail-on-page-error']) options.failOnPageError = true;
+  if (values['fail-on-error'] !== undefined) options.failOnError = values['fail-on-error'];
+  if (values['fail-on-empty-page'] !== undefined) {
+    options.failOnEmptyPage = values['fail-on-empty-page'];
+  }
+  if (values['fail-on-console-error']) options.failOnConsoleError = true;
+  if (values['fail-on-failed-request']) options.failOnFailedRequest = true;
+  if (values['fail-on-duplicate-content']) options.failOnDuplicateContent = true;
+  if (values['fail-on-build-origin-metadata']) options.failOnBuildOriginMetadata = true;
+  if (values['fail-on-undetected']) options.failOnUndetected = true;
+  if (values['fail-on-verification-error']) options.failOnVerificationError = true;
   if (values['no-block-third-party']) options.blockThirdParty = false;
   if (values['block-third-party']) options.blockThirdParty = true;
   if (values['allowed-hosts'] !== undefined)
@@ -221,6 +308,7 @@ function cliOptions({ values, positionals }) {
   if (values.flat) options.flatOutput = true;
   if (values['not-found'] !== undefined) options.notFoundRoute = values['not-found'];
   if (values.destination !== undefined) options.destination = values.destination;
+  if (values['clean-destination']) options.cleanDestination = true;
   if (values['save-as'] !== undefined) options.saveAs = values['save-as'];
   if (values['user-agent'] !== undefined) options.userAgent = values['user-agent'];
   const metadata = metadataOptions(values);
@@ -234,8 +322,10 @@ function cliOptions({ values, positionals }) {
   if (values['minify-html']) options.minifyHtml = true;
   if (values['minify-css']) options.minifyCss = true;
   if (values['no-preconnect-third-party']) options.preconnectThirdParty = false;
+  if (values['preconnect-third-party']) options.preconnectThirdParty = true;
   if (values['preload-images']) options.preloadImages = true;
   if (values['preload-manifest']) options.preloadManifest = true;
+  if (values.sitemap) options.sitemap = true;
   if (values['ignore-for-preload'] !== undefined) {
     options.ignoreForPreload = splitList(values['ignore-for-preload']);
   }
@@ -245,6 +335,11 @@ function cliOptions({ values, positionals }) {
   if (values['async-scripts']) options.asyncScriptTags = true;
   if (values['external-scripts']) options.externalScripts = true;
   if (values['no-remove-blobs']) options.removeBlobs = false;
+  if (values['remove-attributes'] !== undefined) {
+    options.removeAttributes = splitList(values['remove-attributes']);
+  }
+  if (values['no-strip-build-origin-metadata']) options.stripBuildOriginMetadata = false;
+  if (values['prune-stale-routes']) options.pruneStaleRoutes = true;
   if (values['ignore-https-errors']) options.ignoreHTTPSErrors = true;
   if (values['browser-args'] !== undefined) options.browserArgs = splitList(values['browser-args']);
   if (values.url !== undefined) options.url = values.url;
@@ -287,14 +382,14 @@ async function runInit({ values }) {
     force: values.force === true,
   });
   log.success(`Wrote ${CONFIG_FILE}`);
-  printNextSteps({ detected, log });
+  printNextSteps({ detected, log, tag: readVersion().includes('-') ? '@next' : '' });
 }
 
 // Turns the flags the CLI already understands into the wizard's starting answers.
 function defaultsFromFlags(detected, flags) {
   return {
     sourceDir: flags.sourceDir ?? detected.sourceDir,
-    base: flags.base ?? '/',
+    base: flags.base ?? detected.base ?? '/',
     include: flags.include ?? ['/'],
     crawl: flags.crawl ?? true,
     siteUrl: flags.metadata?.siteUrl ?? '',
@@ -377,9 +472,9 @@ function validatedAnswers(answers, log) {
 }
 
 // Prints what to do next: install if needed, run it, and where the head layer goes.
-function printNextSteps({ detected, log }) {
+function printNextSteps({ detected, log, tag = '' }) {
   const run = runnerCommand(detected.runner, PACKAGE_NAME);
-  const install = installCommand(detected.runner);
+  const install = installCommand(detected.runner, tag);
   log.info('');
   log.info('Next steps:');
   if (!detected.installed) log.info(`  Install      ${install}`);
@@ -408,10 +503,11 @@ function runnerCommand(runner, binary) {
 }
 
 // Builds the command that installs this package as a dev dependency.
-function installCommand(runner) {
-  if (runner === 'pnpm') return `pnpm add -D ${PACKAGE_NAME}`;
-  if (runner === 'yarn') return `yarn add -D ${PACKAGE_NAME}`;
-  return `npm install --save-dev ${PACKAGE_NAME}`;
+function installCommand(runner, tag = '') {
+  const pkg = `${PACKAGE_NAME}${tag}`;
+  if (runner === 'pnpm') return `pnpm add -D ${pkg}`;
+  if (runner === 'yarn') return `yarn add -D ${pkg}`;
+  return `npm install --save-dev ${pkg}`;
 }
 
 // Collects the metadata flags, or null when none was passed so a config file value survives.
@@ -430,12 +526,15 @@ function metadataOptions(values) {
   if (values['metadata-trailing-slash'] !== undefined) {
     metadata.trailingSlash = values['metadata-trailing-slash'];
   }
+  if (values['metadata-base'] !== undefined) metadata.base = values['metadata-base'];
   return Object.keys(metadata).length > 0 ? metadata : null;
 }
 
 /**
  * Loads a JS config file. An explicitly requested file must exist; the implicit
- * snappy.config.js is optional. The default export must be a plain options object.
+ * snappy.config.js is optional. The default export is either a plain options object or an
+ * async function that returns one, which is how routes are computed from a CMS, a sitemap or
+ * a file listing without a separate prebuild script.
  */
 async function loadConfigFile(explicitPath) {
   if (explicitPath === undefined) {
@@ -445,9 +544,17 @@ async function loadConfigFile(explicitPath) {
     throw new Error(`Config file not found: ${explicitPath}`);
   }
   const module = await import(pathToFileURL(explicitPath).href);
-  const loaded = module.default ?? {};
+  const exported = module.default ?? {};
+  let loaded;
+  try {
+    loaded = typeof exported === 'function' ? await exported() : exported;
+  } catch (error) {
+    throw new Error(`Config file failed to load: ${error.message}`);
+  }
   if (typeof loaded !== 'object' || loaded === null || Array.isArray(loaded)) {
-    throw new Error(`Config file must export a plain options object: ${explicitPath}`);
+    throw new Error(
+      `Config file must export a plain options object or a function returning one: ${explicitPath}`,
+    );
   }
   return loaded;
 }

package/src/core/artifacts.js

@@ -0,0 +1,110 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { writeFileIfChanged } from './output.js';
+import { buildPreloadManifest } from './preload-manifest.js';
+import { toPublicPath } from './routes.js';
+import { buildSitemap, withSitemapDirective } from './sitemap.js';
+
+/**
+ * Writes the preload manifest: per route, the scripts and stylesheets the final HTML
+ * references, as Link header values a host can serve for Early Hints or HTTP headers.
+ */
+export async function writePreloadManifest({ manifestEntries, outputDir, config, log, report }) {
+  const manifest = buildPreloadManifest(manifestEntries, {
+    ignoreForPreload: config.ignoreForPreload,
+    base: config.base,
+    flatOutput: config.flatOutput,
+    trailingSlash: config.metadata?.trailingSlash,
+    notFoundRoute: config.notFoundRoute,
+  });
+  const file = 'preload-manifest.json';
+  const contents = `${JSON.stringify(manifest, null, 2)}\n`;
+  if (config.dryRun) {
+    report.files.push({ route: null, file, status: 'dry-run', bytes: Buffer.byteLength(contents) });
+    log.info(`Dry run: would write preload manifest covering ${manifest.length} route(s)`);
+    return { file, routes: manifest.length, status: 'dry-run' };
+  }
+  const written = await writeFileIfChanged({ dir: outputDir, file, contents });
+  report.files.push({ route: null, ...written });
+  log.info(
+    `${written.status === 'unchanged' ? 'Unchanged' : 'Wrote'} preload manifest covering ${manifest.length} route(s)`,
+  );
+  return { file, routes: manifest.length, status: written.status };
+}
+
+/**
+ * Writes sitemap.xml from the routes the run produced, and points an existing robots.txt at
+ * it. Every loc must be absolute, which is why the option needs metadata.siteUrl.
+ */
+export async function writeSitemap({ outputDir, config, log, report }) {
+  if (config.saveAs !== 'html') {
+    log.warn(
+      `sitemap is skipped when saveAs writes ${config.saveAs} files, as there are no pages to list`,
+    );
+    return;
+  }
+  const sitemap = buildSitemap({ files: report.files, config });
+  if (!sitemap) {
+    log.warn('sitemap is skipped: no pages were written');
+    return;
+  }
+
+  const siteUrl = config.metadata.siteUrl.replace(/\/$/, '');
+  const sitemapUrl = `${siteUrl}${toPublicPath('/sitemap.xml', config.base)}`;
+  const robots = await fs.readFile(path.join(outputDir, 'robots.txt'), 'utf8').catch(() => null);
+  const patched = robots === null ? null : withSitemapDirective(robots, sitemapUrl);
+  const names = sitemap.files.map((entry) => entry.name);
+
+  if (config.dryRun) {
+    for (const entry of sitemap.files) {
+      report.files.push({
+        route: null,
+        file: entry.name,
+        status: 'dry-run',
+        bytes: Buffer.byteLength(entry.contents),
+      });
+    }
+    report.sitemap = {
+      file: 'sitemap.xml',
+      routes: sitemap.routes,
+      robotsUpdated: patched !== null,
+      files: names,
+    };
+    log.info(`Dry run: would write ${names.join(', ')} listing ${sitemap.routes} route(s)`);
+    return;
+  }
+
+  const statuses = [];
+  for (const entry of sitemap.files) {
+    const written = await writeFileIfChanged({
+      dir: outputDir,
+      file: entry.name,
+      contents: entry.contents,
+    });
+    statuses.push(written.status);
+    report.files.push({ route: null, ...written });
+  }
+  report.sitemap = {
+    file: 'sitemap.xml',
+    routes: sitemap.routes,
+    robotsUpdated: patched !== null,
+    files: names,
+  };
+  log.info(
+    `${statuses.every((status) => status === 'unchanged') ? 'Unchanged' : 'Wrote'} ${names.join(', ')} listing ${sitemap.routes} route(s)`,
+  );
+
+  if (robots === null) {
+    log.info(`Add "Sitemap: ${sitemapUrl}" to a robots.txt so crawlers find it`);
+    return;
+  }
+  if (patched !== null) {
+    const written = await writeFileIfChanged({
+      dir: outputDir,
+      file: 'robots.txt',
+      contents: patched,
+    });
+    report.files.push({ route: null, ...written });
+    log.info('Added the sitemap location to robots.txt');
+  }
+}

package/src/core/blocking.js

@@ -1,3 +1,36 @@
+/**
+ * Reports whether a host is on the allowlist. Entries may be a hostname, a hostname
+ * with a port, a leading-dot or wildcard suffix, or a full URL; matching is
+ * case-insensitive.
+ */
+export function isHostAllowed(hostname, port, allowedHosts) {
+  const host = hostname.toLowerCase();
+  const withPort = port ? `${host}:${port}` : host;
+  for (const raw of allowedHosts) {
+    const entry = String(raw).trim().toLowerCase();
+    if (entry.length === 0) continue;
+    if (entry.includes('://')) {
+      try {
+        const parsed = new URL(entry);
+        if (parsed.hostname.toLowerCase() === host && (!parsed.port || parsed.port === port)) {
+          return true;
+        }
+      } catch {
+        // A malformed entry is simply not a match.
+      }
+      continue;
+    }
+    if (entry === withPort || entry === host) return true;
+    if (entry.startsWith('*.') && host.endsWith(entry.slice(1)) && host !== entry.slice(2)) {
+      return true;
+    }
+    if (entry.startsWith('.') && host.endsWith(entry) && host !== entry.slice(1)) {
+      return true;
+    }
+  }
+  return false;
+}
+
 /**
  * Aborts requests to third-party origins so analytics, ads and external widgets
  * cannot mutate the DOM mid-capture or make output non-deterministic. Used by both
@@ -15,11 +48,30 @@ export async function blockThirdPartyRequests(page, origin, allowedHosts) {
       return allow(route);
     }
     if (parsed.origin === origin) return allow(route);
-    if (allowed.has(parsed.hostname.toLowerCase())) return allow(route);
+    if (isHostAllowed(parsed.hostname, parsed.port, allowed)) return allow(route);
     // A route can already be handled or cancelled, and that rejection is not this hook's to
     // report, so it is swallowed rather than surfacing as an unhandled rejection.
     return route.abort().catch(() => {});
   });
+
+  // page.route does not see WebSockets; route them separately when the Playwright
+  // version provides the hook.
+  if (typeof page.routeWebSocket === 'function') {
+    await page.routeWebSocket('**/*', (socket) => {
+      let parsed;
+      try {
+        parsed = new URL(socket.url());
+      } catch {
+        socket.close();
+        return;
+      }
+      if (parsed.origin === origin || isHostAllowed(parsed.hostname, parsed.port, allowed)) {
+        socket.connectToServer();
+      } else {
+        socket.close();
+      }
+    });
+  }
 }
 
 // Continues a request, ignoring the rejection that follows an already-handled route.