snappy-prerender 2.6.0 → 2.7.0

package/src/core/audit/index.js

@@ -0,0 +1,114 @@
+import { auditLinks, auditOrphans } from './links.js';
+import { auditMetadata } from './metadata.js';
+import { auditSitemap } from './sitemap.js';
+
+/**
+ * Audits the written site and returns report-only findings. Errors are the things a
+ * static site should not ship (broken links, sitemap disagreement, duplicate titles);
+ * warnings cover metadata that is useful but legitimately optional.
+ */
+export async function auditRun({ report, config, outputDir }) {
+  const errors = [];
+  const warnings = [];
+  const add = (severity, rule, message, details = {}) => {
+    const finding = { rule, severity, message, ...details };
+    if (severity === 'error') errors.push(finding);
+    else warnings.push(finding);
+  };
+
+  const pages = report.routeDetails.filter((detail) => detail.file?.endsWith('.html'));
+  auditRouteHealth({ report, add });
+  auditMetadata({ pages, config, add });
+  auditSitemap({ pages, config, report, add });
+  const linkAudit = await auditLinks({ report, config, outputDir, add });
+  auditOrphans({ pages, report, add });
+  auditCspHygiene({ pages, config, report, add });
+  removeDuplicateFindings({ errors, warnings });
+
+  return {
+    errors,
+    warnings,
+    linksChecked: linkAudit.checked,
+  };
+}
+
+// Mirrors route-level failures into the audit so failOnAuditError covers route health
+// without requiring callers to inspect a second collection.
+function auditRouteHealth({ report, add }) {
+  for (const error of report.errors ?? []) {
+    add('error', 'route-error', error.message, { route: error.route ?? undefined });
+  }
+  if (report.truncated) {
+    add(
+      'error',
+      'route-cap',
+      `maxRoutes (${report.truncated.limit}) stopped the crawl after ${report.truncated.routes} route(s)`,
+      { droppedRoutes: report.truncated.droppedRoutes ?? [] },
+    );
+  }
+  for (const route of report.emptyPages ?? []) {
+    add('error', 'empty-page', 'rendered an empty page', { route });
+  }
+  for (const group of report.duplicatePages ?? []) {
+    for (const route of group.routes) {
+      const others = group.routes.filter((other) => other !== route).join(', ');
+      add('error', 'duplicate-output', `identical output to ${others}`, { route });
+    }
+  }
+}
+
+// Inline style attributes cannot be covered by a CSP hash, and browser console errors
+// reveal assets the generated policy blocked.
+function auditCspHygiene({ pages, config, report, add }) {
+  if (config.csp !== 'hash') return;
+  for (const route of report.cspSkipped ?? []) {
+    add(
+      'warning',
+      'existing-csp-meta',
+      `${route} already has a CSP meta, so no hashes were injected`,
+      { route },
+    );
+  }
+  const violations = new Map();
+  const collect = (route, message) => {
+    if (!/content security policy|violates the following/i.test(message)) return;
+    const messages = violations.get(route) ?? new Set();
+    messages.add(message);
+    violations.set(route, messages);
+  };
+  for (const { route, message } of report.consoleErrors ?? []) collect(route, message);
+  for (const page of pages) {
+    for (const message of page.consoleErrors ?? []) collect(page.route, message);
+  }
+  for (const [route, messages] of violations) {
+    add(
+      'warning',
+      'csp-console-violation',
+      `${route} logged ${messages.size} Content-Security-Policy violation(s)`,
+      { route },
+    );
+  }
+  for (const page of pages) {
+    if (page.meta?.inlineStyleAttributes > 0) {
+      add(
+        'warning',
+        'inline-style-attributes',
+        `${page.route} has ${page.meta.inlineStyleAttributes} inline style attribute(s); CSP hashes cannot cover them`,
+        { route: page.route },
+      );
+    }
+  }
+}
+
+// A finding can be produced by more than one check; keep the first copy.
+function removeDuplicateFindings({ errors, warnings }) {
+  for (const list of [errors, warnings]) {
+    const seen = new Set();
+    for (let index = list.length - 1; index >= 0; index -= 1) {
+      const finding = list[index];
+      const key = `${finding.rule}|${finding.route ?? ''}|${finding.message}`;
+      if (seen.has(key)) list.splice(index, 1);
+      else seen.add(key);
+    }
+  }
+}

package/src/core/audit/links.js

@@ -0,0 +1,63 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { routeToFile } from '../output.js';
+import { matchesPattern, normaliseRoute } from '../routes.js';
+
+/**
+ * Audits every internal link the rendered pages emitted and warns about indexable pages
+ * no other page links to.
+ */
+export async function auditLinks({ report, config, outputDir, add }) {
+  const routes = new Set(report.routeDetails.map((detail) => detail.route));
+  const files = new Set(report.files.map((file) => file.file).filter(Boolean));
+  const cache = new Map();
+  const seenPairs = new Set();
+  let checkedLinks = 0;
+
+  const exists = async (target) => {
+    if (cache.has(target)) return cache.get(target);
+    const result = await fs
+      .stat(path.join(outputDir, target))
+      .then((stats) => stats.isFile())
+      .catch(() => false);
+    cache.set(target, result);
+    return result;
+  };
+
+  for (const link of report.links) {
+    const key = `${link.from}|${link.to}`;
+    if (seenPairs.has(key)) continue;
+    seenPairs.add(key);
+    if (config.auditIgnoreLinks.some((pattern) => matchesPattern(link.to, pattern))) continue;
+    checkedLinks += 1;
+    const route = normaliseRoute(link.to);
+    if (routes.has(route)) continue;
+    const file = routeToFile(route, config);
+    if (files.has(file)) continue;
+    const asset = route.startsWith('/') ? route.slice(1) : route;
+    if (asset && (await exists(asset))) continue;
+    add('error', 'broken-internal-link', `${link.from} links to missing ${link.to}`, {
+      from: link.from,
+      to: link.to,
+      reason: 'missing',
+    });
+  }
+  return { checked: checkedLinks };
+}
+
+/**
+ * Warns about indexable pages no rendered page links to. A link from a page outside the
+ * include list is legitimately absent, so this stays a warning.
+ */
+export function auditOrphans({ pages, report, add }) {
+  const linked = new Set();
+  for (const link of report.links) linked.add(normaliseRoute(link.to));
+  for (const page of pages) {
+    if (page.route === '/' || page.notFound || page.noindex) continue;
+    if (!linked.has(page.route)) {
+      add('warning', 'orphan-page', `${page.route} is not linked from another page`, {
+        route: page.route,
+      });
+    }
+  }
+}

package/src/core/audit/metadata.js

@@ -0,0 +1,88 @@
+import { toPublicPath } from '../routes.js';
+import { pagePath } from '../url-policy.js';
+
+/**
+ * Checks the metadata a crawler reads on every written page: titles, descriptions,
+ * canonicals, headings, build-origin leakage and duplicate values across routes.
+ */
+export function auditMetadata({ pages, config, add }) {
+  const titles = new Map();
+  const descriptions = new Map();
+  for (const page of pages) {
+    const meta = page.meta ?? {};
+    if (!meta.title) {
+      add('warning', 'missing-title', `no <title> on ${page.route}`, { route: page.route });
+    }
+    if (!meta.description) {
+      add('warning', 'missing-description', `no meta description on ${page.route}`, {
+        route: page.route,
+      });
+    }
+    if (!meta.canonical) {
+      add('warning', 'missing-canonical', `no canonical link on ${page.route}`, {
+        route: page.route,
+      });
+    }
+    if (meta.titleCount > 1) {
+      add('error', 'multiple-titles', `${page.route} contains more than one <title>`, {
+        route: page.route,
+      });
+    }
+    if (meta.h1Count !== 1) {
+      add('warning', 'heading-count', `${page.route} has ${meta.h1Count} <h1> element(s)`, {
+        route: page.route,
+      });
+    }
+    if ((page.buildOriginUrls ?? []).length > 0) {
+      add(
+        'warning',
+        'build-origin-metadata',
+        `${page.route} still points at the build server: ${page.buildOriginUrls.join(', ')}`,
+        { route: page.route },
+      );
+    }
+    if (meta.title) titles.set(meta.title, [...(titles.get(meta.title) ?? []), page.route]);
+    if (meta.description) {
+      descriptions.set(meta.description, [
+        ...(descriptions.get(meta.description) ?? []),
+        page.route,
+      ]);
+    }
+    if (config.metadata?.siteUrl && meta.canonical) {
+      const expected = expectedPageUrl(page.route, config);
+      if (expected && meta.canonical !== expected) {
+        add(
+          'warning',
+          'canonical-mismatch',
+          `${page.route} canonical is ${meta.canonical}, expected ${expected}`,
+          { route: page.route },
+        );
+      }
+    }
+  }
+  for (const [title, routes] of titles) {
+    if (routes.length < 2) continue;
+    add('error', 'duplicate-title', `routes share the title "${title}": ${routes.join(', ')}`, {
+      routes,
+    });
+  }
+  for (const [description, routes] of descriptions) {
+    if (routes.length < 2) continue;
+    add(
+      'warning',
+      'duplicate-description',
+      `routes share the meta description "${description}": ${routes.join(', ')}`,
+      { routes },
+    );
+  }
+}
+
+// Resolves the public URL a page should canonicalise and be listed at.
+export function expectedPageUrl(route, config) {
+  if (!config.metadata?.siteUrl) return null;
+  const path = pagePath(route, {
+    flatOutput: config.flatOutput,
+    trailingSlash: config.metadata.trailingSlash,
+  });
+  return `${config.metadata.siteUrl.replace(/\/$/, '')}${toPublicPath(path, config.base)}`;
+}

package/src/core/audit/sitemap.js

@@ -0,0 +1,39 @@
+import { expectedPageUrl } from './metadata.js';
+
+/**
+ * Ensures the sitemap and the indexable pages describe the same set of URLs. A page
+ * missing from the sitemap, a loc with no page, or a noindex page listed are all errors.
+ */
+export function auditSitemap({ pages, config, report, add }) {
+  if (!config.sitemap) return;
+  const indexable = new Map();
+  const noindex = new Map();
+  for (const page of pages) {
+    const loc = expectedPageUrl(page.route, config);
+    if (!loc) continue;
+    if (page.noindex || page.notFound) {
+      noindex.set(loc, page.route);
+      continue;
+    }
+    indexable.set(loc, page.route);
+  }
+  const locs = new Set(report.sitemap?.locs ?? []);
+  for (const [loc, route] of indexable) {
+    if (!locs.has(loc)) {
+      add('error', 'sitemap-missing-page', `${route} is missing from the sitemap`, { route });
+    }
+  }
+  for (const loc of locs) {
+    if (noindex.has(loc)) {
+      add('error', 'sitemap-noindex-page', `sitemap lists noindex ${noindex.get(loc)}`, {
+        route: noindex.get(loc),
+      });
+      continue;
+    }
+    if (!indexable.has(loc)) {
+      add('error', 'sitemap-unknown-page', `sitemap lists ${loc} with no indexable page`, {
+        to: loc,
+      });
+    }
+  }
+}

package/src/core/config.js

@@ -11,7 +11,8 @@ export const MAX_DEFAULT_CONCURRENCY = 8;
 const LOG_LEVELS = ['silent', 'error', 'warn', 'info', 'debug'];
 const BROWSER_CHOICES = ['auto', 'chrome', 'msedge', 'chromium'];
 const INLINE_CSS_STRATEGIES = [false, true, 'inline', 'critical'];
-const CSP_MODES = ['off', 'strict'];
+const CSP_MODES = ['off', 'strict', 'hash'];
+const FRAMEWORK_CHOICES = ['auto', 'react', 'vue', 'svelte', 'preact'];
 const SAVE_AS_CHOICES = ['html', 'png', 'jpeg'];
 const CRITICAL_CSS_PRELOAD = ['body', 'media', 'swap', 'swap-high', 'swap-low', 'js', 'js-lazy'];
 const NULLABLE_KEYS = [
@@ -27,6 +28,10 @@ const NULLABLE_KEYS = [
   'browserDownloadVersion',
   'url',
   'serveCmd',
+  'reportPath',
+  'routesFile',
+  'routesFromSitemap',
+  'transformHtml',
 ];
 const ARRAY_KEYS = [
   'include',
@@ -35,6 +40,9 @@ const ARRAY_KEYS = [
   'browserArgs',
   'ignoreForPreload',
   'removeAttributes',
+  'auditIgnoreLinks',
+  'routePriority',
+  'routeOverrides',
 ];
 
 export const DEFAULTS = Object.freeze({
@@ -87,11 +95,26 @@ export const DEFAULTS = Object.freeze({
   preloadImages: false,
   preloadManifest: false,
   sitemap: false,
+  imageSitemap: false,
   ignoreForPreload: ['service-worker.js'],
   cacheAjaxRequests: false,
   maxCachedBytes: 5 * 1024 * 1024,
   flatOutput: false,
   notFoundRoute: '/404',
+  audit: true,
+  failOnAuditError: false,
+  auditIgnoreLinks: [],
+  reportPath: null,
+  routePriority: [],
+  routeOverrides: [],
+  transformHtml: null,
+  routesFile: null,
+  routesFromSitemap: null,
+  preloadLinks: false,
+  retries: 0,
+  retryDelay: 500,
+  traceOnError: false,
+  framework: 'auto',
   verify: true,
   verifyChangedOnly: false,
   failOnHydrationError: false,
@@ -128,6 +151,9 @@ for (const key of [
   'browserArgs',
   'ignoreForPreload',
   'removeAttributes',
+  'auditIgnoreLinks',
+  'routePriority',
+  'routeOverrides',
 ]) {
   Object.freeze(DEFAULTS[key]);
 }
@@ -156,9 +182,11 @@ export function resolveConfig(userOptions = {}) {
     !(config.include.length === 1 && config.include[0] === '/');
   // The CSP dial decides the defaults for the pieces that can move. Explicit options still
   // win, so a project can take 'strict' and put one part back.
-  if (userOptions.externalScripts === undefined) config.externalScripts = config.csp === 'strict';
+  if (userOptions.externalScripts === undefined) {
+    config.externalScripts = config.csp === 'strict' || config.csp === 'hash';
+  }
   if (userOptions.criticalCssPreload === undefined) {
-    config.criticalCssPreload = config.csp === 'strict' ? false : 'media';
+    config.criticalCssPreload = config.csp === 'strict' || config.csp === 'hash' ? false : 'media';
   }
   config.warnings = [];
   validateConfig(config);
@@ -201,6 +229,32 @@ function addConfigWarnings(config, userOptions) {
         );
       }
     }
+    if (userOptions.framework !== undefined) {
+      warn(
+        `framework has no effect because hydration verification is skipped with ${config.serveCmd ? 'serveCmd' : `saveAs: '${config.saveAs}'`}`,
+      );
+    }
+  }
+  if (!config.verify && userOptions.framework !== undefined) {
+    warn('framework has no effect because verify is false');
+  }
+  if (!config.audit && config.failOnAuditError) {
+    warn('failOnAuditError has no effect because audit is false');
+  }
+  if (!config.audit && config.auditIgnoreLinks.length > 0) {
+    warn('auditIgnoreLinks has no effect because audit is false');
+  }
+  if (config.preloadLinks && config.saveAs !== 'html') {
+    warn(`preloadLinks has no effect when saveAs is '${config.saveAs}'`);
+  }
+  if (config.preloadLinks && config.removeScriptTags) {
+    warn('preloadLinks skips script hints because removeScriptTags strips every script');
+  }
+  if (config.imageSitemap && !config.sitemap) {
+    warn('imageSitemap has no effect because sitemap is false');
+  }
+  if (config.traceOnError && config.dryRun) {
+    warn('traceOnError has no effect in dry-run mode');
   }
   if (
     userOptions.criticalCssPreload !== undefined &&
@@ -221,7 +275,7 @@ function addConfigWarnings(config, userOptions) {
     );
   }
   if (
-    config.csp === 'strict' &&
+    (config.csp === 'strict' || config.csp === 'hash') &&
     userOptions.criticalCssPreload !== undefined &&
     config.criticalCssPreload !== false
   ) {
@@ -305,9 +359,25 @@ export function validateConfig(config) {
   stringList(config.browserArgs, 'browserArgs');
   stringList(config.ignoreForPreload, 'ignoreForPreload');
   patternList(config.removeAttributes, 'removeAttributes');
-  for (const key of ['readyFlag', 'readySelector', 'waitFor', 'notFoundRoute']) nullableString(key);
+  patternList(config.auditIgnoreLinks, 'auditIgnoreLinks');
+  patternList(config.routePriority, 'routePriority');
+  for (const key of [
+    'readyFlag',
+    'readySelector',
+    'waitFor',
+    'notFoundRoute',
+    'reportPath',
+    'routesFile',
+    'routesFromSitemap',
+  ]) {
+    nullableString(key);
+  }
   for (const key of ['userAgent', 'destination', 'browserDownloadVersion']) nullableString(key);
+  if (config.transformHtml !== null && typeof config.transformHtml !== 'function') {
+    throw new TypeError('transformHtml must be a function or null');
+  }
   for (const key of BOOLEAN_KEYS) booleanOption(key);
+  validateRouteOverrides(config.routeOverrides);
 
   if (config.browserDownloadHash !== null && !/^[a-f0-9]{64}$/i.test(config.browserDownloadHash)) {
     throw new TypeError('browserDownloadHash must be a 64 character SHA-256 hex digest');
@@ -333,6 +403,12 @@ export function validateConfig(config) {
   if (config.maxRoutes !== null && (!Number.isInteger(config.maxRoutes) || config.maxRoutes < 1)) {
     throw new TypeError('maxRoutes must be null or a positive integer');
   }
+  if (!Number.isInteger(config.retries) || config.retries < 0) {
+    throw new TypeError('retries must be a non-negative integer');
+  }
+  if (!Number.isFinite(config.retryDelay) || config.retryDelay <= 0) {
+    throw new TypeError('retryDelay must be a positive number of milliseconds');
+  }
   if (!Number.isFinite(config.maxCachedBytes) || config.maxCachedBytes <= 0) {
     throw new TypeError('maxCachedBytes must be a positive number of bytes');
   }
@@ -349,6 +425,9 @@ export function validateConfig(config) {
       `browser must be one of: ${BROWSER_CHOICES.join(', ')} or a path to an executable`,
     );
   }
+  if (!FRAMEWORK_CHOICES.includes(config.framework)) {
+    throw new TypeError(`framework must be one of: ${FRAMEWORK_CHOICES.join(', ')}`);
+  }
   if (config.storageState !== null) {
     if (typeof config.storageState !== 'string' || !fs.existsSync(config.storageState)) {
       throw new TypeError('storageState must point to an existing Playwright storage state file');
@@ -405,6 +484,51 @@ export function validateConfig(config) {
   }
 }
 
+// Per-route overrides may only change the settings that make sense for one route.
+const ROUTE_OVERRIDE_KEYS = new Set([
+  'match',
+  'timeout',
+  'waitFor',
+  'readySelector',
+  'readyFlag',
+  'crawl',
+  'storageState',
+]);
+
+/**
+ * Validates per-route overrides. Matching patterns stay strings or RegExp; every
+ * override key is checked so a typo cannot silently leave the base config in place.
+ */
+function validateRouteOverrides(overrides) {
+  for (const override of overrides) {
+    if (typeof override !== 'object' || override === null || Array.isArray(override)) {
+      throw new TypeError('routeOverrides entries must be objects');
+    }
+    for (const key of Object.keys(override)) {
+      if (!ROUTE_OVERRIDE_KEYS.has(key)) {
+        throw new TypeError(`routeOverrides.${key} is not a recognised option`);
+      }
+    }
+    if (!(typeof override.match === 'string' || override.match instanceof RegExp)) {
+      throw new TypeError('routeOverrides entries need a string or RegExp match');
+    }
+    if (override.timeout !== undefined) {
+      if (!Number.isFinite(override.timeout) || override.timeout <= 0) {
+        throw new TypeError('routeOverrides.timeout must be a positive number of milliseconds');
+      }
+    }
+    for (const key of ['waitFor', 'readySelector', 'readyFlag', 'storageState']) {
+      const value = override[key];
+      if (value !== undefined && typeof value !== 'string') {
+        throw new TypeError(`routeOverrides.${key} must be a string`);
+      }
+    }
+    if (override.crawl !== undefined && typeof override.crawl !== 'boolean') {
+      throw new TypeError('routeOverrides.crawl must be a boolean');
+    }
+  }
+}
+
 // The metadata keys the head component reads, so a typo is reported instead of ignored.
 const METADATA_KEYS = new Set([
   'siteUrl',