snappy-prerender 2.6.0 → 2.7.0

package/src/core/report.js

@@ -0,0 +1,108 @@
+import path from 'node:path';
+import { writeFileIfChanged } from './output.js';
+
+/**
+ * Writes the machine-readable run report. The JSON is deliberately limited to route,
+ * audit and artifact data; configuration and captured state are never serialised.
+ */
+export async function writeReport({ report, config, log }) {
+  if (!config.reportPath) return null;
+  const contents = `${JSON.stringify(buildReport(report, config), null, 2)}\n`;
+  if (config.dryRun) {
+    log.info(`Dry run: would write report to ${config.reportPath}`);
+    return { file: config.reportPath, status: 'dry-run' };
+  }
+  const target = path.resolve(config.reportPath);
+  const written = await writeFileIfChanged({
+    dir: path.dirname(target),
+    file: path.basename(target),
+    contents,
+  });
+  log.info(`Wrote report to ${config.reportPath}`);
+  return { file: config.reportPath, status: written.status };
+}
+
+/**
+ * Shapes the public report document. `routes` here is an array of per-route records;
+ * the programmatic `report.routes` stays a list of route strings. URLs are reduced to
+ * origin and path, and the configured storage-state path is redacted everywhere, so a
+ * report can be attached to a build log without leaking credentials or session data.
+ */
+export function buildReport(report, config = null) {
+  const audit = report.audit ?? { errors: [], warnings: [], linksChecked: 0 };
+  const document = {
+    version: 1,
+    generatedAt: new Date().toISOString(),
+    durationMs: report.durationMs,
+    ok: report.ok,
+    summary: {
+      routes: report.routeDetails.length,
+      files: report.files.filter((file) => file.status !== 'dry-run').length,
+      errors: uniqueErrorCount(report, audit),
+      warnings: audit.warnings.length,
+    },
+    routes: report.routeDetails,
+    verification: report.verification
+      ? {
+          ok: report.verification.ok,
+          modes: report.verification.modes,
+          frameworks: report.verification.frameworks,
+          skipped: report.verification.skipped ?? 0,
+          errors: report.verification.errors,
+        }
+      : null,
+    audit,
+    sitemap: report.sitemap,
+    preloadManifest: report.preloadManifest,
+    truncated: report.truncated,
+    notFoundRoutes: report.notFoundRoutes,
+    duplicateOutputs: report.duplicatePages,
+    routeSources: report.routeSources ?? [],
+    cspSkipped: report.cspSkipped ?? [],
+    traces: report.traces ?? [],
+  };
+  return sanitiseValue(document, config);
+}
+
+// Counts route errors once even when the audit mirrors them as findings.
+function uniqueErrorCount(report, audit) {
+  const seen = new Set();
+  for (const error of [...(report.errors ?? []), ...(audit.errors ?? [])]) {
+    seen.add(`${error.route ?? ''}|${error.message ?? ''}`);
+  }
+  return seen.size;
+}
+
+// Recursively redacts URLs and the configured storage-state path from report strings.
+function sanitiseValue(value, config) {
+  if (Array.isArray(value)) return value.map((entry) => sanitiseValue(entry, config));
+  if (value !== null && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, entry]) => [key, sanitiseValue(entry, config)]),
+    );
+  }
+  if (typeof value !== 'string') return value;
+  let text = value;
+  if (config?.storageState) {
+    for (const secret of [config.storageState, path.resolve(config.storageState)]) {
+      text = text.split(secret).join('[redacted]');
+    }
+  }
+  return text.replace(/https?:\/\/[^\s"'<>]+/gi, redactUrl);
+}
+
+// Removes credentials, query strings and fragments from an absolute URL.
+function redactUrl(raw) {
+  const trailing = /[.,;)\]]+$/.exec(raw)?.[0] ?? '';
+  const urlText = trailing ? raw.slice(0, -trailing.length) : raw;
+  try {
+    const url = new URL(urlText);
+    url.username = '';
+    url.password = '';
+    url.search = '';
+    url.hash = '';
+    return `${url.href}${trailing}`;
+  } catch {
+    return raw;
+  }
+}

package/src/core/resources.js

@@ -18,6 +18,7 @@ export function createResourceCollector({
   const images = new Set();
   const scripts = new Set();
   const styles = new Set();
+  const fonts = new Map();
   const json = new Map();
   const pending = [];
 
@@ -26,6 +27,7 @@ export function createResourceCollector({
     images,
     scripts,
     styles,
+    fonts,
     json,
     // Waits for in-flight JSON body reads so the cache is complete before injection.
     async settled() {
@@ -57,7 +59,9 @@ export function createResourceCollector({
         if (resourceType === 'image') images.add(path);
         else if (resourceType === 'script') scripts.add(path);
         else if (resourceType === 'stylesheet') styles.add(path);
-        else if (collectJson && isJson(response)) {
+        else if (resourceType === 'font') {
+          fonts.set(path, response.headers()['content-type'] ?? null);
+        } else if (collectJson && isJson(response)) {
           const length = Number(response.headers()['content-length'] ?? Number.NaN);
           if (Number.isFinite(length) && length > maxCachedBytes) return;
           const capture = response

package/src/core/route-config.js

@@ -0,0 +1,17 @@
+import { matchesPattern } from './routes.js';
+
+/**
+ * Resolves one route's effective config by applying matching routeOverrides in order.
+ * Later entries win per key; the base config is never mutated.
+ */
+export function resolveRouteConfig(config, route) {
+  let resolved = config;
+  for (const override of config.routeOverrides ?? []) {
+    if (!matchesPattern(route, override.match)) continue;
+    const values = Object.fromEntries(
+      Object.entries(override).filter(([key, value]) => key !== 'match' && value !== undefined),
+    );
+    resolved = { ...resolved, ...values };
+  }
+  return resolved;
+}

package/src/core/route-sources.js

@@ -0,0 +1,209 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { isSafeRoute, normaliseRoute, toAppRoute } from './routes.js';
+
+const MAX_SOURCE_ROUTES = 100000;
+const FETCH_TIMEOUT = 10000;
+const XML_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
+
+/**
+ * Loads concrete route seeds from a text/JSON file and/or a sitemap. These are seeds,
+ * not a router: directory-style app paths still have to be handled by the app.
+ */
+export async function loadRouteSeeds(config) {
+  const routes = [];
+  const sources = [];
+  if (config.routesFile) {
+    const read = await readRouteFile(config.routesFile, config);
+    routes.push(...read.routes);
+    sources.push({
+      source: config.routesFile,
+      type: 'file',
+      routes: read.routes.length,
+      skipped: read.skipped,
+    });
+  }
+  if (config.routesFromSitemap) {
+    const read = await readRouteSitemap(config.routesFromSitemap, config);
+    routes.push(...read.routes);
+    sources.push({
+      source: config.routesFromSitemap,
+      type: 'sitemap',
+      routes: read.routes.length,
+      skipped: read.skipped,
+    });
+  }
+  const unique = [...new Set(routes)];
+  if (unique.length > MAX_SOURCE_ROUTES) {
+    throw new Error(`route sources produced more than ${MAX_SOURCE_ROUTES} routes`);
+  }
+  return { routes: unique, sources };
+}
+
+// Reads a text or JSON route list. Text accepts one route per line and # comments.
+async function readRouteFile(file, config) {
+  const contents = await fs.readFile(path.resolve(file), 'utf8');
+  let values;
+  const trimmed = contents.trim();
+  if (trimmed.startsWith('[') || trimmed.startsWith('{')) {
+    let parsed;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch (error) {
+      throw new Error(`routesFile is not valid JSON: ${error.message}`);
+    }
+    values = Array.isArray(parsed) ? parsed : parsed?.routes;
+    if (!Array.isArray(values)) {
+      throw new Error('routesFile JSON must be an array or an object with a routes array');
+    }
+  } else {
+    values = trimmed
+      .split(/\r?\n/)
+      .map((line) => line.replace(/#.*$/, '').trim())
+      .filter((line) => line.length > 0);
+  }
+  return normaliseAndFilter(values, 'routesFile', config);
+}
+
+// Reads a sitemap, following a sitemap index one level deep.
+async function readRouteSitemap(location, config) {
+  const pageUrls = [];
+  await readSitemapLevel(location, pageUrls, 0);
+  const sourceOrigin = /^https?:\/\//i.test(location) ? new URL(location).origin : null;
+  return normaliseAndFilter(pageUrls, 'routesFromSitemap', config, sourceOrigin, true);
+}
+
+async function readSitemapLevel(location, pageUrls, depth) {
+  if (depth > 1) return;
+  const { text, base } = await readSource(location);
+  if (!/<(?:urlset|sitemapindex)\b/i.test(text)) {
+    throw new Error(`${location} is not a sitemap: no urlset or sitemapindex element found`);
+  }
+  if (/<sitemapindex\b/i.test(text)) {
+    for (const child of extractLocs(text)) {
+      await readSitemapLevel(resolveChild(child, base), pageUrls, depth + 1);
+    }
+    return;
+  }
+  pageUrls.push(...extractLocs(text));
+}
+
+// Reads a local path or an http(s) sitemap.
+async function readSource(location) {
+  if (/^https?:\/\//i.test(location)) {
+    const response = await fetch(location, {
+      signal: AbortSignal.timeout(FETCH_TIMEOUT),
+      headers: { accept: 'application/xml,text/xml,*/*' },
+    });
+    if (!response.ok) throw new Error(`sitemap ${location} answered HTTP ${response.status}`);
+    return { text: await response.text(), base: location };
+  }
+  const file = path.resolve(location);
+  return { text: await fs.readFile(file, 'utf8'), base: file };
+}
+
+// Resolves a child sitemap location relative to its parent.
+function resolveChild(child, base) {
+  if (/^https?:\/\//i.test(child)) return child;
+  if (/^https?:\/\//i.test(base)) return new URL(child, base).href;
+  return path.resolve(path.dirname(base), child);
+}
+
+// Extracts and unescapes every <loc> value.
+function extractLocs(xml) {
+  const values = [];
+  for (const match of xml.matchAll(/<loc\b[^>]*>([\s\S]*?)<\/loc>/gi)) {
+    values.push(unescapeXml(match[1].trim()));
+  }
+  return values;
+}
+
+function unescapeXml(value) {
+  return value.replace(/&(amp|lt|gt|quot|apos);/g, (_, name) => XML_ENTITIES[name]);
+}
+
+// Converts source values to app-relative routes and drops unsafe or off-origin entries.
+function normaliseAndFilter(
+  values,
+  source,
+  config = null,
+  sourceOrigin = null,
+  enforceOrigin = false,
+) {
+  const siteOrigin = config?.metadata?.siteUrl ? new URL(config.metadata.siteUrl).origin : null;
+  const referenceOrigin = siteOrigin ?? sourceOrigin;
+  const base = config?.base ?? '/';
+  const routes = [];
+  const seen = new Set();
+  let skipped = 0;
+  for (const value of values) {
+    if (typeof value !== 'string') {
+      skipped += 1;
+      continue;
+    }
+    const absolute = parseAbsolute(value, referenceOrigin);
+    if (/^https?:\/\//i.test(value) && !absolute) {
+      skipped += 1;
+      continue;
+    }
+    let route;
+    if (absolute) {
+      if (!referenceOrigin) {
+        if (enforceOrigin) {
+          skipped += 1;
+          continue;
+        }
+      } else if (absolute.origin !== referenceOrigin) {
+        skipped += 1;
+        continue;
+      }
+      route = toAppRoute(absolute.href, base);
+    } else if (value.startsWith('//')) {
+      // A protocol-relative URL needs an origin to resolve against; without one it is
+      // not safe to guess which host it belongs to.
+      skipped += 1;
+      continue;
+    } else {
+      route = normaliseRoute(stripBase(value, base));
+    }
+    if (!route || !isSafeRoute(route) || seen.has(route)) {
+      skipped += 1;
+      continue;
+    }
+    seen.add(route);
+    routes.push(route);
+  }
+  if (routes.length === 0 && values.length > 0) {
+    throw new Error(`${source} did not contain any usable routes`);
+  }
+  return { routes, skipped };
+}
+
+// Resolves an absolute or protocol-relative URL, returning null for paths.
+function parseAbsolute(value, referenceOrigin) {
+  if (/^https?:\/\//i.test(value)) {
+    try {
+      return new URL(value);
+    } catch {
+      return null;
+    }
+  }
+  if (value.startsWith('//') && referenceOrigin) {
+    try {
+      return new URL(value, referenceOrigin);
+    } catch {
+      return null;
+    }
+  }
+  return null;
+}
+
+// Removes the public base prefix from a root-relative path, leaving app-relative paths
+// alone so a sitemap can list either form.
+function stripBase(value, base) {
+  if (base === '/') return value;
+  const prefix = base.slice(0, -1);
+  if (value === prefix) return '/';
+  if (value.startsWith(`${prefix}/`)) return value.slice(prefix.length);
+  return value;
+}

package/src/core/run.js

@@ -1,21 +1,17 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { writePreloadManifest, writeSitemap } from './artifacts.js';
+import { auditRun } from './audit/index.js';
 import { launchBrowser } from './browser.js';
 import { resolveConfig } from './config.js';
+import { renderAllRoutes } from './crawl.js';
 import { prepareDestination } from './destination.js';
 import { createLogger } from './log.js';
 import { routeToFile } from './output.js';
 import { pruneStaleFiles, readPruneManifest, writePruneManifest } from './prune.js';
-import { renderRoute } from './render.js';
-import {
-  crawlRoots,
-  filterRoutes,
-  isIncludedRoute,
-  normaliseRoute,
-  withinDepth,
-} from './routes.js';
-import { createPool } from './scheduler.js';
+import { writeReport } from './report.js';
+import { loadRouteSeeds } from './route-sources.js';
+import { normaliseRoute } from './routes.js';
 import { startCommandServer, startStaticServer } from './server.js';
 import { verifyRoutes } from './verify.js';
 import { writeAllRoutes } from './write.js';
@@ -38,6 +34,11 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
   const outputDir = await resolveOutputDir({ sourceDir, config, log });
   const previousManifest =
     config.pruneStaleRoutes && !config.dryRun ? await readPruneManifest(outputDir) : null;
+  const routeSources = await loadRouteSeeds(config);
+  for (const source of routeSources.sources) {
+    const skipped = source.skipped > 0 ? `, ${source.skipped} skipped` : '';
+    log.info(`Route source ${source.source}: ${source.routes} route(s)${skipped}`);
+  }
 
   if (config.saveAs !== 'html' && config.verify) {
     log.warn(`saveAs: '${config.saveAs}' writes images, so hydration verification is skipped`);
@@ -62,7 +63,14 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
     emptyPages: [],
     duplicateTitles: [],
     duplicatePages: [],
+    links: [],
+    notFoundRoutes: [],
+    routeSources: routeSources.sources,
+    routeDetails: [],
+    cspSkipped: [],
+    traces: [],
     verification: null,
+    audit: null,
     preloadManifest: null,
     sitemap: null,
     pruned: [],
@@ -85,6 +93,8 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
       config,
       log,
       report,
+      outputDir,
+      sourceSeeds: routeSources.routes,
     });
     if (report.routes.length === 0) {
       report.errors.push({ route: null, message: 'no routes were rendered' });
@@ -173,9 +183,11 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
         origin: server.origin,
         routes: toVerify.sort(),
         config,
+        outputDir,
       });
       report.verification.skipped = skipped;
       logVerification(report.verification, config, log);
+      mergeVerification(report);
     }
 
     if (
@@ -211,6 +223,15 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
   }
 
   report.durationMs = Date.now() - startedAt;
+  if (config.audit) {
+    try {
+      report.audit = await auditRun({ report, config, outputDir });
+      logAudit(report.audit, log);
+    } catch (error) {
+      report.errors.push({ route: null, message: `audit failed: ${error.message}` });
+      log.error(`audit failed: ${error.message}`);
+    }
+  }
   const reRendered = report.verification?.modes['re-rendered'] ?? 0;
   const undetected = report.verification
     ? (report.verification.modes.undetected ?? 0) + (report.verification.modes.unknown ?? 0)
@@ -222,10 +243,18 @@ export async function prerender(userOptions = {}, { log: injectedLog } = {}) {
     !(config.failOnConsoleError && report.consoleErrors.length > 0) &&
     !(config.failOnFailedRequest && report.failedRequests.length > 0) &&
     !(config.failOnDuplicateContent && report.duplicatePages.length > 0) &&
+    !(config.failOnAuditError && (report.audit?.errors.length ?? 0) > 0) &&
     !(config.failOnHydrationError && report.verification && !report.verification.ok) &&
     !(config.failOnVerificationError && report.verification?.errors.length > 0) &&
     !(config.failOnUndetected && undetected > 0) &&
     !(config.failOnRerender && reRendered > 0);
+  try {
+    report.reportFile = await writeReport({ report, config, log });
+  } catch (error) {
+    report.errors.push({ route: null, message: `report write failed: ${error.message}` });
+    log.error(`report write failed: ${error.message}`);
+    if (config.failOnError) report.ok = false;
+  }
   return report;
 }
 
@@ -269,114 +298,40 @@ async function resolveOutputDir({ sourceDir, config, log }) {
   });
 }
 
-/**
- * Runs the crawl and render pool. Each rendered page contributes its output, any new
- * same-origin routes, and the resources recorded for the link-hint features.
- */
-async function renderAllRoutes({ browser, server, config, log, report }) {
-  const visited = new Set();
-  const scheduled = new Set();
-  const rendered = new Map();
-  const cap = config.maxRoutes ?? Number.POSITIVE_INFINITY;
-
-  // Queues routes up to maxRoutes, flagging the report the first time the cap bites.
-  // A seed used only to reach wildcard routes is given output: false and is rendered
-  // for its links without being written.
-  const accept = (routes, outputFor = () => true) => {
-    const accepted = [];
-    for (const route of routes) {
-      if (scheduled.has(route)) continue;
-      if (scheduled.size >= cap) {
-        report.truncated = { limit: config.maxRoutes, routes: scheduled.size };
-        break;
-      }
-      scheduled.add(route);
-      accepted.push({ route, output: outputFor(route) });
-    }
-    return accepted;
-  };
-
-  const pool = createPool({
-    concurrency: config.concurrency,
-    handler: async ({ route, depth, output }) => {
-      if (visited.has(route)) return;
-      visited.add(route);
-      log.debug(`rendering ${route}${output ? '' : ' as a crawl seed'}`);
-      try {
-        const result = await renderRoute({
-          browser,
-          origin: server.origin,
-          route,
-          config,
-          screenshot: config.saveAs !== 'html',
-        });
-        if (output && result.empty) {
-          report.emptyPages.push(route);
-          if (config.failOnEmptyPage) {
-            report.errors.push({ route, message: 'rendered an empty page' });
-            log.error(`  empty page ${route}: no app content was found`);
-            return;
-          }
-          log.warn(`  empty page ${route}: no app content was found`);
-        }
-        if (output) {
-          rendered.set(route, result);
-          report.routes.push(route);
-        }
-        for (const message of result.pageErrors) {
-          report.pageErrors.push({ route, message });
-          log.warn(`  page error on ${route}: ${message}`);
-        }
-        for (const message of result.consoleErrors) {
-          report.consoleErrors.push({ route, message });
-        }
-        if (result.consoleErrors.length > 0) {
-          log.debug(`  console error on ${route}: ${result.consoleErrors[0]}`);
-          if (config.failOnConsoleError) {
-            report.errors.push({ route, message: `console error: ${result.consoleErrors[0]}` });
-          }
-        }
-        for (const failure of result.failedRequests) {
-          report.failedRequests.push({ route, ...failure });
-          const detail = failure.status ? `HTTP ${failure.status}` : failure.error;
-          log.warn(`  failed request on ${route}: ${failure.url} (${detail})`);
-        }
-        if (result.failedRequests.length > 0 && config.failOnFailedRequest) {
-          const first = result.failedRequests[0];
-          report.errors.push({
-            route,
-            message: `failed request: ${first.url}${first.status ? ` (HTTP ${first.status})` : ''}`,
-          });
-        }
-        if (config.crawl && withinDepth(depth + 1, config.maxDepth)) {
-          const discovered = filterRoutes(result.links, {
-            include: config.includeProvided ? config.include : [],
-            exclude: config.exclude,
-          }).filter((candidate) => !scheduled.has(candidate));
-          pool.push(accept(discovered).map((item) => ({ ...item, depth: depth + 1 })));
-        }
-      } catch (error) {
-        report.errors.push({ route, message: error.message });
-        log.error(`  failed ${route}: ${error.message}`);
-      }
-    },
-  });
+// Adds the verification result to each route record for the JSON report.
+function mergeVerification(report) {
+  for (const entry of report.verification?.routes ?? []) {
+    const detail = report.routeDetails.find((record) => record.route === entry.route);
+    if (!detail) continue;
+    detail.mode = entry.mode;
+    detail.framework = entry.framework ?? null;
+    detail.hydrationErrors = entry.hydrationErrors;
+    detail.pageErrors = [...new Set([...(detail.pageErrors ?? []), ...entry.pageErrors])];
+    detail.consoleErrors = [...new Set([...(detail.consoleErrors ?? []), ...entry.consoleErrors])];
+    detail.verificationStatus = entry.status;
+    detail.empty = entry.empty || detail.empty;
+    detail.firstMismatch = entry.firstMismatch ?? null;
+  }
+}
 
-  const roots = accept(
-    crawlRoots(config),
-    (route) => !config.includeProvided || isIncludedRoute(route, config.include),
-  );
-  pool.push(roots.map((item) => ({ ...item, depth: 0 })));
-  const { errors } = await pool.run();
-  for (const { item, error } of errors) {
-    report.errors.push({ route: item.route, message: error.message });
-    log.error(`  pool error on ${item.route}: ${error.message}`);
+// Prints an audit summary and a bounded sample of findings.
+function logAudit(audit, log) {
+  const errors = audit?.errors?.length ?? 0;
+  const warnings = audit?.warnings?.length ?? 0;
+  if (errors === 0 && warnings === 0) {
+    log.success(`Audit: no issues (${audit?.linksChecked ?? 0} links checked)`);
+    return;
+  }
+  log.info(`Audit: ${errors} error(s), ${warnings} warning(s)`);
+  for (const finding of [...(audit?.errors ?? []), ...(audit?.warnings ?? [])].slice(0, 20)) {
+    const message = `${finding.rule}: ${finding.message}`;
+    if (finding.severity === 'error') log.error(`  ${message}`);
+    else log.warn(`  ${message}`);
   }
-  return rendered;
 }
 
-// Notes routes whose output carries more than one title, which is what React 19 native
-// metadata and the head layer together produce.
+// Notes duplicate title tags, which React 19 native metadata and the head layer can
+// each produce.
 function logDuplicateTitles(report, log) {
   if (report.duplicateTitles.length === 0) return;
   const sample = report.duplicateTitles.slice(0, 5).join(', ');