snappy-prerender 2.6.0-rc.3 → 2.7.0

package/src/core/audit/index.js

@@ -0,0 +1,114 @@
+import { auditLinks, auditOrphans } from './links.js';
+import { auditMetadata } from './metadata.js';
+import { auditSitemap } from './sitemap.js';
+
+/**
+ * Audits the written site and returns report-only findings. Errors are the things a
+ * static site should not ship (broken links, sitemap disagreement, duplicate titles);
+ * warnings cover metadata that is useful but legitimately optional.
+ */
+export async function auditRun({ report, config, outputDir }) {
+  const errors = [];
+  const warnings = [];
+  const add = (severity, rule, message, details = {}) => {
+    const finding = { rule, severity, message, ...details };
+    if (severity === 'error') errors.push(finding);
+    else warnings.push(finding);
+  };
+
+  const pages = report.routeDetails.filter((detail) => detail.file?.endsWith('.html'));
+  auditRouteHealth({ report, add });
+  auditMetadata({ pages, config, add });
+  auditSitemap({ pages, config, report, add });
+  const linkAudit = await auditLinks({ report, config, outputDir, add });
+  auditOrphans({ pages, report, add });
+  auditCspHygiene({ pages, config, report, add });
+  removeDuplicateFindings({ errors, warnings });
+
+  return {
+    errors,
+    warnings,
+    linksChecked: linkAudit.checked,
+  };
+}
+
+// Mirrors route-level failures into the audit so failOnAuditError covers route health
+// without requiring callers to inspect a second collection.
+function auditRouteHealth({ report, add }) {
+  for (const error of report.errors ?? []) {
+    add('error', 'route-error', error.message, { route: error.route ?? undefined });
+  }
+  if (report.truncated) {
+    add(
+      'error',
+      'route-cap',
+      `maxRoutes (${report.truncated.limit}) stopped the crawl after ${report.truncated.routes} route(s)`,
+      { droppedRoutes: report.truncated.droppedRoutes ?? [] },
+    );
+  }
+  for (const route of report.emptyPages ?? []) {
+    add('error', 'empty-page', 'rendered an empty page', { route });
+  }
+  for (const group of report.duplicatePages ?? []) {
+    for (const route of group.routes) {
+      const others = group.routes.filter((other) => other !== route).join(', ');
+      add('error', 'duplicate-output', `identical output to ${others}`, { route });
+    }
+  }
+}
+
+// Inline style attributes cannot be covered by a CSP hash, and browser console errors
+// reveal assets the generated policy blocked.
+function auditCspHygiene({ pages, config, report, add }) {
+  if (config.csp !== 'hash') return;
+  for (const route of report.cspSkipped ?? []) {
+    add(
+      'warning',
+      'existing-csp-meta',
+      `${route} already has a CSP meta, so no hashes were injected`,
+      { route },
+    );
+  }
+  const violations = new Map();
+  const collect = (route, message) => {
+    if (!/content security policy|violates the following/i.test(message)) return;
+    const messages = violations.get(route) ?? new Set();
+    messages.add(message);
+    violations.set(route, messages);
+  };
+  for (const { route, message } of report.consoleErrors ?? []) collect(route, message);
+  for (const page of pages) {
+    for (const message of page.consoleErrors ?? []) collect(page.route, message);
+  }
+  for (const [route, messages] of violations) {
+    add(
+      'warning',
+      'csp-console-violation',
+      `${route} logged ${messages.size} Content-Security-Policy violation(s)`,
+      { route },
+    );
+  }
+  for (const page of pages) {
+    if (page.meta?.inlineStyleAttributes > 0) {
+      add(
+        'warning',
+        'inline-style-attributes',
+        `${page.route} has ${page.meta.inlineStyleAttributes} inline style attribute(s); CSP hashes cannot cover them`,
+        { route: page.route },
+      );
+    }
+  }
+}
+
+// A finding can be produced by more than one check; keep the first copy.
+function removeDuplicateFindings({ errors, warnings }) {
+  for (const list of [errors, warnings]) {
+    const seen = new Set();
+    for (let index = list.length - 1; index >= 0; index -= 1) {
+      const finding = list[index];
+      const key = `${finding.rule}|${finding.route ?? ''}|${finding.message}`;
+      if (seen.has(key)) list.splice(index, 1);
+      else seen.add(key);
+    }
+  }
+}

package/src/core/audit/links.js

@@ -0,0 +1,63 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { routeToFile } from '../output.js';
+import { matchesPattern, normaliseRoute } from '../routes.js';
+
+/**
+ * Audits every internal link the rendered pages emitted and warns about indexable pages
+ * no other page links to.
+ */
+export async function auditLinks({ report, config, outputDir, add }) {
+  const routes = new Set(report.routeDetails.map((detail) => detail.route));
+  const files = new Set(report.files.map((file) => file.file).filter(Boolean));
+  const cache = new Map();
+  const seenPairs = new Set();
+  let checkedLinks = 0;
+
+  const exists = async (target) => {
+    if (cache.has(target)) return cache.get(target);
+    const result = await fs
+      .stat(path.join(outputDir, target))
+      .then((stats) => stats.isFile())
+      .catch(() => false);
+    cache.set(target, result);
+    return result;
+  };
+
+  for (const link of report.links) {
+    const key = `${link.from}|${link.to}`;
+    if (seenPairs.has(key)) continue;
+    seenPairs.add(key);
+    if (config.auditIgnoreLinks.some((pattern) => matchesPattern(link.to, pattern))) continue;
+    checkedLinks += 1;
+    const route = normaliseRoute(link.to);
+    if (routes.has(route)) continue;
+    const file = routeToFile(route, config);
+    if (files.has(file)) continue;
+    const asset = route.startsWith('/') ? route.slice(1) : route;
+    if (asset && (await exists(asset))) continue;
+    add('error', 'broken-internal-link', `${link.from} links to missing ${link.to}`, {
+      from: link.from,
+      to: link.to,
+      reason: 'missing',
+    });
+  }
+  return { checked: checkedLinks };
+}
+
+/**
+ * Warns about indexable pages no rendered page links to. A link from a page outside the
+ * include list is legitimately absent, so this stays a warning.
+ */
+export function auditOrphans({ pages, report, add }) {
+  const linked = new Set();
+  for (const link of report.links) linked.add(normaliseRoute(link.to));
+  for (const page of pages) {
+    if (page.route === '/' || page.notFound || page.noindex) continue;
+    if (!linked.has(page.route)) {
+      add('warning', 'orphan-page', `${page.route} is not linked from another page`, {
+        route: page.route,
+      });
+    }
+  }
+}

package/src/core/audit/metadata.js

@@ -0,0 +1,88 @@
+import { toPublicPath } from '../routes.js';
+import { pagePath } from '../url-policy.js';
+
+/**
+ * Checks the metadata a crawler reads on every written page: titles, descriptions,
+ * canonicals, headings, build-origin leakage and duplicate values across routes.
+ */
+export function auditMetadata({ pages, config, add }) {
+  const titles = new Map();
+  const descriptions = new Map();
+  for (const page of pages) {
+    const meta = page.meta ?? {};
+    if (!meta.title) {
+      add('warning', 'missing-title', `no <title> on ${page.route}`, { route: page.route });
+    }
+    if (!meta.description) {
+      add('warning', 'missing-description', `no meta description on ${page.route}`, {
+        route: page.route,
+      });
+    }
+    if (!meta.canonical) {
+      add('warning', 'missing-canonical', `no canonical link on ${page.route}`, {
+        route: page.route,
+      });
+    }
+    if (meta.titleCount > 1) {
+      add('error', 'multiple-titles', `${page.route} contains more than one <title>`, {
+        route: page.route,
+      });
+    }
+    if (meta.h1Count !== 1) {
+      add('warning', 'heading-count', `${page.route} has ${meta.h1Count} <h1> element(s)`, {
+        route: page.route,
+      });
+    }
+    if ((page.buildOriginUrls ?? []).length > 0) {
+      add(
+        'warning',
+        'build-origin-metadata',
+        `${page.route} still points at the build server: ${page.buildOriginUrls.join(', ')}`,
+        { route: page.route },
+      );
+    }
+    if (meta.title) titles.set(meta.title, [...(titles.get(meta.title) ?? []), page.route]);
+    if (meta.description) {
+      descriptions.set(meta.description, [
+        ...(descriptions.get(meta.description) ?? []),
+        page.route,
+      ]);
+    }
+    if (config.metadata?.siteUrl && meta.canonical) {
+      const expected = expectedPageUrl(page.route, config);
+      if (expected && meta.canonical !== expected) {
+        add(
+          'warning',
+          'canonical-mismatch',
+          `${page.route} canonical is ${meta.canonical}, expected ${expected}`,
+          { route: page.route },
+        );
+      }
+    }
+  }
+  for (const [title, routes] of titles) {
+    if (routes.length < 2) continue;
+    add('error', 'duplicate-title', `routes share the title "${title}": ${routes.join(', ')}`, {
+      routes,
+    });
+  }
+  for (const [description, routes] of descriptions) {
+    if (routes.length < 2) continue;
+    add(
+      'warning',
+      'duplicate-description',
+      `routes share the meta description "${description}": ${routes.join(', ')}`,
+      { routes },
+    );
+  }
+}
+
+// Resolves the public URL a page should canonicalise and be listed at.
+export function expectedPageUrl(route, config) {
+  if (!config.metadata?.siteUrl) return null;
+  const path = pagePath(route, {
+    flatOutput: config.flatOutput,
+    trailingSlash: config.metadata.trailingSlash,
+  });
+  return `${config.metadata.siteUrl.replace(/\/$/, '')}${toPublicPath(path, config.base)}`;
+}

package/src/core/audit/sitemap.js

@@ -0,0 +1,39 @@
+import { expectedPageUrl } from './metadata.js';
+
+/**
+ * Ensures the sitemap and the indexable pages describe the same set of URLs. A page
+ * missing from the sitemap, a loc with no page, or a noindex page listed are all errors.
+ */
+export function auditSitemap({ pages, config, report, add }) {
+  if (!config.sitemap) return;
+  const indexable = new Map();
+  const noindex = new Map();
+  for (const page of pages) {
+    const loc = expectedPageUrl(page.route, config);
+    if (!loc) continue;
+    if (page.noindex || page.notFound) {
+      noindex.set(loc, page.route);
+      continue;
+    }
+    indexable.set(loc, page.route);
+  }
+  const locs = new Set(report.sitemap?.locs ?? []);
+  for (const [loc, route] of indexable) {
+    if (!locs.has(loc)) {
+      add('error', 'sitemap-missing-page', `${route} is missing from the sitemap`, { route });
+    }
+  }
+  for (const loc of locs) {
+    if (noindex.has(loc)) {
+      add('error', 'sitemap-noindex-page', `sitemap lists noindex ${noindex.get(loc)}`, {
+        route: noindex.get(loc),
+      });
+      continue;
+    }
+    if (!indexable.has(loc)) {
+      add('error', 'sitemap-unknown-page', `sitemap lists ${loc} with no indexable page`, {
+        to: loc,
+      });
+    }
+  }
+}

package/src/core/blocking.js

@@ -1,3 +1,36 @@
+/**
+ * Reports whether a host is on the allowlist. Entries may be a hostname, a hostname
+ * with a port, a leading-dot or wildcard suffix, or a full URL; matching is
+ * case-insensitive.
+ */
+export function isHostAllowed(hostname, port, allowedHosts) {
+  const host = hostname.toLowerCase();
+  const withPort = port ? `${host}:${port}` : host;
+  for (const raw of allowedHosts) {
+    const entry = String(raw).trim().toLowerCase();
+    if (entry.length === 0) continue;
+    if (entry.includes('://')) {
+      try {
+        const parsed = new URL(entry);
+        if (parsed.hostname.toLowerCase() === host && (!parsed.port || parsed.port === port)) {
+          return true;
+        }
+      } catch {
+        // A malformed entry is simply not a match.
+      }
+      continue;
+    }
+    if (entry === withPort || entry === host) return true;
+    if (entry.startsWith('*.') && host.endsWith(entry.slice(1)) && host !== entry.slice(2)) {
+      return true;
+    }
+    if (entry.startsWith('.') && host.endsWith(entry) && host !== entry.slice(1)) {
+      return true;
+    }
+  }
+  return false;
+}
+
 /**
  * Aborts requests to third-party origins so analytics, ads and external widgets
  * cannot mutate the DOM mid-capture or make output non-deterministic. Used by both
@@ -15,11 +48,30 @@ export async function blockThirdPartyRequests(page, origin, allowedHosts) {
       return allow(route);
     }
     if (parsed.origin === origin) return allow(route);
-    if (allowed.has(parsed.hostname.toLowerCase())) return allow(route);
+    if (isHostAllowed(parsed.hostname, parsed.port, allowed)) return allow(route);
     // A route can already be handled or cancelled, and that rejection is not this hook's to
     // report, so it is swallowed rather than surfacing as an unhandled rejection.
     return route.abort().catch(() => {});
   });
+
+  // page.route does not see WebSockets; route them separately when the Playwright
+  // version provides the hook.
+  if (typeof page.routeWebSocket === 'function') {
+    await page.routeWebSocket('**/*', (socket) => {
+      let parsed;
+      try {
+        parsed = new URL(socket.url());
+      } catch {
+        socket.close();
+        return;
+      }
+      if (parsed.origin === origin || isHostAllowed(parsed.hostname, parsed.port, allowed)) {
+        socket.connectToServer();
+      } else {
+        socket.close();
+      }
+    });
+  }
 }
 
 // Continues a request, ignoring the rejection that follows an already-handled route.

package/src/core/browser.js

@@ -1,14 +1,26 @@
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
-import * as browsers from '@puppeteer/browsers';
 import { chromium } from 'playwright-core';
 import { isExecutablePath } from './config.js';
 
 /**
- * Resolves a usable Chromium-compatible browser, preferring the browser already on
- * the machine, then any Playwright cache, then a one-time Chrome for Testing
- * download. Returns the launched browser plus a human-readable source for logs.
+ * Lists the resolution steps for a browser option. Explicit choices only ever try the
+ * browser they name, so a machine without Chrome cannot silently prerender in Edge;
+ * `auto` is the only choice that falls through the machine, the Playwright cache, a
+ * cached Chrome for Testing, and finally a download.
+ */
+export function browserSteps(browser) {
+  if (browser === 'auto') return ['chrome', 'msedge', 'playwright', 'cache', 'download'];
+  if (browser === 'chromium') return ['playwright'];
+  if (browser === 'chrome' || browser === 'msedge') return [browser];
+  return [];
+}
+
+/**
+ * Resolves a usable Chromium-compatible browser. Explicit channel choices fail rather
+ * than falling back, while `auto` prefers the machine's browser, then any cached
+ * download, then a one-time Chrome for Testing install.
  */
 export async function launchBrowser(config, log) {
   if (isExecutablePath(config.browser)) {
@@ -19,40 +31,54 @@ export async function launchBrowser(config, log) {
   }
 
   const attempts = [];
-  const channels =
-    config.browser === 'auto'
-      ? ['chrome', 'msedge']
-      : config.browser === 'chromium'
-        ? []
-        : [config.browser];
-
-  for (const channel of channels) {
+  for (const step of browserSteps(config.browser)) {
     try {
-      const browser = await launch(config, { channel });
-      return { browser, source: `system ${channel}` };
+      const launched = await runStep(step, config, log);
+      if (launched) return launched;
     } catch (error) {
-      attempts.push(`${channel}: ${firstLine(error.message)}`);
+      attempts.push(`${step}: ${firstLine(error.message)}`);
     }
   }
 
-  try {
-    const browser = await launch(config, {});
-    return { browser, source: 'playwright cache' };
-  } catch (error) {
-    attempts.push(`playwright cache: ${firstLine(error.message)}`);
-  }
-
   if (!config.browserDownload) {
     throw new Error(
       `No usable browser found and browserDownload is disabled. Tried:\n  ${attempts.join('\n  ')}`,
     );
   }
+  if (config.browser === 'chromium') {
+    throw new Error(
+      `No Playwright-managed Chromium is installed. Run "npx playwright install chromium", or set browser to auto or chrome. Tried:\n  ${attempts.join('\n  ')}`,
+    );
+  }
+  throw new Error(`No usable browser found. Tried:\n  ${attempts.join('\n  ')}`);
+}
 
-  const executablePath = await downloadChrome(config, log);
-  return {
-    browser: await launch(config, { executablePath }),
-    source: `downloaded ${executablePath}`,
-  };
+// Runs one browser resolution step and returns the launched browser, or null when the
+// step has nothing to offer on this machine.
+async function runStep(step, config, log) {
+  if (step === 'chrome' || step === 'msedge') {
+    return { browser: await launch(config, { channel: step }), source: `system ${step}` };
+  }
+  if (step === 'playwright') {
+    return { browser: await launch(config, {}), source: 'playwright cache' };
+  }
+  if (step === 'cache') {
+    const cached = await findCachedChrome(config);
+    if (!cached) return null;
+    return {
+      browser: await launch(config, { executablePath: cached.executablePath }),
+      source: `cached Chrome for Testing ${cached.buildId}`,
+    };
+  }
+  if (step === 'download') {
+    if (!config.browserDownload) return null;
+    const executablePath = await downloadChrome(config, log);
+    return {
+      browser: await launch(config, { executablePath }),
+      source: `downloaded ${executablePath}`,
+    };
+  }
+  return null;
 }
 
 // Launches Chromium with the configured headless mode, extra args and target.
@@ -70,20 +96,64 @@ function firstLine(message) {
 }
 
 /**
- * Downloads Chrome for Testing into the tool's own cache directory. Used only when
- * neither a system browser nor a Playwright-managed browser is available. The
- * archive comes from Google's version-pinned HTTPS bucket via @puppeteer/browsers,
- * which verifies it against browserDownloadHash when one is configured and deletes
- * it on a mismatch. Without a configured hash the download is trusted on the
- * strength of HTTPS alone, which is logged as a warning.
+ * Chooses a cached Chrome for Testing install. Kept pure so the offline/version-pinned
+ * selection can be tested without touching the machine's browser cache.
+ */
+export function selectCachedChrome(
+  installed,
+  { browser = 'chrome', platform, buildId = null, executableExists = () => true },
+) {
+  return (
+    installed.find(
+      (entry) =>
+        entry.browser === browser &&
+        entry.platform === platform &&
+        (!buildId || entry.buildId === buildId) &&
+        executableExists(entry.executablePath),
+    ) ?? null
+  );
+}
+
+// Formats download progress for the debug log; a missing content length reports 0%.
+export function downloadPercent(downloaded, total) {
+  return total > 0 ? Math.round((downloaded / total) * 100) : 0;
+}
+
+/**
+ * Finds a Chrome for Testing build already present in the tool's cache, so a rebuild
+ * never needs the version feed or a download when a browser is available offline.
+ * Honours browserDownloadVersion when one is configured.
+ */
+async function findCachedChrome(config) {
+  const browsers = await import('@puppeteer/browsers');
+  const platform = browsers.detectBrowserPlatform();
+  if (!platform) return null;
+  const installed = browsers.getInstalledBrowsers({ cacheDir: browserCacheDir() });
+  return selectCachedChrome(installed, {
+    browser: browsers.Browser.CHROME,
+    platform,
+    buildId: config.browserDownloadVersion,
+    executableExists: fs.existsSync,
+  });
+}
+
+/**
+ * Downloads Chrome for Testing into the tool's own cache directory. The archive comes
+ * from Google's version-pinned HTTPS bucket via @puppeteer/browsers, which verifies it
+ * against browserDownloadHash when one is configured and deletes it on a mismatch.
+ * Without a configured hash the download is trusted on the strength of HTTPS alone,
+ * which is logged as a warning.
  */
 async function downloadChrome(config, log) {
+  const browsers = await import('@puppeteer/browsers');
+  const platform = browsers.detectBrowserPlatform();
+  if (!platform) {
+    throw new Error(`Cannot download a browser for ${os.platform()} (${os.arch()})`);
+  }
   const cacheDir = browserCacheDir();
-  const buildId = await browsers.resolveBuildId(
-    browsers.Browser.CHROME,
-    browsers.detectBrowserPlatform(),
-    'stable',
-  );
+  const buildId =
+    config.browserDownloadVersion ??
+    (await browsers.resolveBuildId(browsers.Browser.CHROME, platform, 'stable'));
   if (config.browserDownloadHash) {
     log.info(
       `Downloading Chrome for Testing ${buildId}, verifying SHA-256 against browserDownloadHash`,
@@ -99,7 +169,7 @@ async function downloadChrome(config, log) {
     cacheDir,
     expectedHash: config.browserDownloadHash ?? undefined,
     downloadProgressCallback: (downloaded, total) =>
-      log.debug(`  downloaded ${total > 0 ? Math.round((downloaded / total) * 100) : 0}%`),
+      log.debug(`  downloaded ${downloadPercent(downloaded, total)}%`),
   });
   if (!fs.existsSync(installed.executablePath)) {
     throw new Error(`Chrome download reported success but ${installed.executablePath} is missing`);

package/src/core/build-origin.js

@@ -0,0 +1,19 @@
+import { parse } from 'parse5';
+import { collectBuildOrigins, isMetadataUrlNode, metadataUrlValue } from './metadata-origin.js';
+
+/**
+ * Collects build-server origins from the metadata URLs in a serialized document.
+ * Only canonical and URL-bearing social tags are inspected, so a loopback API URL in
+ * application state does not trip the optional build-origin gate.
+ */
+export function findBuildOriginUrls(html) {
+  const origins = new Set();
+  const walk = (node) => {
+    if (isMetadataUrlNode(node)) {
+      for (const origin of collectBuildOrigins(metadataUrlValue(node))) origins.add(origin);
+    }
+    for (const child of node.childNodes ?? []) walk(child);
+  };
+  walk(parse(html));
+  return [...origins].slice(0, 3);
+}