snappy-prerender 2.6.0-rc.2

package/src/core/runtime-styles.js

@@ -0,0 +1,42 @@
+/**
+ * Captures styles that exist only in the CSS object model. Runtimes that use
+ * CSSStyleSheet.insertRule (emotion and styled-components in their production
+ * "speedy" modes, JSS, and others) leave the style element empty in the DOM, so
+ * serialising the page would drop every rule. This copies those rules into the
+ * element text, and folds document-level constructable stylesheets into a style tag.
+ * A no-op for ordinary style elements and for pages without runtime styles.
+ */
+export async function captureRuntimeStyles(page) {
+  await page.evaluate(() => {
+    // Reads the rules a style element holds in the CSSOM, tolerating odd sheets.
+    const rulesOf = (sheet) => {
+      try {
+        return Array.from(sheet.cssRules ?? []);
+      } catch {
+        return [];
+      }
+    };
+
+    for (const style of document.querySelectorAll('style')) {
+      if (style.textContent.trim().length > 0) continue;
+      if (!style.sheet) continue;
+      const css = rulesOf(style.sheet)
+        .map((rule) => rule.cssText)
+        .join('\n');
+      if (css.length > 0) style.textContent = css;
+    }
+
+    const adopted = Array.from(document.adoptedStyleSheets ?? []);
+    if (adopted.length === 0) return;
+    const css = adopted
+      .flatMap((sheet) => rulesOf(sheet))
+      .map((rule) => rule.cssText)
+      .join('\n');
+    if (css.trim().length === 0) return;
+
+    const style = document.createElement('style');
+    style.setAttribute('data-snappy-adopted', '');
+    style.textContent = css;
+    document.head.appendChild(style);
+  });
+}

package/src/core/scheduler.js

@@ -0,0 +1,83 @@
+/**
+ * Dynamic work pool used for route rendering. Tasks may enqueue more tasks while
+ * running, which is what lets the crawler discover links from rendered pages.
+ * Workers exit only when the queue is empty and nothing is in flight, so `run()`
+ * resolves after the last discovered route finishes.
+ */
+export function createPool({ concurrency, handler }) {
+  const queue = [];
+  const waiters = [];
+  const errors = [];
+  let outstanding = 0;
+  let finished = false;
+
+  // Resolves every worker parked on an empty queue so they can re-check for work.
+  function wakeAll() {
+    while (waiters.length > 0) waiters.shift()();
+  }
+
+  // Worker loop: drains the queue, parks when empty, exits once nothing is left.
+  async function loop() {
+    while (true) {
+      if (queue.length === 0) {
+        if (outstanding === 0) return;
+        await new Promise((resolve) => waiters.push(resolve));
+        continue;
+      }
+      const item = queue.shift();
+      try {
+        await handler(item);
+      } catch (error) {
+        errors.push({ item, error });
+      } finally {
+        outstanding -= 1;
+        if (outstanding === 0) wakeAll();
+      }
+    }
+  }
+
+  return {
+    push(items) {
+      if (finished) {
+        throw new Error('cannot push work after the pool has finished; push before run() resolves');
+      }
+      for (const item of items) {
+        queue.push(item);
+        outstanding += 1;
+      }
+      wakeAll();
+    },
+    async run() {
+      await Promise.all(Array.from({ length: concurrency }, () => loop()));
+      finished = true;
+      return { errors };
+    },
+  };
+}
+
+/**
+ * Maps a fixed list of items through an async worker with bounded concurrency and
+ * per-item error capture, so one failed item never aborts the rest.
+ */
+export async function mapLimit(items, concurrency, worker) {
+  const results = new Array(items.length).fill(undefined);
+  const errors = [];
+  let cursor = 0;
+
+  // Worker loop: claims indices from the shared cursor until the list is consumed.
+  async function loop() {
+    while (cursor < items.length) {
+      const index = cursor;
+      cursor += 1;
+      try {
+        results[index] = await worker(items[index]);
+      } catch (error) {
+        errors.push({ item: items[index], error });
+      }
+    }
+  }
+
+  const workerCount = Math.max(1, Math.min(concurrency, items.length || 1));
+  await Promise.all(Array.from({ length: workerCount }, () => loop()));
+  return { results, errors };
+}

package/src/core/server.js

@@ -0,0 +1,216 @@
+import { spawn } from 'node:child_process';
+import http from 'node:http';
+import sirv from 'sirv';
+
+/**
+ * Serves the built output for rendering and verification. Runs in sirv dev mode so
+ * HTML files written during the run are picked up by the verification pass, and on
+ * an ephemeral loopback port so parallel runs never collide.
+ */
+export async function startStaticServer({ dir, base }) {
+  const handler = sirv(dir, { dev: true, etag: true, single: true });
+  const server = http.createServer((request, response) => {
+    if (base !== '/') {
+      const stripped = stripBase(request.url, base);
+      if (stripped === null) {
+        response.statusCode = 404;
+        response.end('Not found');
+        return;
+      }
+      request.url = stripped;
+    }
+    handler(request, response);
+  });
+
+  await new Promise((resolve, reject) => {
+    server.once('error', reject);
+    server.listen(0, '127.0.0.1', resolve);
+  });
+
+  const { port } = server.address();
+  return {
+    origin: `http://127.0.0.1:${port}`,
+    async close() {
+      server.closeAllConnections();
+      await new Promise((resolve) => server.close(resolve));
+    },
+  };
+}
+
+/**
+ * Removes the configured base prefix from a request URL, keeping any query string
+ * intact. Returns null when the request falls outside the base path.
+ */
+function stripBase(url, base) {
+  const prefix = base.slice(0, -1);
+  const [pathname, search] = url.split('?');
+  const query = search === undefined ? '' : `?${search}`;
+  if (pathname === prefix) return `/${query}`;
+  if (pathname.startsWith(`${prefix}/`)) return `${pathname.slice(prefix.length)}${query}`;
+  return null;
+}
+
+/**
+ * Runs a user-provided server command (for apps whose routes need a live API or
+ * runtime) and waits until the given URL answers before rendering starts.
+ */
+export async function startCommandServer({ command, url, timeout, shutdownTimeout }) {
+  const child = spawn(command, {
+    shell: true,
+    stdio: ['ignore', 'pipe', 'pipe'],
+    detached: process.platform !== 'win32',
+  });
+
+  const output = [];
+  const capture = (chunk) => {
+    output.push(chunk.toString());
+    if (output.length > 50) output.shift();
+  };
+  child.stdout.on('data', capture);
+  child.stderr.on('data', capture);
+
+  let spawnError = null;
+  let exited = false;
+  child.once('error', (error) => {
+    spawnError = error;
+  });
+  child.once('exit', () => {
+    exited = true;
+  });
+
+  try {
+    await waitForUrl(url, timeout, () => {
+      if (spawnError) return spawnError;
+      return exited ? new Error('process exited before responding') : null;
+    });
+  } catch (error) {
+    await stopProcessTree(child, shutdownTimeout);
+    const tail = output.join('').trim();
+    throw new Error(
+      `serveCmd did not become reachable at ${url}: ${error.message}${tail ? `\n${tail}` : ''}`,
+    );
+  }
+
+  return {
+    origin: new URL(url).origin,
+    async close() {
+      return stopProcessTree(child, shutdownTimeout);
+    },
+  };
+}
+
+/**
+ * Polls the target URL until it answers with a success status, the process dies, or
+ * the timeout expires. `failure()` reports a terminal reason when one exists.
+ */
+async function waitForUrl(url, timeout, failure) {
+  const startedAt = Date.now();
+  while (true) {
+    const reason = failure();
+    if (reason) throw reason;
+    try {
+      const response = await fetch(url, { signal: AbortSignal.timeout(1000) });
+      if (response.ok) return;
+    } catch {
+      // Server not up yet; retry until the deadline.
+    }
+    if (Date.now() - startedAt > timeout) throw new Error('timed out waiting for a response');
+    await new Promise((resolve) => setTimeout(resolve, 200));
+  }
+}
+
+/**
+ * Terminates a spawned server process tree. Windows uses taskkill /T /F because
+ * detached process groups are not available there. On POSIX the whole process group
+ * is signalled so shell wrappers and grandchildren die too, escalating to SIGKILL
+ * after the grace period. The child must have been spawned with `detached: true` on
+ * POSIX so it leads its own group; liveness is judged by the group, never by the
+ * direct child's exit, because a shell can die while the server it launched lives on.
+ * Returns false when the tree is still detectable at the end of the deadline.
+ */
+export async function stopProcessTree(child, shutdownTimeout, io = defaultProcessIo) {
+  if (!child.pid) return true;
+
+  if (io.platform === 'win32') {
+    if (io.hasExited(child)) return true;
+    await io.runTaskkill(child.pid);
+    return waitForChildExit(child, shutdownTimeout, io);
+  }
+
+  if (!io.isGroupAlive(child.pid)) return true;
+
+  io.signalGroup(child.pid, 'SIGTERM');
+  if (await waitForGroupExit(child.pid, shutdownTimeout, io)) return true;
+
+  io.signalGroup(child.pid, 'SIGKILL');
+  return waitForGroupExit(child.pid, shutdownTimeout, io);
+}
+
+// Platform primitives used by stopProcessTree. Injectable so the POSIX escalation
+// path can be exercised from any host, including Windows.
+const defaultProcessIo = {
+  platform: process.platform,
+  hasExited,
+  isGroupAlive,
+  signalGroup,
+  runTaskkill,
+};
+
+// Reports whether the spawned child has already been reaped.
+function hasExited(child) {
+  return child.exitCode !== null || child.signalCode !== null;
+}
+
+// Reports whether any process in the child's process group still exists.
+function isGroupAlive(pid) {
+  try {
+    process.kill(-pid, 0);
+    return true;
+  } catch (error) {
+    return error.code === 'EPERM';
+  }
+}
+
+// Signals the whole process group, falling back to the direct child when the group
+// is already gone or cannot be signalled.
+function signalGroup(pid, signal) {
+  try {
+    process.kill(-pid, signal);
+  } catch {
+    try {
+      process.kill(pid, signal);
+    } catch {
+      // Nothing left to signal.
+    }
+  }
+}
+
+// Polls until the direct child is reaped, covering the gap between taskkill
+// returning and the exit event being observed.
+async function waitForChildExit(child, timeout, io) {
+  const startedAt = Date.now();
+  while (!io.hasExited(child)) {
+    if (Date.now() - startedAt > timeout) return false;
+    await new Promise((resolve) => setTimeout(resolve, 50));
+  }
+  return true;
+}
+
+// Polls until the process group disappears or the deadline passes.
+async function waitForGroupExit(pid, timeout, io) {
+  const startedAt = Date.now();
+  while (io.isGroupAlive(pid)) {
+    if (Date.now() - startedAt > timeout) return false;
+    await new Promise((resolve) => setTimeout(resolve, 50));
+  }
+  return true;
+}
+
+// Force-kills a process tree on Windows, resolving regardless of taskkill's outcome.
+function runTaskkill(pid) {
+  return new Promise((resolve) => {
+    const killer = spawn('taskkill', ['/pid', String(pid), '/T', '/F'], { stdio: 'ignore' });
+    killer.once('exit', resolve);
+    killer.once('error', resolve);
+  });
+}

package/src/core/settle.js

@@ -0,0 +1,141 @@
+const FREEZE_CSS = [
+  '*, *::before, *::after {',
+  '  animation-duration: 0.001s !important;',
+  '  animation-delay: 0s !important;',
+  '  transition-duration: 0.001s !important;',
+  '  transition-delay: 0s !important;',
+  '  caret-color: transparent !important;',
+  '}',
+].join('\n');
+
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Tracks in-flight network requests so the renderer can wait for the page to go
+ * quiet. Listeners must be attached before navigation to count the initial load.
+ */
+export function trackNetwork(page) {
+  let inflight = 0;
+  const onStart = () => {
+    inflight += 1;
+  };
+  const onEnd = () => {
+    inflight = Math.max(0, inflight - 1);
+  };
+  page.on('request', onStart);
+  page.on('requestfinished', onEnd);
+  page.on('requestfailed', onEnd);
+
+  return {
+    async waitForQuiet({ quietPeriod, timeout }) {
+      const pollInterval = Math.max(10, Math.min(50, Math.floor(quietPeriod / 10)));
+      const startedAt = Date.now();
+      let quietSince = null;
+      while (true) {
+        if (inflight === 0) {
+          quietSince ??= Date.now();
+          if (Date.now() - quietSince >= quietPeriod) return;
+        } else {
+          quietSince = null;
+        }
+        if (Date.now() - startedAt > timeout) {
+          throw new Error(
+            `network did not settle within ${timeout}ms (${inflight} request(s) in flight)`,
+          );
+        }
+        await delay(pollInterval);
+      }
+    },
+    stop() {
+      page.off('request', onStart);
+      page.off('requestfinished', onEnd);
+      page.off('requestfailed', onEnd);
+    },
+  };
+}
+
+/**
+ * Waits until the app declares itself ready. Apps opt in by defining the ready flag
+ * early and setting it to true when data has loaded; otherwise the network
+ * quiescence check below is the only signal.
+ */
+export async function waitForReady(page, config) {
+  // Best-effort: one slow third-party asset should not cost a route its whole timeout, since
+  // the quiet-network check in settlePage is the real gate.
+  await page.waitForLoadState('load', { timeout: config.timeout }).catch(() => {});
+
+  if (config.waitFor) {
+    await page.waitForSelector(config.waitFor, { timeout: config.timeout });
+  }
+
+  if (config.readyFlag) {
+    const declared = await page.evaluate(
+      (flag) => typeof window[flag] !== 'undefined',
+      config.readyFlag,
+    );
+    if (declared) {
+      await page.waitForFunction((flag) => window[flag] === true, config.readyFlag, {
+        timeout: config.timeout,
+        polling: 100,
+      });
+    }
+  }
+
+  if (config.readySelector) {
+    await page.waitForSelector(config.readySelector, { timeout: config.timeout });
+  }
+}
+
+/**
+ * Waits for fonts and two animation frames so late layout work lands before the
+ * DOM is serialised.
+ */
+export async function waitForPaint(page) {
+  await page.evaluate(async () => {
+    await document.fonts.ready;
+    await new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve)));
+  });
+}
+
+/**
+ * Full settle sequence for one page: ready contract, quiet network, then paint.
+ */
+export async function settlePage(page, config, tracker) {
+  await waitForReady(page, config);
+  await tracker.waitForQuiet(config);
+  await waitForPaint(page);
+}
+
+/**
+ * Neutralises CSS animations and transitions before capture, so route HTML does not
+ * freeze mid-animation. The injected style is tagged and stripped during
+ * normalisation.
+ */
+export async function freezeAnimations(page) {
+  await page.evaluate((css) => {
+    const style = document.createElement('style');
+    style.id = 'snappy-freeze';
+    style.textContent = css;
+    document.head.appendChild(style);
+  }, FREEZE_CSS);
+  await page.evaluate(
+    () => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve))),
+  );
+}
+
+/**
+ * Scrolls through the page in viewport steps to trigger IntersectionObserver-based
+ * content, then returns to the top.
+ */
+export async function scrollThroughPage(page, { scrollStepDelay }) {
+  await page.evaluate(async (stepDelay) => {
+    const step = window.innerHeight;
+    const height = document.body.scrollHeight;
+    for (let offset = 0; offset < height; offset += step) {
+      window.scrollTo(0, offset);
+      await new Promise((resolve) => setTimeout(resolve, stepDelay));
+    }
+    window.scrollTo(0, 0);
+    await new Promise((resolve) => setTimeout(resolve, stepDelay));
+  }, scrollStepDelay);
+}

package/src/core/text-separators.js

@@ -0,0 +1,30 @@
+/**
+ * Inserts comment separators between adjacent text nodes before the DOM is
+ * serialised. HTML cannot express two neighbouring text nodes, so serialising a
+ * client-rendered DOM merges them, and hydration then finds one node where the
+ * client expects two — which React reports as a failed hydration. React's own server
+ * rendering emits the same separators, so this makes captured markup equivalent.
+ */
+export async function separateTextNodes(page) {
+  await page.evaluate(() => {
+    // Elements whose children are raw text, where a comment would corrupt the content.
+    const RAW_TEXT = new Set(['SCRIPT', 'STYLE', 'TEXTAREA', 'TITLE']);
+
+    // Walks the tree inserting a separator between each pair of adjacent text nodes.
+    const separate = (parent) => {
+      for (const child of Array.from(parent.childNodes)) {
+        if (child.nodeType === Node.ELEMENT_NODE) {
+          if (!RAW_TEXT.has(child.tagName)) separate(child);
+          continue;
+        }
+        if (child.nodeType !== Node.TEXT_NODE) continue;
+        const next = child.nextSibling;
+        if (next?.nodeType === Node.TEXT_NODE) {
+          parent.insertBefore(document.createComment(' '), next);
+        }
+      }
+    };
+
+    separate(document.documentElement);
+  });
+}

package/src/core/verify.js

@@ -0,0 +1,148 @@
+import { blockThirdPartyRequests } from './blocking.js';
+import { toServerUrl } from './routes.js';
+import { mapLimit } from './scheduler.js';
+import { settlePage, trackNetwork } from './settle.js';
+
+const HYDRATION_PATTERNS = [
+  /hydrat/i,
+  /did not match/i,
+  /server-rendered HTML/i,
+  /react\.dev\/errors\/(418|419|421|422|423|424|425)/i,
+  /Minified React error #(418|419|421|422|423|424|425)\b/i,
+];
+
+// Containers used by the common React setups, so the probe can find the prerendered
+// subtree without configuration.
+const CONTAINER_SELECTOR = '#root, #app, #__next';
+
+/**
+ * Page-side probe installed before navigation. It remembers the first prerendered
+ * child of the app container, which is what lets the verifier tell hydration from a
+ * client re-render: hydration adopts those nodes, createRoot discards them.
+ */
+function bootProbe() {
+  window.__snappyBoot = { prerenderedChild: null };
+
+  // Records the container's first child while the parser is still building the page.
+  const capture = () => {
+    if (window.__snappyBoot.prerenderedChild) return;
+    const container =
+      document.querySelector('#root, #app, #__next') ?? document.body?.firstElementChild;
+    if (!container) return;
+    window.__snappyBoot.prerenderedChild =
+      container.firstElementChild ?? container.firstChild ?? null;
+  };
+
+  const observer = new MutationObserver(capture);
+  observer.observe(document, { childList: true, subtree: true });
+  window.addEventListener('load', () => {
+    observer.disconnect();
+    capture();
+  });
+}
+
+/**
+ * Classifies a console or page error as a React hydration failure. Exported so the
+ * pattern set is testable without a browser. Patterns stay narrow on purpose: a
+ * false positive fails a user's build, so unrelated 4xx React codes and generic
+ * "does not match" messages are deliberately not treated as hydration errors.
+ */
+export function isHydrationError(message) {
+  return HYDRATION_PATTERNS.some((pattern) => pattern.test(message));
+}
+
+/**
+ * Classifies how a prerendered page booted. `hydrated` means the app adopted the
+ * markup, `re-rendered` means it discarded it, `undetected` means no React root was
+ * found on the container, and `unknown` means the prerendered child was never seen.
+ */
+export function classifyBoot({ markupPreserved, frameworkMounted }) {
+  if (markupPreserved === null) return 'unknown';
+  if (!frameworkMounted) return 'undetected';
+  return markupPreserved ? 'hydrated' : 're-rendered';
+}
+
+/**
+ * Reloads every written route with the real client bundle and reports React
+ * hydration errors plus how each route booted. This is the pass that catches markup
+ * that only breaks once the app re-attaches, and apps that never attach to it at all.
+ */
+export async function verifyRoutes({ browser, origin, routes, config }) {
+  const { results, errors } = await mapLimit(routes, config.concurrency, (route) =>
+    verifyRoute({ browser, origin, route, config }),
+  );
+  const verified = results.filter(Boolean);
+  const modes = { hydrated: 0, 're-rendered': 0, undetected: 0, unknown: 0 };
+  for (const entry of verified) modes[entry.mode] += 1;
+  return {
+    routes: verified,
+    modes,
+    ok: verified.every((entry) => entry.ok) && errors.length === 0,
+    errors: errors.map(({ item, error }) => ({ route: item, message: error.message })),
+  };
+}
+
+/**
+ * Verifies one route against the same request rules as the render pass, so what is
+ * verified matches what was rendered.
+ */
+async function verifyRoute({ browser, origin, route, config }) {
+  const context = await browser.newContext({
+    viewport: config.viewport,
+    storageState: config.storageState ?? undefined,
+    userAgent: config.userAgent ?? undefined,
+    ignoreHTTPSErrors: config.ignoreHTTPSErrors,
+  });
+  const page = await context.newPage();
+  const hydrationErrors = [];
+  const consoleErrors = [];
+  const pageErrors = [];
+  // Hydration messages arrive through either channel, so both are classified the same way.
+  const collect = (message, target) => {
+    if (isHydrationError(message)) hydrationErrors.push(message);
+    else target.push(message);
+  };
+  page.on('console', (message) => {
+    if (message.type() === 'error') collect(message.text(), consoleErrors);
+  });
+  page.on('pageerror', (error) => collect(error.message, pageErrors));
+
+  const tracker = trackNetwork(page);
+  try {
+    if (config.blockThirdParty) {
+      await blockThirdPartyRequests(page, origin, config.allowedHosts);
+    }
+    await page.addInitScript(bootProbe);
+    await page.goto(toServerUrl(route, config.base, origin), {
+      waitUntil: 'domcontentloaded',
+      timeout: config.timeout,
+    });
+    await settlePage(page, config, tracker);
+
+    const boot = await page.evaluate((selector) => {
+      const probe = window.__snappyBoot ?? {};
+      const container =
+        document.querySelector(selector) ?? document.body?.firstElementChild ?? null;
+      const frameworkMounted = container
+        ? Object.keys(container).some(
+            (key) => key.startsWith('__reactContainer') || key === '_reactRootContainer',
+          )
+        : false;
+      const child = probe.prerenderedChild ?? null;
+      return { frameworkMounted, markupPreserved: child ? child.isConnected : null };
+    }, CONTAINER_SELECTOR);
+
+    return {
+      route,
+      hydrationErrors,
+      consoleErrors,
+      pageErrors,
+      ok: hydrationErrors.length === 0,
+      mode: classifyBoot(boot),
+      markupPreserved: boot.markupPreserved,
+    };
+  } finally {
+    tracker.stop();
+    await context.close();
+  }
+}