snappy-prerender 2.6.0-rc.3 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +191 -0
  2. package/README.md +149 -107
  3. package/docs/limitations.md +58 -0
  4. package/docs/react-snap-migration.md +45 -0
  5. package/docs/report.md +102 -0
  6. package/package.json +9 -5
  7. package/src/cli-init.js +205 -0
  8. package/src/cli.js +159 -168
  9. package/src/core/artifacts.js +112 -0
  10. package/src/core/audit/index.js +114 -0
  11. package/src/core/audit/links.js +63 -0
  12. package/src/core/audit/metadata.js +88 -0
  13. package/src/core/audit/sitemap.js +39 -0
  14. package/src/core/blocking.js +53 -1
  15. package/src/core/browser.js +109 -39
  16. package/src/core/build-origin.js +19 -0
  17. package/src/core/config.js +256 -6
  18. package/src/core/crawl.js +300 -0
  19. package/src/core/csp.js +127 -0
  20. package/src/core/css-url.js +34 -0
  21. package/src/core/destination.js +28 -3
  22. package/src/core/frameworks.js +94 -0
  23. package/src/core/hydration-diff.js +93 -0
  24. package/src/core/init.js +187 -31
  25. package/src/core/inline-css.js +34 -13
  26. package/src/core/metadata-origin.js +11 -2
  27. package/src/core/normalize.js +59 -2
  28. package/src/core/output.js +32 -9
  29. package/src/core/page-content.js +13 -0
  30. package/src/core/page-meta.js +108 -0
  31. package/src/core/preload-links.js +120 -0
  32. package/src/core/preload-manifest.js +61 -7
  33. package/src/core/prune.js +65 -0
  34. package/src/core/render.js +147 -11
  35. package/src/core/report.js +108 -0
  36. package/src/core/resources.js +6 -1
  37. package/src/core/route-config.js +17 -0
  38. package/src/core/route-sources.js +209 -0
  39. package/src/core/routes.js +9 -0
  40. package/src/core/run.js +186 -358
  41. package/src/core/server.js +30 -4
  42. package/src/core/sitemap.js +128 -20
  43. package/src/core/url-policy.js +32 -0
  44. package/src/core/verify.js +103 -49
  45. package/src/core/write.js +363 -0
  46. package/src/head/index.js +2 -2
  47. package/src/head/manager.js +54 -13
  48. package/src/head/tags.js +71 -32
@@ -0,0 +1,65 @@
1
+ import fs from 'node:fs/promises';
2
+ import path from 'node:path';
3
+
4
+ export const MANIFEST_FILE = '.snappy-manifest.json';
5
+
6
+ /**
7
+ * Reads the file list the previous run wrote. A missing or malformed manifest simply
8
+ * disables pruning, so an older output directory can be upgraded safely.
9
+ */
10
+ export async function readPruneManifest(dir) {
11
+ try {
12
+ const parsed = JSON.parse(await fs.readFile(path.join(dir, MANIFEST_FILE), 'utf8'));
13
+ return Array.isArray(parsed.files) ? parsed.files : null;
14
+ } catch {
15
+ return null;
16
+ }
17
+ }
18
+
19
+ /**
20
+ * Deletes files the previous run generated that this run no longer produces, then
21
+ * removes emptied route directories. Only paths recorded in the manifest are touched,
22
+ * so assets and files the user added are never at risk.
23
+ */
24
+ export async function pruneStaleFiles({ dir, previous, current, log }) {
25
+ if (!Array.isArray(previous)) return [];
26
+ const keep = new Set(current);
27
+ const root = path.resolve(dir);
28
+ const removed = [];
29
+ for (const file of previous) {
30
+ if (typeof file !== 'string' || keep.has(file)) continue;
31
+ if (file === MANIFEST_FILE || file === 'robots.txt') continue;
32
+ const target = path.resolve(root, file);
33
+ if (target !== root && !target.startsWith(`${root}${path.sep}`)) continue;
34
+ try {
35
+ await fs.rm(target, { force: true });
36
+ removed.push(file);
37
+ await removeEmptyParents(root, path.dirname(target));
38
+ } catch (error) {
39
+ log?.warn(`could not prune ${file}: ${error.message}`);
40
+ }
41
+ }
42
+ return removed;
43
+ }
44
+
45
+ // Writes the manifest for the next run. Files that are not ours to own are excluded.
46
+ export async function writePruneManifest({ dir, files }) {
47
+ const owned = [...new Set(files)].filter(
48
+ (file) => file !== MANIFEST_FILE && file !== 'robots.txt',
49
+ );
50
+ const contents = `${JSON.stringify({ version: 1, files: owned.sort() }, null, 2)}\n`;
51
+ await fs.writeFile(path.join(dir, MANIFEST_FILE), contents, 'utf8');
52
+ }
53
+
54
+ // Removes parent directories that became empty, stopping at the output root.
55
+ async function removeEmptyParents(root, start) {
56
+ let current = start;
57
+ while (current !== root && current.startsWith(`${root}${path.sep}`)) {
58
+ try {
59
+ await fs.rmdir(current);
60
+ } catch {
61
+ return;
62
+ }
63
+ current = path.dirname(current);
64
+ }
65
+ }
@@ -1,9 +1,8 @@
1
- import fs from 'node:fs/promises';
2
- import path from 'node:path';
3
1
  import { captureFormState } from './form-state.js';
4
2
  import { inlineStylesheets } from './inline-css.js';
3
+ import { isPageEmpty } from './page-content.js';
5
4
  import { createResourceCollector } from './resources.js';
6
- import { toAppRoute, toServerUrl } from './routes.js';
5
+ import { normaliseRoute, toAppRoute, toServerUrl } from './routes.js';
7
6
  import { captureRuntimeStyles } from './runtime-styles.js';
8
7
  import { freezeAnimations, scrollThroughPage, settlePage, trackNetwork } from './settle.js';
9
8
  import { separateTextNodes } from './text-separators.js';
@@ -27,20 +26,55 @@ function escapeForScript(value) {
27
26
  * storage from leaking between routes. When a screenshot path is supplied the route
28
27
  * is captured as an image instead of HTML.
29
28
  */
30
- export async function renderRoute({ browser, origin, route, config, screenshotPath }) {
29
+ export async function renderRoute({
30
+ browser,
31
+ origin,
32
+ route,
33
+ config,
34
+ screenshot = false,
35
+ traceFile = null,
36
+ }) {
31
37
  const context = await browser.newContext({
32
38
  viewport: config.viewport,
33
39
  storageState: config.storageState ?? undefined,
34
40
  userAgent: config.userAgent ?? undefined,
35
41
  ignoreHTTPSErrors: config.ignoreHTTPSErrors,
36
42
  reducedMotion: config.freezeAnimations ? 'reduce' : 'no-preference',
43
+ serviceWorkers: 'block',
37
44
  });
45
+ let tracing = false;
46
+ if (traceFile) {
47
+ await context.tracing.start({ screenshots: false, snapshots: true, sources: false });
48
+ tracing = true;
49
+ }
38
50
  const page = await context.newPage();
39
51
  const pageErrors = [];
52
+ const consoleErrors = [];
53
+ const failedRequests = [];
40
54
  let inlineCssSkipped = [];
41
55
  page.on('pageerror', (error) => pageErrors.push(error.message));
56
+ page.on('console', (message) => {
57
+ if (message.type() === 'error') consoleErrors.push(message.text());
58
+ });
59
+ // Only same-origin failures are the build's own to fix; a blocked analytics host is
60
+ // not a broken page. Aborted requests are navigations or cancelled fetches, not 404s.
61
+ page.on('response', (response) => {
62
+ if (response.status() < 400) return;
63
+ const parsed = parseHttpUrl(response.url());
64
+ if (parsed?.origin !== origin) return;
65
+ failedRequests.push({ url: response.url(), status: response.status() });
66
+ });
67
+ page.on('requestfailed', (request) => {
68
+ const parsed = parseHttpUrl(request.url());
69
+ if (parsed?.origin !== origin) return;
70
+ const error = request.failure()?.errorText ?? 'request failed';
71
+ if (error.includes('ERR_ABORTED')) return;
72
+ failedRequests.push({ url: request.url(), status: null, error });
73
+ });
42
74
 
43
- const metadataDefaults = config.metadata ? { base: config.base, ...config.metadata } : null;
75
+ const metadataDefaults = config.metadata
76
+ ? { base: config.base, flatOutput: config.flatOutput, ...config.metadata }
77
+ : null;
44
78
  if (metadataDefaults) {
45
79
  await page.addInitScript((defaults) => {
46
80
  window.__SNAPPY_META__ = defaults;
@@ -56,13 +90,51 @@ export async function renderRoute({ browser, origin, route, config, screenshotPa
56
90
  maxCachedBytes: config.maxCachedBytes,
57
91
  });
58
92
  await collector.install(page);
93
+ await page.addInitScript(() => {
94
+ window.__snappyLcp = [];
95
+ try {
96
+ const observer = new PerformanceObserver((list) => {
97
+ for (const entry of list.getEntries()) {
98
+ const element = entry.element;
99
+ window.__snappyLcp.push({
100
+ url: entry.url || element?.currentSrc || element?.src || null,
101
+ });
102
+ }
103
+ });
104
+ observer.observe({ type: 'largest-contentful-paint', buffered: true });
105
+ } catch {
106
+ // Older engines simply do not report LCP; preload injection falls back to no hint.
107
+ }
108
+ });
59
109
  const tracker = trackNetwork(page);
60
110
  try {
61
- await page.goto(toServerUrl(route, config.base, origin), {
111
+ const response = await page.goto(toServerUrl(route, config.base, origin), {
62
112
  waitUntil: 'domcontentloaded',
63
113
  timeout: config.timeout,
64
114
  });
115
+ const notFoundRoute = config.notFoundRoute ? normaliseRoute(config.notFoundRoute) : null;
116
+ const isNotFoundRoute = notFoundRoute !== null && normaliseRoute(route) === notFoundRoute;
117
+ if (response && response.status() >= 400) {
118
+ // A configured 404 route may legitimately answer with HTTP 404; every other status
119
+ // is a failure, and the marker cannot be read before the navigation completes.
120
+ if (!(response.status() === 404 && isNotFoundRoute)) {
121
+ const error = new Error(
122
+ `route responded with HTTP ${response.status()}${response.statusText() ? ` ${response.statusText()}` : ''}`,
123
+ );
124
+ // A server error can be transient and is retried; a client error is the app's own.
125
+ error.retryable = response.status() >= 500;
126
+ throw error;
127
+ }
128
+ }
65
129
  await settlePage(page, config, tracker);
130
+ const empty = await isPageEmpty(page);
131
+ const statusCode = await page.evaluate(() => {
132
+ const marker = document.querySelector('meta[name="prerender-status-code"]');
133
+ if (!marker) return null;
134
+ const value = Number.parseInt(marker.getAttribute('content') ?? '', 10);
135
+ return Number.isFinite(value) ? value : null;
136
+ });
137
+ const finalUrl = page.url();
66
138
 
67
139
  if (config.captureRuntimeStyles) await captureRuntimeStyles(page);
68
140
  if (config.captureFormState) await captureFormState(page);
@@ -78,32 +150,86 @@ export async function renderRoute({ browser, origin, route, config, screenshotPa
78
150
  if (metadataDefaults) await injectMetadataDefaults(page, metadataDefaults);
79
151
  if (config.freezeAnimations) await freezeAnimations(page);
80
152
  if (config.scrollToBottom) await scrollThroughPage(page, config);
153
+ const lcpUrl = await page.evaluate(() => window.__snappyLcp?.at(-1)?.url ?? null);
154
+ let lcpImage = null;
155
+ if (lcpUrl) {
156
+ try {
157
+ const parsed = new URL(lcpUrl);
158
+ if (parsed.origin === origin) lcpImage = `${parsed.pathname}${parsed.search}`;
159
+ } catch {
160
+ // Ignore malformed performance entries.
161
+ }
162
+ }
81
163
  await separateTextNodes(page);
82
164
 
83
165
  const links = await collectLinks(page, config.base, origin);
84
166
 
85
- if (screenshotPath) {
86
- await fs.mkdir(path.dirname(screenshotPath), { recursive: true });
87
- await page.screenshot({ path: screenshotPath, fullPage: true });
167
+ if (screenshot) {
168
+ const buffer = await page.screenshot({
169
+ fullPage: true,
170
+ type: config.saveAs === 'jpeg' ? 'jpeg' : 'png',
171
+ });
88
172
  return {
89
173
  route,
90
174
  links,
91
175
  pageErrors,
176
+ consoleErrors,
177
+ failedRequests,
92
178
  collector,
93
179
  inlineCssSkipped,
94
180
  stateScript,
95
- screenshot: true,
181
+ screenshot: buffer,
182
+ empty,
183
+ statusCode,
184
+ httpStatus: response?.status() ?? null,
185
+ finalUrl,
186
+ lcpImage,
96
187
  };
97
188
  }
98
189
 
99
190
  const html = await page.content();
100
- return { route, html, links, pageErrors, collector, inlineCssSkipped, stateScript };
191
+ return {
192
+ route,
193
+ html,
194
+ links,
195
+ pageErrors,
196
+ consoleErrors,
197
+ failedRequests,
198
+ collector,
199
+ inlineCssSkipped,
200
+ stateScript,
201
+ empty,
202
+ statusCode,
203
+ httpStatus: response?.status() ?? null,
204
+ finalUrl,
205
+ lcpImage,
206
+ };
207
+ } catch (error) {
208
+ error.retryable ??= isRetryableRenderError(error);
209
+ if (tracing) {
210
+ tracing = false;
211
+ await context.tracing.stop({ path: traceFile }).then(
212
+ () => {
213
+ error.tracePath = traceFile;
214
+ },
215
+ // A failed trace must never replace the render error that explains the build failure.
216
+ () => {},
217
+ );
218
+ }
219
+ throw error;
101
220
  } finally {
102
221
  tracker.stop();
222
+ if (tracing) await context.tracing.stop().catch(() => {});
103
223
  await context.close();
104
224
  }
105
225
  }
106
226
 
227
+ // Navigation and network failures are transient; a bad selector or an HTTP status is not.
228
+ function isRetryableRenderError(error) {
229
+ if (error?.name === 'TimeoutError') return true;
230
+ return /net::ERR_|timed? ?out|ECONNRESET|ECONNREFUSED|socket hang up/i.test(error?.message ?? '');
231
+ }
232
+
107
233
  /**
108
234
  * Injects captured JSON responses as window.snapStore and anything the app exposes
109
235
  * through window.snapSaveState, before the first script runs. The client can then
@@ -226,3 +352,13 @@ async function collectLinks(page, base, origin) {
226
352
  }
227
353
  return routes;
228
354
  }
355
+
356
+ // Parses an HTTP URL, returning null for data, blob and other non-network URLs.
357
+ function parseHttpUrl(value) {
358
+ try {
359
+ const parsed = new URL(value);
360
+ return parsed.protocol === 'http:' || parsed.protocol === 'https:' ? parsed : null;
361
+ } catch {
362
+ return null;
363
+ }
364
+ }
@@ -0,0 +1,108 @@
1
+ import path from 'node:path';
2
+ import { writeFileIfChanged } from './output.js';
3
+
4
+ /**
5
+ * Writes the machine-readable run report. The JSON is deliberately limited to route,
6
+ * audit and artifact data; configuration and captured state are never serialised.
7
+ */
8
+ export async function writeReport({ report, config, log }) {
9
+ if (!config.reportPath) return null;
10
+ const contents = `${JSON.stringify(buildReport(report, config), null, 2)}\n`;
11
+ if (config.dryRun) {
12
+ log.info(`Dry run: would write report to ${config.reportPath}`);
13
+ return { file: config.reportPath, status: 'dry-run' };
14
+ }
15
+ const target = path.resolve(config.reportPath);
16
+ const written = await writeFileIfChanged({
17
+ dir: path.dirname(target),
18
+ file: path.basename(target),
19
+ contents,
20
+ });
21
+ log.info(`Wrote report to ${config.reportPath}`);
22
+ return { file: config.reportPath, status: written.status };
23
+ }
24
+
25
+ /**
26
+ * Shapes the public report document. `routes` here is an array of per-route records;
27
+ * the programmatic `report.routes` stays a list of route strings. URLs are reduced to
28
+ * origin and path, and the configured storage-state path is redacted everywhere, so a
29
+ * report can be attached to a build log without leaking credentials or session data.
30
+ */
31
+ export function buildReport(report, config = null) {
32
+ const audit = report.audit ?? { errors: [], warnings: [], linksChecked: 0 };
33
+ const document = {
34
+ version: 1,
35
+ generatedAt: new Date().toISOString(),
36
+ durationMs: report.durationMs,
37
+ ok: report.ok,
38
+ summary: {
39
+ routes: report.routeDetails.length,
40
+ files: report.files.filter((file) => file.status !== 'dry-run').length,
41
+ errors: uniqueErrorCount(report, audit),
42
+ warnings: audit.warnings.length,
43
+ },
44
+ routes: report.routeDetails,
45
+ verification: report.verification
46
+ ? {
47
+ ok: report.verification.ok,
48
+ modes: report.verification.modes,
49
+ frameworks: report.verification.frameworks,
50
+ skipped: report.verification.skipped ?? 0,
51
+ errors: report.verification.errors,
52
+ }
53
+ : null,
54
+ audit,
55
+ sitemap: report.sitemap,
56
+ preloadManifest: report.preloadManifest,
57
+ truncated: report.truncated,
58
+ notFoundRoutes: report.notFoundRoutes,
59
+ duplicateOutputs: report.duplicatePages,
60
+ routeSources: report.routeSources ?? [],
61
+ cspSkipped: report.cspSkipped ?? [],
62
+ traces: report.traces ?? [],
63
+ };
64
+ return sanitiseValue(document, config);
65
+ }
66
+
67
+ // Counts route errors once even when the audit mirrors them as findings.
68
+ function uniqueErrorCount(report, audit) {
69
+ const seen = new Set();
70
+ for (const error of [...(report.errors ?? []), ...(audit.errors ?? [])]) {
71
+ seen.add(`${error.route ?? ''}|${error.message ?? ''}`);
72
+ }
73
+ return seen.size;
74
+ }
75
+
76
+ // Recursively redacts URLs and the configured storage-state path from report strings.
77
+ function sanitiseValue(value, config) {
78
+ if (Array.isArray(value)) return value.map((entry) => sanitiseValue(entry, config));
79
+ if (value !== null && typeof value === 'object') {
80
+ return Object.fromEntries(
81
+ Object.entries(value).map(([key, entry]) => [key, sanitiseValue(entry, config)]),
82
+ );
83
+ }
84
+ if (typeof value !== 'string') return value;
85
+ let text = value;
86
+ if (config?.storageState) {
87
+ for (const secret of [config.storageState, path.resolve(config.storageState)]) {
88
+ text = text.split(secret).join('[redacted]');
89
+ }
90
+ }
91
+ return text.replace(/https?:\/\/[^\s"'<>]+/gi, redactUrl);
92
+ }
93
+
94
+ // Removes credentials, query strings and fragments from an absolute URL.
95
+ function redactUrl(raw) {
96
+ const trailing = /[.,;)\]]+$/.exec(raw)?.[0] ?? '';
97
+ const urlText = trailing ? raw.slice(0, -trailing.length) : raw;
98
+ try {
99
+ const url = new URL(urlText);
100
+ url.username = '';
101
+ url.password = '';
102
+ url.search = '';
103
+ url.hash = '';
104
+ return `${url.href}${trailing}`;
105
+ } catch {
106
+ return raw;
107
+ }
108
+ }
@@ -18,6 +18,7 @@ export function createResourceCollector({
18
18
  const images = new Set();
19
19
  const scripts = new Set();
20
20
  const styles = new Set();
21
+ const fonts = new Map();
21
22
  const json = new Map();
22
23
  const pending = [];
23
24
 
@@ -26,6 +27,7 @@ export function createResourceCollector({
26
27
  images,
27
28
  scripts,
28
29
  styles,
30
+ fonts,
29
31
  json,
30
32
  // Waits for in-flight JSON body reads so the cache is complete before injection.
31
33
  async settled() {
@@ -49,6 +51,7 @@ export function createResourceCollector({
49
51
  const url = request.url();
50
52
  const parsed = parseUrl(url);
51
53
  if (!parsed || parsed.origin !== origin) return;
54
+ if (response.status() >= 400) return;
52
55
  const path = toAppPath(url, base);
53
56
  if (!path) return;
54
57
 
@@ -56,7 +59,9 @@ export function createResourceCollector({
56
59
  if (resourceType === 'image') images.add(path);
57
60
  else if (resourceType === 'script') scripts.add(path);
58
61
  else if (resourceType === 'stylesheet') styles.add(path);
59
- else if (collectJson && isJson(response)) {
62
+ else if (resourceType === 'font') {
63
+ fonts.set(path, response.headers()['content-type'] ?? null);
64
+ } else if (collectJson && isJson(response)) {
60
65
  const length = Number(response.headers()['content-length'] ?? Number.NaN);
61
66
  if (Number.isFinite(length) && length > maxCachedBytes) return;
62
67
  const capture = response
@@ -0,0 +1,17 @@
1
+ import { matchesPattern } from './routes.js';
2
+
3
+ /**
4
+ * Resolves one route's effective config by applying matching routeOverrides in order.
5
+ * Later entries win per key; the base config is never mutated.
6
+ */
7
+ export function resolveRouteConfig(config, route) {
8
+ let resolved = config;
9
+ for (const override of config.routeOverrides ?? []) {
10
+ if (!matchesPattern(route, override.match)) continue;
11
+ const values = Object.fromEntries(
12
+ Object.entries(override).filter(([key, value]) => key !== 'match' && value !== undefined),
13
+ );
14
+ resolved = { ...resolved, ...values };
15
+ }
16
+ return resolved;
17
+ }
@@ -0,0 +1,209 @@
1
+ import fs from 'node:fs/promises';
2
+ import path from 'node:path';
3
+ import { isSafeRoute, normaliseRoute, toAppRoute } from './routes.js';
4
+
5
+ const MAX_SOURCE_ROUTES = 100000;
6
+ const FETCH_TIMEOUT = 10000;
7
+ const XML_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
8
+
9
+ /**
10
+ * Loads concrete route seeds from a text/JSON file and/or a sitemap. These are seeds,
11
+ * not a router: directory-style app paths still have to be handled by the app.
12
+ */
13
+ export async function loadRouteSeeds(config) {
14
+ const routes = [];
15
+ const sources = [];
16
+ if (config.routesFile) {
17
+ const read = await readRouteFile(config.routesFile, config);
18
+ routes.push(...read.routes);
19
+ sources.push({
20
+ source: config.routesFile,
21
+ type: 'file',
22
+ routes: read.routes.length,
23
+ skipped: read.skipped,
24
+ });
25
+ }
26
+ if (config.routesFromSitemap) {
27
+ const read = await readRouteSitemap(config.routesFromSitemap, config);
28
+ routes.push(...read.routes);
29
+ sources.push({
30
+ source: config.routesFromSitemap,
31
+ type: 'sitemap',
32
+ routes: read.routes.length,
33
+ skipped: read.skipped,
34
+ });
35
+ }
36
+ const unique = [...new Set(routes)];
37
+ if (unique.length > MAX_SOURCE_ROUTES) {
38
+ throw new Error(`route sources produced more than ${MAX_SOURCE_ROUTES} routes`);
39
+ }
40
+ return { routes: unique, sources };
41
+ }
42
+
43
+ // Reads a text or JSON route list. Text accepts one route per line and # comments.
44
+ async function readRouteFile(file, config) {
45
+ const contents = await fs.readFile(path.resolve(file), 'utf8');
46
+ let values;
47
+ const trimmed = contents.trim();
48
+ if (trimmed.startsWith('[') || trimmed.startsWith('{')) {
49
+ let parsed;
50
+ try {
51
+ parsed = JSON.parse(trimmed);
52
+ } catch (error) {
53
+ throw new Error(`routesFile is not valid JSON: ${error.message}`);
54
+ }
55
+ values = Array.isArray(parsed) ? parsed : parsed?.routes;
56
+ if (!Array.isArray(values)) {
57
+ throw new Error('routesFile JSON must be an array or an object with a routes array');
58
+ }
59
+ } else {
60
+ values = trimmed
61
+ .split(/\r?\n/)
62
+ .map((line) => line.replace(/#.*$/, '').trim())
63
+ .filter((line) => line.length > 0);
64
+ }
65
+ return normaliseAndFilter(values, 'routesFile', config);
66
+ }
67
+
68
+ // Reads a sitemap, following a sitemap index one level deep.
69
+ async function readRouteSitemap(location, config) {
70
+ const pageUrls = [];
71
+ await readSitemapLevel(location, pageUrls, 0);
72
+ const sourceOrigin = /^https?:\/\//i.test(location) ? new URL(location).origin : null;
73
+ return normaliseAndFilter(pageUrls, 'routesFromSitemap', config, sourceOrigin, true);
74
+ }
75
+
76
+ async function readSitemapLevel(location, pageUrls, depth) {
77
+ if (depth > 1) return;
78
+ const { text, base } = await readSource(location);
79
+ if (!/<(?:urlset|sitemapindex)\b/i.test(text)) {
80
+ throw new Error(`${location} is not a sitemap: no urlset or sitemapindex element found`);
81
+ }
82
+ if (/<sitemapindex\b/i.test(text)) {
83
+ for (const child of extractLocs(text)) {
84
+ await readSitemapLevel(resolveChild(child, base), pageUrls, depth + 1);
85
+ }
86
+ return;
87
+ }
88
+ pageUrls.push(...extractLocs(text));
89
+ }
90
+
91
+ // Reads a local path or an http(s) sitemap.
92
+ async function readSource(location) {
93
+ if (/^https?:\/\//i.test(location)) {
94
+ const response = await fetch(location, {
95
+ signal: AbortSignal.timeout(FETCH_TIMEOUT),
96
+ headers: { accept: 'application/xml,text/xml,*/*' },
97
+ });
98
+ if (!response.ok) throw new Error(`sitemap ${location} answered HTTP ${response.status}`);
99
+ return { text: await response.text(), base: location };
100
+ }
101
+ const file = path.resolve(location);
102
+ return { text: await fs.readFile(file, 'utf8'), base: file };
103
+ }
104
+
105
+ // Resolves a child sitemap location relative to its parent.
106
+ function resolveChild(child, base) {
107
+ if (/^https?:\/\//i.test(child)) return child;
108
+ if (/^https?:\/\//i.test(base)) return new URL(child, base).href;
109
+ return path.resolve(path.dirname(base), child);
110
+ }
111
+
112
+ // Extracts and unescapes every <loc> value.
113
+ function extractLocs(xml) {
114
+ const values = [];
115
+ for (const match of xml.matchAll(/<loc\b[^>]*>([\s\S]*?)<\/loc>/gi)) {
116
+ values.push(unescapeXml(match[1].trim()));
117
+ }
118
+ return values;
119
+ }
120
+
121
+ function unescapeXml(value) {
122
+ return value.replace(/&(amp|lt|gt|quot|apos);/g, (_, name) => XML_ENTITIES[name]);
123
+ }
124
+
125
+ // Converts source values to app-relative routes and drops unsafe or off-origin entries.
126
+ function normaliseAndFilter(
127
+ values,
128
+ source,
129
+ config = null,
130
+ sourceOrigin = null,
131
+ enforceOrigin = false,
132
+ ) {
133
+ const siteOrigin = config?.metadata?.siteUrl ? new URL(config.metadata.siteUrl).origin : null;
134
+ const referenceOrigin = siteOrigin ?? sourceOrigin;
135
+ const base = config?.base ?? '/';
136
+ const routes = [];
137
+ const seen = new Set();
138
+ let skipped = 0;
139
+ for (const value of values) {
140
+ if (typeof value !== 'string') {
141
+ skipped += 1;
142
+ continue;
143
+ }
144
+ const absolute = parseAbsolute(value, referenceOrigin);
145
+ if (/^https?:\/\//i.test(value) && !absolute) {
146
+ skipped += 1;
147
+ continue;
148
+ }
149
+ let route;
150
+ if (absolute) {
151
+ if (!referenceOrigin) {
152
+ if (enforceOrigin) {
153
+ skipped += 1;
154
+ continue;
155
+ }
156
+ } else if (absolute.origin !== referenceOrigin) {
157
+ skipped += 1;
158
+ continue;
159
+ }
160
+ route = toAppRoute(absolute.href, base);
161
+ } else if (value.startsWith('//')) {
162
+ // A protocol-relative URL needs an origin to resolve against; without one it is
163
+ // not safe to guess which host it belongs to.
164
+ skipped += 1;
165
+ continue;
166
+ } else {
167
+ route = normaliseRoute(stripBase(value, base));
168
+ }
169
+ if (!route || !isSafeRoute(route) || seen.has(route)) {
170
+ skipped += 1;
171
+ continue;
172
+ }
173
+ seen.add(route);
174
+ routes.push(route);
175
+ }
176
+ if (routes.length === 0 && values.length > 0) {
177
+ throw new Error(`${source} did not contain any usable routes`);
178
+ }
179
+ return { routes, skipped };
180
+ }
181
+
182
+ // Resolves an absolute or protocol-relative URL, returning null for paths.
183
+ function parseAbsolute(value, referenceOrigin) {
184
+ if (/^https?:\/\//i.test(value)) {
185
+ try {
186
+ return new URL(value);
187
+ } catch {
188
+ return null;
189
+ }
190
+ }
191
+ if (value.startsWith('//') && referenceOrigin) {
192
+ try {
193
+ return new URL(value, referenceOrigin);
194
+ } catch {
195
+ return null;
196
+ }
197
+ }
198
+ return null;
199
+ }
200
+
201
+ // Removes the public base prefix from a root-relative path, leaving app-relative paths
202
+ // alone so a sitemap can list either form.
203
+ function stripBase(value, base) {
204
+ if (base === '/') return value;
205
+ const prefix = base.slice(0, -1);
206
+ if (value === prefix) return '/';
207
+ if (value.startsWith(`${prefix}/`)) return value.slice(prefix.length);
208
+ return value;
209
+ }
@@ -43,6 +43,15 @@ export function matchesPattern(route, pattern) {
43
43
  function matchesAny(route, patterns) {
44
44
  return patterns.some((pattern) => matchesPattern(route, pattern));
45
45
  }
46
+
47
+ /**
48
+ * Reports whether a route belongs in the output. An empty pattern list allows every
49
+ * route; a supplied list is an allowlist for output as well as for discovery, so a
50
+ * seed used only to reach wildcard routes is not written itself.
51
+ */
52
+ export function isIncludedRoute(route, patterns = []) {
53
+ return patterns.length === 0 || matchesAny(route, patterns);
54
+ }
46
55
  /**
47
56
  * Reports whether a route is safe to map onto the output directory. Literal dot
48
57
  * segments are already resolved, so this catches the encoded forms: percent-encoded