prerender-crawler 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/crawl.js CHANGED
@@ -1,17 +1,20 @@
1
1
  import { mkdir, writeFile } from "node:fs/promises";
2
- import { dirname, join } from "node:path";
3
- import { extractLinks, normalizeLink, normalizePath } from "./links.js";
2
+ import { dirname, resolve } from "node:path";
3
+ import { extractLinks, normalizeLink, normalizeRoute, splitRoute } from "./links.js";
4
4
  import { outputFilename } from "./output.js";
5
5
  const wait = (ms) => new Promise(resolve => setTimeout(resolve, ms));
6
- async function resolveSeeds(pages) {
6
+ async function resolveSeeds(pages, origin, links) {
7
7
  const source = typeof pages === "function" ? await pages() : (pages ?? ["/"]);
8
- // Seed sources overlap routinely (an explicit list plus a route-manifest
9
- // scan both naming `/`): one render per path, the first spelling wins.
8
+ // Seeds are spelled by people and route manifests — `about/`, `/a#top`,
9
+ // `/posts?page=2` and get the same normalization a crawled link does,
10
+ // so a seed and a link to the same page meet in one queue entry. Seed
11
+ // sources overlap routinely (an explicit list plus a route-manifest scan
12
+ // both naming `/`): one render per path, the first spelling wins.
10
13
  const byPath = new Map();
11
14
  for (const entry of source) {
12
- const page = typeof entry === "string"
13
- ? { path: normalizePath(entry) }
14
- : { ...entry, path: normalizePath(entry.path) };
15
+ const spelled = typeof entry === "string" ? entry : entry.path;
16
+ const path = normalizeRoute(new URL(spelled, origin), links);
17
+ const page = typeof entry === "string" ? { path } : { ...entry, path };
15
18
  if (!byPath.has(page.path))
16
19
  byPath.set(page.path, page);
17
20
  }
@@ -24,20 +27,37 @@ async function resolveSeeds(pages) {
24
27
  * identical loop.
25
28
  */
26
29
  export async function runPrerender(options) {
27
- const { transport, outDir, mode = "static", crawlLinks = true, hintHeader = "x-prerender", filter, concurrency = 8, interval = 0, retries = 2, retryDelay = 500, failOnError = true, maxRedirects = 5,
30
+ const { transport, outDir, mode = "static", crawlLinks = true, hintHeader = "x-prerender", filter, keepQuery = false, concurrency = 8, interval = 0, retries = 2, retryDelay = 500, failOnError = true,
28
31
  // hybrid's crawl bakes data; writing its HTML would shadow live SSR
29
- emitPages = mode === "static", autoSubfolderIndex = true, origin = "http://localhost", onRendered, integrations = [] } = options;
32
+ emitPages = mode === "static", autoSubfolderIndex = true, origin = "http://localhost", onRendered, integrations = [],
33
+ // an integration turning redirects into host rules makes stubs
34
+ // redundant — and on hosts where a file shadows a rule, harmful
35
+ redirectStubs = !integrations.some(integration => integration.handlesRedirects) } = options;
30
36
  const originUrl = new URL(origin);
37
+ const links = { keepQuery };
31
38
  const rendered = [];
39
+ const redirects = [];
32
40
  const skipped = [];
33
41
  const emitted = [];
34
42
  const context = {
35
43
  mode,
36
44
  origin,
37
45
  outDir,
46
+ pages: rendered,
47
+ redirects,
48
+ skipped,
49
+ files: emitted,
38
50
  emitFile: file => void emitted.push(file)
39
51
  };
40
- const seeds = await resolveSeeds(options.pages);
52
+ // A query spelling without a filename of its own has nowhere correct to
53
+ // go: `posts/index.html` is `/posts`, and a static host serves it for
54
+ // every query. It renders (data, links) and leaves no file.
55
+ const shouldEmit = (entry) => {
56
+ if (!entry.filename && entry.path.includes("?"))
57
+ return false;
58
+ return entry.emit ?? (typeof emitPages === "function" ? emitPages(entry.path) : emitPages);
59
+ };
60
+ const seeds = await resolveSeeds(options.pages, originUrl, links);
41
61
  const seen = new Set(seeds.map(page => page.path));
42
62
  const queue = [...seeds];
43
63
  // Provenance: which pages named each discovered path. Recorded for every
@@ -57,46 +77,73 @@ export async function runPrerender(options) {
57
77
  seen.add(path);
58
78
  queue.push({ path });
59
79
  };
60
- // The throttle: every request start claims the next slot on a shared
61
- // timeline, so starts are at least `interval` apart no matter how many
62
- // workers are running.
80
+ // The throttle. Two mechanisms, because the guarantee is about ACTUAL
81
+ // starts: claiming the next slot on a shared timeline keeps concurrent
82
+ // workers apart in advance, and holding until `interval` has passed since
83
+ // the last real start covers the case a claim cannot — a start that ran
84
+ // late (busy event loop) must not be crowded by the next one's on-time
85
+ // slot. The hold's check-then-record runs without interleaving, so two
86
+ // workers waking together cannot both pass it.
63
87
  let nextSlot = 0;
88
+ let lastStart = -Infinity;
64
89
  async function pace() {
65
90
  if (interval <= 0)
66
91
  return;
67
- const now = Date.now();
92
+ const now = performance.now();
68
93
  const slot = Math.max(now, nextSlot);
69
94
  nextSlot = slot + interval;
70
95
  if (slot > now)
71
96
  await wait(slot - now);
97
+ let started = performance.now();
98
+ while (started - lastStart < interval) {
99
+ await wait(lastStart + interval - started);
100
+ started = performance.now();
101
+ }
102
+ lastStart = started;
103
+ if (nextSlot < started + interval)
104
+ nextSlot = started + interval;
72
105
  }
73
- async function fetchFollowingRedirects(path) {
74
- let url = new URL(path, originUrl);
75
- for (let hop = 0;; hop++) {
76
- await pace();
77
- const response = await transport.fetch(new Request(url, { headers: { accept: "text/html,*/*", [hintHeader]: "1" } }));
78
- const location = response.headers.get("location");
79
- if (response.status < 300 || response.status >= 400 || !location)
80
- return response;
81
- if (hop >= maxRedirects) {
82
- throw new Error(`Redirect chain from ${path} exceeded ${maxRedirects} hops`);
83
- }
84
- const target = new URL(location, url);
85
- if (target.origin !== originUrl.origin) {
86
- // an external redirect terminates the chain; the page becomes a stub
87
- return response;
88
- }
89
- url = target;
106
+ // Redirects are not followed in place: a 3xx makes the path a redirect
107
+ // record (and a stub, see finalizeRedirects), and its same-origin target
108
+ // enters the queue as a page in its own right — so the destination is
109
+ // rendered once, at its own URL, and a chain is one record per hop the
110
+ // way a host's redirect rules would spell it. Cycles are harmless: the
111
+ // seen-set admits each path once.
112
+ // `started` is taken after pacing: a page's duration is its own, not the
113
+ // throttle's.
114
+ async function fetchUrl(url) {
115
+ await pace();
116
+ const started = performance.now();
117
+ const response = await transport.fetch(new Request(url, { headers: { accept: "text/html,*/*", [hintHeader]: "1" } }));
118
+ return { response, started };
119
+ }
120
+ // A redirect to a spelling of the SAME page (`/posts` -> `/posts/`, the
121
+ // trailing-slash canonicalization static servers do) is not a redirect
122
+ // between pages: it is followed here, once, and the page renders as
123
+ // itself. Anything else is the caller's to record.
124
+ async function fetchPage(path) {
125
+ const url = new URL(path, originUrl);
126
+ const fetched = await fetchUrl(url);
127
+ const { response } = fetched;
128
+ const location = response.headers.get("location");
129
+ if (!location || response.status < 300 || response.status >= 400)
130
+ return fetched;
131
+ const target = new URL(location, url);
132
+ if (target.origin !== originUrl.origin || normalizeRoute(target, links) !== path) {
133
+ return fetched;
90
134
  }
135
+ return { ...(await fetchUrl(target)), started: fetched.started };
91
136
  }
137
+ const pendingRedirects = [];
92
138
  async function renderPage(entry) {
93
139
  let response;
140
+ let started = 0;
94
141
  let error;
95
142
  for (let attempt = 0; attempt <= retries; attempt++) {
96
143
  if (attempt > 0)
97
144
  await wait(retryDelay);
98
145
  try {
99
- response = await fetchFollowingRedirects(entry.path);
146
+ ({ response, started } = await fetchPage(entry.path));
100
147
  error = undefined;
101
148
  if (response.status < 500)
102
149
  break; // retry only what might heal
@@ -122,40 +169,43 @@ export async function runPrerender(options) {
122
169
  skipped.push({ path: entry.path, error: failure, referrers: [] });
123
170
  return;
124
171
  }
125
- const filename = entry.filename ?? outputFilename(entry.path, autoSubfolderIndex);
126
- let html;
127
- const contentType = response.headers.get("content-type") ?? "";
128
- const isHTML = contentType.includes("text/html");
172
+ const filename = entry.filename ?? outputFilename(splitRoute(entry.path).pathname, autoSubfolderIndex);
173
+ const hints = response.headers.get(hintHeader);
174
+ if (hints) {
175
+ for (const hint of hints.split(",")) {
176
+ const path = normalizeLink(hint.trim(), originUrl, originUrl.origin, links);
177
+ if (path !== undefined)
178
+ discovered(path, entry.path);
179
+ }
180
+ }
129
181
  const location = response.headers.get("location");
130
182
  if (location && response.status >= 300 && response.status < 400) {
131
- // external redirect (internal ones were followed): a meta-refresh stub
132
- // keeps the path working on hosts without redirect support
133
- html = redirectStub(location);
134
- }
135
- else {
136
- html = await response.text();
183
+ const pageUrl = new URL(entry.path, originUrl);
184
+ const target = new URL(location, pageUrl);
185
+ const internal = target.origin === originUrl.origin;
186
+ const to = internal ? normalizeRoute(target, links) : target.href;
187
+ const redirect = { from: entry.path, to, status: response.status };
188
+ redirects.push(redirect);
189
+ if (internal)
190
+ discovered(to, entry.path);
191
+ // the stub needs the chain's end, known only once the crawl settles
192
+ const duration = performance.now() - started;
193
+ pendingRedirects.push({ entry, filename, response, duration, redirect });
194
+ return;
137
195
  }
196
+ const html = await response.text();
197
+ const duration = performance.now() - started;
138
198
  // Emission is policy, rendering is not: an unemitted page has still
139
199
  // fully executed (integration capture happened server-side) and its
140
200
  // links still feed the crawl — it just leaves no HTML file behind to
141
201
  // shadow a live server's SSR of the route.
142
- const emitted = entry.emit ?? (typeof emitPages === "function" ? emitPages(entry.path) : emitPages);
202
+ const emitted = shouldEmit(entry);
143
203
  if (emitted)
144
204
  await writeOutput(outDir, filename, html);
145
- if (isHTML) {
205
+ if (crawlLinks && (response.headers.get("content-type") ?? "").includes("text/html")) {
146
206
  const pageUrl = new URL(entry.path, originUrl);
147
- if (crawlLinks) {
148
- for (const path of extractLinks(html, pageUrl))
149
- discovered(path, entry.path);
150
- }
151
- }
152
- const hints = response.headers.get(hintHeader);
153
- if (hints) {
154
- for (const hint of hints.split(",")) {
155
- const path = normalizeLink(hint.trim(), originUrl, originUrl.origin);
156
- if (path !== undefined)
157
- discovered(path, entry.path);
158
- }
207
+ for (const path of extractLinks(html, pageUrl, links))
208
+ discovered(path, entry.path);
159
209
  }
160
210
  const page = {
161
211
  path: entry.path,
@@ -163,12 +213,51 @@ export async function runPrerender(options) {
163
213
  filename,
164
214
  emitted,
165
215
  response,
216
+ duration,
166
217
  html
167
218
  };
168
219
  rendered.push(page);
169
220
  if (onRendered)
170
221
  await onRendered(page);
171
222
  }
223
+ // Redirected paths become pages holding a meta-refresh stub — the old URL
224
+ // keeps working on hosts with no redirect support — pointing straight at
225
+ // the chain's final destination so a visitor hops once, not per record.
226
+ async function finalizeRedirects() {
227
+ const hops = new Map(redirects.map(record => [record.from, record.to]));
228
+ // the chain's end — or, in a cycle (which has none), one hop on
229
+ const destination = (from) => {
230
+ const next = hops.get(from);
231
+ const visited = new Set([from]);
232
+ let at = next;
233
+ while (hops.has(at)) {
234
+ if (visited.has(at))
235
+ return next;
236
+ visited.add(at);
237
+ at = hops.get(at);
238
+ }
239
+ return at;
240
+ };
241
+ for (const { entry, filename, response, duration, redirect } of pendingRedirects) {
242
+ const html = redirectStub(destination(entry.path));
243
+ const emitted = redirectStubs && shouldEmit(entry);
244
+ if (emitted)
245
+ await writeOutput(outDir, filename, html);
246
+ const page = {
247
+ path: entry.path,
248
+ referrers: referrersOf(entry.path),
249
+ filename,
250
+ emitted,
251
+ response,
252
+ duration,
253
+ html,
254
+ redirect
255
+ };
256
+ rendered.push(page);
257
+ if (onRendered)
258
+ await onRendered(page);
259
+ }
260
+ }
172
261
  try {
173
262
  for (const integration of integrations)
174
263
  await integration.setup?.(context);
@@ -199,6 +288,7 @@ export async function runPrerender(options) {
199
288
  });
200
289
  for (const miss of skipped)
201
290
  miss.referrers = referrersOf(miss.path);
291
+ await finalizeRedirects();
202
292
  for (const integration of integrations)
203
293
  await integration.teardown?.(context);
204
294
  for (const file of emitted) {
@@ -208,15 +298,15 @@ export async function runPrerender(options) {
208
298
  finally {
209
299
  await transport.close?.();
210
300
  }
211
- return { pages: rendered, files: emitted, skipped };
301
+ return { pages: rendered, redirects, files: emitted, skipped };
212
302
  }
213
303
  const describe = (error) => (error instanceof Error ? error.message : String(error));
214
304
  function redirectStub(location) {
215
- const target = String(location).replace(/"/g, "&quot;");
305
+ const target = location.replace(/&/g, "&amp;").replace(/"/g, "&quot;");
216
306
  return `<!DOCTYPE html><html><head><meta http-equiv="refresh" content="0; url=${target}"><link rel="canonical" href="${target}"></head></html>`;
217
307
  }
218
308
  async function writeOutput(outDir, filename, contents) {
219
- const target = join(outDir, filename);
309
+ const target = resolve(outDir, filename);
220
310
  await mkdir(dirname(target), { recursive: true });
221
311
  await writeFile(target, contents);
222
312
  }
package/dist/index.d.ts CHANGED
@@ -1,5 +1,16 @@
1
1
  export { runPrerender } from "./crawl.ts";
2
2
  export type { RunOptions } from "./crawl.ts";
3
- export { extractLinks, normalizeLink, normalizePath } from "./links.ts";
3
+ export { extractLinks, normalizeLink, normalizePath, normalizeRoute, splitRoute } from "./links.ts";
4
+ export type { LinkOptions } from "./links.ts";
4
5
  export { outputFilename } from "./output.ts";
5
- export type { EmittedFile, PageEntry, PagesSource, PrerenderContext, PrerenderIntegration, PrerenderMode, PrerenderOptions, PrerenderResult, RenderedPage, SkippedPage, Transport } from "./types.ts";
6
+ export { formatRedirectsFile, redirects } from "./redirects.ts";
7
+ export type { RedirectsIntegrationOptions } from "./redirects.ts";
8
+ export { report } from "./report.ts";
9
+ export { HINT_HEADER, announcePages } from "./announce.ts";
10
+ export type { AnnounceOptions } from "./announce.ts";
11
+ export type { PrerenderReport, ReportIntegrationOptions, ReportPage, ReportSkip } from "./report.ts";
12
+ export { formatSitemap, indexable, sitemap } from "./sitemap.ts";
13
+ export type { SitemapChangeFrequency, SitemapEntry, SitemapIntegrationOptions } from "./sitemap.ts";
14
+ export { httpTransport, loadHandler, moduleTransport } from "./transports.ts";
15
+ export type { HttpTransportOptions, RequestHandler } from "./transports.ts";
16
+ export type { EmittedFile, PageEntry, PagesSource, PrerenderContext, PrerenderIntegration, PrerenderMode, PrerenderOptions, PrerenderResult, RedirectRecord, RenderedPage, SkippedPage, Transport } from "./types.ts";
package/dist/index.js CHANGED
@@ -1,3 +1,8 @@
1
1
  export { runPrerender } from "./crawl.js";
2
- export { extractLinks, normalizeLink, normalizePath } from "./links.js";
2
+ export { extractLinks, normalizeLink, normalizePath, normalizeRoute, splitRoute } from "./links.js";
3
3
  export { outputFilename } from "./output.js";
4
+ export { formatRedirectsFile, redirects } from "./redirects.js";
5
+ export { report } from "./report.js";
6
+ export { HINT_HEADER, announcePages } from "./announce.js";
7
+ export { formatSitemap, indexable, sitemap } from "./sitemap.js";
8
+ export { httpTransport, loadHandler, moduleTransport } from "./transports.js";
package/dist/links.d.ts CHANGED
@@ -2,23 +2,39 @@
2
2
  * Link discovery: which hrefs in a rendered page name more pages of this
3
3
  * site. The rules here are correctness fixes other prerenderers earned one
4
4
  * bug report at a time — resolve relative hrefs against the PAGE's URL
5
- * (not the origin), honor <base href>, strip queries and fragments before
6
- * dedupe, and never leave the origin.
5
+ * (not the origin), honor <base href>, strip queries (unless asked to keep
6
+ * them) and fragments before dedupe, and never leave the origin.
7
7
  */
8
+ export interface LinkOptions {
9
+ /** Keep the query string as part of the page's identity. @default false */
10
+ keepQuery?: boolean;
11
+ }
8
12
  /**
9
13
  * Extracts the crawlable same-origin route paths from a page's HTML.
10
- * Returned paths are normalized (`/about`, no query, no fragment, no
11
- * trailing slash except the root) and deduped.
14
+ * Returned paths are normalized (`/about`, no fragment, no trailing slash
15
+ * except the root, query only with `keepQuery`) and deduped.
12
16
  */
13
- export declare function extractLinks(html: string, pageUrl: URL): string[];
17
+ export declare function extractLinks(html: string, pageUrl: URL, options?: LinkOptions): string[];
14
18
  /**
15
19
  * Resolves one href to a normalized same-origin path, or undefined when it
16
20
  * is not a page of this site (foreign origin, unparseable).
17
21
  */
18
- export declare function normalizeLink(href: string, base: URL, origin: string): string | undefined;
22
+ export declare function normalizeLink(href: string, base: URL, origin: string, options?: LinkOptions): string | undefined;
19
23
  /**
20
- * One spelling per page: query and fragment never reach here (URL parsing
21
- * split them off), the trailing slash is dropped (except the root), and
22
- * percent-encoding is left exactly as the URL parser produced it.
24
+ * One spelling per page: the fragment never reaches here (URL parsing split
25
+ * it off), the trailing slash is dropped (except the root), percent-encoding
26
+ * is left exactly as the URL parser produced it, and the query is kept only
27
+ * on request — with its parameters sorted, so `?a=1&b=2` and `?b=2&a=1`
28
+ * are the one page they are.
29
+ */
30
+ export declare function normalizeRoute(url: URL, options?: LinkOptions): string;
31
+ /**
32
+ * The pathname half of normalization: the trailing slash is dropped (except
33
+ * the root) and nothing else is touched.
23
34
  */
24
35
  export declare function normalizePath(pathname: string): string;
36
+ /** Splits a normalized route into its pathname and its (possibly empty) query. */
37
+ export declare function splitRoute(route: string): {
38
+ pathname: string;
39
+ search: string;
40
+ };
package/dist/links.js CHANGED
@@ -2,8 +2,8 @@
2
2
  * Link discovery: which hrefs in a rendered page name more pages of this
3
3
  * site. The rules here are correctness fixes other prerenderers earned one
4
4
  * bug report at a time — resolve relative hrefs against the PAGE's URL
5
- * (not the origin), honor <base href>, strip queries and fragments before
6
- * dedupe, and never leave the origin.
5
+ * (not the origin), honor <base href>, strip queries (unless asked to keep
6
+ * them) and fragments before dedupe, and never leave the origin.
7
7
  */
8
8
  const LINK_PATTERN = /<a\s[^>]*?href\s*=\s*(?:"([^"]*)"|'([^']*)')/gis;
9
9
  const BASE_PATTERN = /<base\s[^>]*?href\s*=\s*(?:"([^"]*)"|'([^']*)')/is;
@@ -11,10 +11,10 @@ const BASE_PATTERN = /<base\s[^>]*?href\s*=\s*(?:"([^"]*)"|'([^']*)')/is;
11
11
  const NON_PAGE_SCHEME = /^(?:mailto|tel|javascript|data|blob|about):/i;
12
12
  /**
13
13
  * Extracts the crawlable same-origin route paths from a page's HTML.
14
- * Returned paths are normalized (`/about`, no query, no fragment, no
15
- * trailing slash except the root) and deduped.
14
+ * Returned paths are normalized (`/about`, no fragment, no trailing slash
15
+ * except the root, query only with `keepQuery`) and deduped.
16
16
  */
17
- export function extractLinks(html, pageUrl) {
17
+ export function extractLinks(html, pageUrl, options = {}) {
18
18
  // <base href> shifts what relative hrefs resolve against, exactly as the
19
19
  // browser would resolve them.
20
20
  const baseMatch = BASE_PATTERN.exec(html);
@@ -33,7 +33,7 @@ export function extractLinks(html, pageUrl) {
33
33
  const href = (match[1] ?? match[2] ?? "").trim();
34
34
  if (!href || NON_PAGE_SCHEME.test(href))
35
35
  continue;
36
- const path = normalizeLink(href, base, pageUrl.origin);
36
+ const path = normalizeLink(href, base, pageUrl.origin, options);
37
37
  if (path !== undefined)
38
38
  found.add(path);
39
39
  }
@@ -43,7 +43,7 @@ export function extractLinks(html, pageUrl) {
43
43
  * Resolves one href to a normalized same-origin path, or undefined when it
44
44
  * is not a page of this site (foreign origin, unparseable).
45
45
  */
46
- export function normalizeLink(href, base, origin) {
46
+ export function normalizeLink(href, base, origin, options = {}) {
47
47
  let url;
48
48
  try {
49
49
  url = new URL(href, base);
@@ -53,15 +53,37 @@ export function normalizeLink(href, base, origin) {
53
53
  }
54
54
  if (url.origin !== origin)
55
55
  return undefined;
56
- return normalizePath(url.pathname);
56
+ return normalizeRoute(url, options);
57
57
  }
58
58
  /**
59
- * One spelling per page: query and fragment never reach here (URL parsing
60
- * split them off), the trailing slash is dropped (except the root), and
61
- * percent-encoding is left exactly as the URL parser produced it.
59
+ * One spelling per page: the fragment never reaches here (URL parsing split
60
+ * it off), the trailing slash is dropped (except the root), percent-encoding
61
+ * is left exactly as the URL parser produced it, and the query is kept only
62
+ * on request — with its parameters sorted, so `?a=1&b=2` and `?b=2&a=1`
63
+ * are the one page they are.
64
+ */
65
+ export function normalizeRoute(url, options = {}) {
66
+ const path = normalizePath(url.pathname);
67
+ if (!options.keepQuery || !url.search)
68
+ return path;
69
+ const params = new URLSearchParams(url.search);
70
+ params.sort();
71
+ const query = params.toString();
72
+ return query ? `${path}?${query}` : path;
73
+ }
74
+ /**
75
+ * The pathname half of normalization: the trailing slash is dropped (except
76
+ * the root) and nothing else is touched.
62
77
  */
63
78
  export function normalizePath(pathname) {
64
79
  if (pathname === "" || pathname === "/")
65
80
  return "/";
66
81
  return pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
67
82
  }
83
+ /** Splits a normalized route into its pathname and its (possibly empty) query. */
84
+ export function splitRoute(route) {
85
+ const at = route.indexOf("?");
86
+ return at === -1
87
+ ? { pathname: route, search: "" }
88
+ : { pathname: route.slice(0, at), search: route.slice(at) };
89
+ }
@@ -0,0 +1,33 @@
1
+ import type { PrerenderIntegration, RedirectRecord } from "./types.ts";
2
+ export interface RedirectsIntegrationOptions {
3
+ /** Output file, relative to the output directory. @default "_redirects" */
4
+ filename?: string;
5
+ /**
6
+ * Produces the file's contents from the run's redirects — for hosts with
7
+ * their own format. Defaults to the `_redirects` line format.
8
+ */
9
+ format?(redirects: readonly RedirectRecord[]): string;
10
+ /**
11
+ * Netlify only: append `!` to force each rule past an existing file at
12
+ * its path. Unneeded when the engine writes no stubs (the default with
13
+ * this integration active), and Cloudflare Pages rejects the syntax.
14
+ * @default false
15
+ */
16
+ force?: boolean;
17
+ }
18
+ /**
19
+ * Emits the crawl's redirects as host rules — by default a `_redirects` file
20
+ * (Netlify, Cloudflare Pages). Declares `handlesRedirects`, so the engine
21
+ * writes no meta-refresh stubs at redirected paths.
22
+ *
23
+ * ```ts
24
+ * import { redirects } from "prerender-crawler";
25
+ * prerender({ integrations: [redirects()] })
26
+ * ```
27
+ */
28
+ export declare function redirects(options?: RedirectsIntegrationOptions): PrerenderIntegration;
29
+ /**
30
+ * The `_redirects` line format: `/from /to status`, one rule per line,
31
+ * sorted by source for a stable file across builds.
32
+ */
33
+ export declare function formatRedirectsFile(redirects: readonly RedirectRecord[], force?: boolean): string;
@@ -0,0 +1,33 @@
1
+ /**
2
+ * Emits the crawl's redirects as host rules — by default a `_redirects` file
3
+ * (Netlify, Cloudflare Pages). Declares `handlesRedirects`, so the engine
4
+ * writes no meta-refresh stubs at redirected paths.
5
+ *
6
+ * ```ts
7
+ * import { redirects } from "prerender-crawler";
8
+ * prerender({ integrations: [redirects()] })
9
+ * ```
10
+ */
11
+ export function redirects(options = {}) {
12
+ const { filename = "_redirects", force = false } = options;
13
+ const format = options.format ?? (records => formatRedirectsFile(records, force));
14
+ return {
15
+ name: "redirects",
16
+ handlesRedirects: true,
17
+ teardown(context) {
18
+ if (context.redirects.length === 0)
19
+ return;
20
+ context.emitFile({ filename, contents: format(context.redirects) });
21
+ }
22
+ };
23
+ }
24
+ /**
25
+ * The `_redirects` line format: `/from /to status`, one rule per line,
26
+ * sorted by source for a stable file across builds.
27
+ */
28
+ export function formatRedirectsFile(redirects, force = false) {
29
+ const lines = [...redirects]
30
+ .sort((a, b) => (a.from < b.from ? -1 : a.from > b.from ? 1 : 0))
31
+ .map(({ from, to, status }) => `${from} ${to} ${status}${force ? "!" : ""}`);
32
+ return lines.join("\n") + "\n";
33
+ }
@@ -0,0 +1,55 @@
1
+ import type { PrerenderIntegration, RedirectRecord } from "./types.ts";
2
+ /**
3
+ * The report integration: what the crawl did, as JSON — every page with
4
+ * its status, timing, output file and the pages that linked to it; every
5
+ * redirect; every skipped page and why; every file other integrations
6
+ * emitted. The answers to "why was this page crawled", "which page is
7
+ * slow" and "what did the build actually produce" are all in here, and
8
+ * they are otherwise gone the moment the process exits.
9
+ */
10
+ export interface ReportIntegrationOptions {
11
+ /**
12
+ * Where to write, resolved against the output directory. The default
13
+ * lands inside it and so deploys with the site; point it outside
14
+ * (`"../prerender-report.json"`) to keep it a build artifact.
15
+ * @default "prerender-report.json"
16
+ */
17
+ filename?: string;
18
+ }
19
+ export interface PrerenderReport {
20
+ generatedAt: string;
21
+ mode: string;
22
+ origin: string;
23
+ totals: {
24
+ pages: number;
25
+ written: number;
26
+ redirects: number;
27
+ skipped: number;
28
+ files: number;
29
+ /** Sum of page durations, milliseconds — render cost, not wall time. */
30
+ duration: number;
31
+ };
32
+ pages: ReportPage[];
33
+ redirects: RedirectRecord[];
34
+ skipped: ReportSkip[];
35
+ /** Files emitted by integrations that ran before this one, this run's pages excluded. */
36
+ files: string[];
37
+ }
38
+ export interface ReportPage {
39
+ path: string;
40
+ status: number;
41
+ contentType: string | null;
42
+ /** Milliseconds, the successful attempt. */
43
+ duration: number;
44
+ filename: string;
45
+ /** Whether the HTML was written (`false` in hybrid mode, for query spellings, ...). */
46
+ written: boolean;
47
+ referrers: string[];
48
+ redirect?: RedirectRecord;
49
+ }
50
+ export interface ReportSkip {
51
+ path: string;
52
+ error: string;
53
+ referrers: string[];
54
+ }
55
+ export declare function report(options?: ReportIntegrationOptions): PrerenderIntegration;
package/dist/report.js ADDED
@@ -0,0 +1,46 @@
1
+ export function report(options = {}) {
2
+ const { filename = "prerender-report.json" } = options;
3
+ return {
4
+ name: "report",
5
+ teardown(context) {
6
+ const pages = context.pages
7
+ .map(page => ({
8
+ path: page.path,
9
+ status: page.response.status,
10
+ contentType: page.response.headers.get("content-type"),
11
+ duration: Math.round(page.duration * 100) / 100,
12
+ filename: page.filename,
13
+ written: page.emitted,
14
+ referrers: [...page.referrers].sort(),
15
+ ...(page.redirect ? { redirect: page.redirect } : {})
16
+ }))
17
+ .sort(byPath);
18
+ const skipped = context.skipped
19
+ .map(miss => ({
20
+ path: miss.path,
21
+ error: miss.error instanceof Error ? miss.error.message : String(miss.error),
22
+ referrers: [...miss.referrers].sort()
23
+ }))
24
+ .sort(byPath);
25
+ const summary = {
26
+ generatedAt: new Date().toISOString(),
27
+ mode: context.mode,
28
+ origin: context.origin,
29
+ totals: {
30
+ pages: pages.length,
31
+ written: pages.filter(page => page.written).length,
32
+ redirects: context.redirects.length,
33
+ skipped: skipped.length,
34
+ files: context.files.length,
35
+ duration: Math.round(pages.reduce((sum, page) => sum + page.duration, 0))
36
+ },
37
+ pages,
38
+ redirects: [...context.redirects],
39
+ skipped,
40
+ files: context.files.map(file => file.filename)
41
+ };
42
+ context.emitFile({ filename, contents: JSON.stringify(summary, null, 2) + "\n" });
43
+ }
44
+ };
45
+ }
46
+ const byPath = (a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0;