prerender-crawler 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +152 -35
- package/dist/announce.d.ts +39 -0
- package/dist/announce.js +41 -0
- package/dist/cli-main.d.ts +7 -0
- package/dist/cli-main.js +181 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +6 -0
- package/dist/crawl.js +150 -60
- package/dist/index.d.ts +13 -2
- package/dist/index.js +6 -1
- package/dist/links.d.ts +25 -9
- package/dist/links.js +33 -11
- package/dist/redirects.d.ts +33 -0
- package/dist/redirects.js +33 -0
- package/dist/report.d.ts +55 -0
- package/dist/report.js +46 -0
- package/dist/sitemap.d.ts +56 -0
- package/dist/sitemap.js +88 -0
- package/dist/transports.d.ts +35 -0
- package/dist/transports.js +74 -0
- package/dist/types.d.ts +69 -3
- package/dist/vite.d.ts +10 -18
- package/dist/vite.js +29 -70
- package/package.json +8 -6
- package/dist/file-routes.d.ts +0 -35
- package/dist/file-routes.js +0 -84
package/dist/crawl.js
CHANGED
|
@@ -1,17 +1,20 @@
|
|
|
1
1
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
2
|
-
import { dirname,
|
|
3
|
-
import { extractLinks, normalizeLink,
|
|
2
|
+
import { dirname, resolve } from "node:path";
|
|
3
|
+
import { extractLinks, normalizeLink, normalizeRoute, splitRoute } from "./links.js";
|
|
4
4
|
import { outputFilename } from "./output.js";
|
|
5
5
|
const wait = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
6
|
-
async function resolveSeeds(pages) {
|
|
6
|
+
async function resolveSeeds(pages, origin, links) {
|
|
7
7
|
const source = typeof pages === "function" ? await pages() : (pages ?? ["/"]);
|
|
8
|
-
//
|
|
9
|
-
//
|
|
8
|
+
// Seeds are spelled by people and route manifests — `about/`, `/a#top`,
|
|
9
|
+
// `/posts?page=2` — and get the same normalization a crawled link does,
|
|
10
|
+
// so a seed and a link to the same page meet in one queue entry. Seed
|
|
11
|
+
// sources overlap routinely (an explicit list plus a route-manifest scan
|
|
12
|
+
// both naming `/`): one render per path, the first spelling wins.
|
|
10
13
|
const byPath = new Map();
|
|
11
14
|
for (const entry of source) {
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
|
|
15
|
+
const spelled = typeof entry === "string" ? entry : entry.path;
|
|
16
|
+
const path = normalizeRoute(new URL(spelled, origin), links);
|
|
17
|
+
const page = typeof entry === "string" ? { path } : { ...entry, path };
|
|
15
18
|
if (!byPath.has(page.path))
|
|
16
19
|
byPath.set(page.path, page);
|
|
17
20
|
}
|
|
@@ -24,20 +27,37 @@ async function resolveSeeds(pages) {
|
|
|
24
27
|
* identical loop.
|
|
25
28
|
*/
|
|
26
29
|
export async function runPrerender(options) {
|
|
27
|
-
const { transport, outDir, mode = "static", crawlLinks = true, hintHeader = "x-prerender", filter, concurrency = 8, interval = 0, retries = 2, retryDelay = 500, failOnError = true,
|
|
30
|
+
const { transport, outDir, mode = "static", crawlLinks = true, hintHeader = "x-prerender", filter, keepQuery = false, concurrency = 8, interval = 0, retries = 2, retryDelay = 500, failOnError = true,
|
|
28
31
|
// hybrid's crawl bakes data; writing its HTML would shadow live SSR
|
|
29
|
-
emitPages = mode === "static", autoSubfolderIndex = true, origin = "http://localhost", onRendered, integrations = []
|
|
32
|
+
emitPages = mode === "static", autoSubfolderIndex = true, origin = "http://localhost", onRendered, integrations = [],
|
|
33
|
+
// an integration turning redirects into host rules makes stubs
|
|
34
|
+
// redundant — and on hosts where a file shadows a rule, harmful
|
|
35
|
+
redirectStubs = !integrations.some(integration => integration.handlesRedirects) } = options;
|
|
30
36
|
const originUrl = new URL(origin);
|
|
37
|
+
const links = { keepQuery };
|
|
31
38
|
const rendered = [];
|
|
39
|
+
const redirects = [];
|
|
32
40
|
const skipped = [];
|
|
33
41
|
const emitted = [];
|
|
34
42
|
const context = {
|
|
35
43
|
mode,
|
|
36
44
|
origin,
|
|
37
45
|
outDir,
|
|
46
|
+
pages: rendered,
|
|
47
|
+
redirects,
|
|
48
|
+
skipped,
|
|
49
|
+
files: emitted,
|
|
38
50
|
emitFile: file => void emitted.push(file)
|
|
39
51
|
};
|
|
40
|
-
|
|
52
|
+
// A query spelling without a filename of its own has nowhere correct to
|
|
53
|
+
// go: `posts/index.html` is `/posts`, and a static host serves it for
|
|
54
|
+
// every query. It renders (data, links) and leaves no file.
|
|
55
|
+
const shouldEmit = (entry) => {
|
|
56
|
+
if (!entry.filename && entry.path.includes("?"))
|
|
57
|
+
return false;
|
|
58
|
+
return entry.emit ?? (typeof emitPages === "function" ? emitPages(entry.path) : emitPages);
|
|
59
|
+
};
|
|
60
|
+
const seeds = await resolveSeeds(options.pages, originUrl, links);
|
|
41
61
|
const seen = new Set(seeds.map(page => page.path));
|
|
42
62
|
const queue = [...seeds];
|
|
43
63
|
// Provenance: which pages named each discovered path. Recorded for every
|
|
@@ -57,46 +77,73 @@ export async function runPrerender(options) {
|
|
|
57
77
|
seen.add(path);
|
|
58
78
|
queue.push({ path });
|
|
59
79
|
};
|
|
60
|
-
// The throttle
|
|
61
|
-
//
|
|
62
|
-
// workers
|
|
80
|
+
// The throttle. Two mechanisms, because the guarantee is about ACTUAL
|
|
81
|
+
// starts: claiming the next slot on a shared timeline keeps concurrent
|
|
82
|
+
// workers apart in advance, and holding until `interval` has passed since
|
|
83
|
+
// the last real start covers the case a claim cannot — a start that ran
|
|
84
|
+
// late (busy event loop) must not be crowded by the next one's on-time
|
|
85
|
+
// slot. The hold's check-then-record runs without interleaving, so two
|
|
86
|
+
// workers waking together cannot both pass it.
|
|
63
87
|
let nextSlot = 0;
|
|
88
|
+
let lastStart = -Infinity;
|
|
64
89
|
async function pace() {
|
|
65
90
|
if (interval <= 0)
|
|
66
91
|
return;
|
|
67
|
-
const now =
|
|
92
|
+
const now = performance.now();
|
|
68
93
|
const slot = Math.max(now, nextSlot);
|
|
69
94
|
nextSlot = slot + interval;
|
|
70
95
|
if (slot > now)
|
|
71
96
|
await wait(slot - now);
|
|
97
|
+
let started = performance.now();
|
|
98
|
+
while (started - lastStart < interval) {
|
|
99
|
+
await wait(lastStart + interval - started);
|
|
100
|
+
started = performance.now();
|
|
101
|
+
}
|
|
102
|
+
lastStart = started;
|
|
103
|
+
if (nextSlot < started + interval)
|
|
104
|
+
nextSlot = started + interval;
|
|
72
105
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
106
|
+
// Redirects are not followed in place: a 3xx makes the path a redirect
|
|
107
|
+
// record (and a stub, see finalizeRedirects), and its same-origin target
|
|
108
|
+
// enters the queue as a page in its own right — so the destination is
|
|
109
|
+
// rendered once, at its own URL, and a chain is one record per hop the
|
|
110
|
+
// way a host's redirect rules would spell it. Cycles are harmless: the
|
|
111
|
+
// seen-set admits each path once.
|
|
112
|
+
// `started` is taken after pacing: a page's duration is its own, not the
|
|
113
|
+
// throttle's.
|
|
114
|
+
async function fetchUrl(url) {
|
|
115
|
+
await pace();
|
|
116
|
+
const started = performance.now();
|
|
117
|
+
const response = await transport.fetch(new Request(url, { headers: { accept: "text/html,*/*", [hintHeader]: "1" } }));
|
|
118
|
+
return { response, started };
|
|
119
|
+
}
|
|
120
|
+
// A redirect to a spelling of the SAME page (`/posts` -> `/posts/`, the
|
|
121
|
+
// trailing-slash canonicalization static servers do) is not a redirect
|
|
122
|
+
// between pages: it is followed here, once, and the page renders as
|
|
123
|
+
// itself. Anything else is the caller's to record.
|
|
124
|
+
async function fetchPage(path) {
|
|
125
|
+
const url = new URL(path, originUrl);
|
|
126
|
+
const fetched = await fetchUrl(url);
|
|
127
|
+
const { response } = fetched;
|
|
128
|
+
const location = response.headers.get("location");
|
|
129
|
+
if (!location || response.status < 300 || response.status >= 400)
|
|
130
|
+
return fetched;
|
|
131
|
+
const target = new URL(location, url);
|
|
132
|
+
if (target.origin !== originUrl.origin || normalizeRoute(target, links) !== path) {
|
|
133
|
+
return fetched;
|
|
90
134
|
}
|
|
135
|
+
return { ...(await fetchUrl(target)), started: fetched.started };
|
|
91
136
|
}
|
|
137
|
+
const pendingRedirects = [];
|
|
92
138
|
async function renderPage(entry) {
|
|
93
139
|
let response;
|
|
140
|
+
let started = 0;
|
|
94
141
|
let error;
|
|
95
142
|
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
96
143
|
if (attempt > 0)
|
|
97
144
|
await wait(retryDelay);
|
|
98
145
|
try {
|
|
99
|
-
response = await
|
|
146
|
+
({ response, started } = await fetchPage(entry.path));
|
|
100
147
|
error = undefined;
|
|
101
148
|
if (response.status < 500)
|
|
102
149
|
break; // retry only what might heal
|
|
@@ -122,40 +169,43 @@ export async function runPrerender(options) {
|
|
|
122
169
|
skipped.push({ path: entry.path, error: failure, referrers: [] });
|
|
123
170
|
return;
|
|
124
171
|
}
|
|
125
|
-
const filename = entry.filename ?? outputFilename(entry.path, autoSubfolderIndex);
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
172
|
+
const filename = entry.filename ?? outputFilename(splitRoute(entry.path).pathname, autoSubfolderIndex);
|
|
173
|
+
const hints = response.headers.get(hintHeader);
|
|
174
|
+
if (hints) {
|
|
175
|
+
for (const hint of hints.split(",")) {
|
|
176
|
+
const path = normalizeLink(hint.trim(), originUrl, originUrl.origin, links);
|
|
177
|
+
if (path !== undefined)
|
|
178
|
+
discovered(path, entry.path);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
129
181
|
const location = response.headers.get("location");
|
|
130
182
|
if (location && response.status >= 300 && response.status < 400) {
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
183
|
+
const pageUrl = new URL(entry.path, originUrl);
|
|
184
|
+
const target = new URL(location, pageUrl);
|
|
185
|
+
const internal = target.origin === originUrl.origin;
|
|
186
|
+
const to = internal ? normalizeRoute(target, links) : target.href;
|
|
187
|
+
const redirect = { from: entry.path, to, status: response.status };
|
|
188
|
+
redirects.push(redirect);
|
|
189
|
+
if (internal)
|
|
190
|
+
discovered(to, entry.path);
|
|
191
|
+
// the stub needs the chain's end, known only once the crawl settles
|
|
192
|
+
const duration = performance.now() - started;
|
|
193
|
+
pendingRedirects.push({ entry, filename, response, duration, redirect });
|
|
194
|
+
return;
|
|
137
195
|
}
|
|
196
|
+
const html = await response.text();
|
|
197
|
+
const duration = performance.now() - started;
|
|
138
198
|
// Emission is policy, rendering is not: an unemitted page has still
|
|
139
199
|
// fully executed (integration capture happened server-side) and its
|
|
140
200
|
// links still feed the crawl — it just leaves no HTML file behind to
|
|
141
201
|
// shadow a live server's SSR of the route.
|
|
142
|
-
const emitted =
|
|
202
|
+
const emitted = shouldEmit(entry);
|
|
143
203
|
if (emitted)
|
|
144
204
|
await writeOutput(outDir, filename, html);
|
|
145
|
-
if (
|
|
205
|
+
if (crawlLinks && (response.headers.get("content-type") ?? "").includes("text/html")) {
|
|
146
206
|
const pageUrl = new URL(entry.path, originUrl);
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
discovered(path, entry.path);
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
const hints = response.headers.get(hintHeader);
|
|
153
|
-
if (hints) {
|
|
154
|
-
for (const hint of hints.split(",")) {
|
|
155
|
-
const path = normalizeLink(hint.trim(), originUrl, originUrl.origin);
|
|
156
|
-
if (path !== undefined)
|
|
157
|
-
discovered(path, entry.path);
|
|
158
|
-
}
|
|
207
|
+
for (const path of extractLinks(html, pageUrl, links))
|
|
208
|
+
discovered(path, entry.path);
|
|
159
209
|
}
|
|
160
210
|
const page = {
|
|
161
211
|
path: entry.path,
|
|
@@ -163,12 +213,51 @@ export async function runPrerender(options) {
|
|
|
163
213
|
filename,
|
|
164
214
|
emitted,
|
|
165
215
|
response,
|
|
216
|
+
duration,
|
|
166
217
|
html
|
|
167
218
|
};
|
|
168
219
|
rendered.push(page);
|
|
169
220
|
if (onRendered)
|
|
170
221
|
await onRendered(page);
|
|
171
222
|
}
|
|
223
|
+
// Redirected paths become pages holding a meta-refresh stub — the old URL
|
|
224
|
+
// keeps working on hosts with no redirect support — pointing straight at
|
|
225
|
+
// the chain's final destination so a visitor hops once, not per record.
|
|
226
|
+
async function finalizeRedirects() {
|
|
227
|
+
const hops = new Map(redirects.map(record => [record.from, record.to]));
|
|
228
|
+
// the chain's end — or, in a cycle (which has none), one hop on
|
|
229
|
+
const destination = (from) => {
|
|
230
|
+
const next = hops.get(from);
|
|
231
|
+
const visited = new Set([from]);
|
|
232
|
+
let at = next;
|
|
233
|
+
while (hops.has(at)) {
|
|
234
|
+
if (visited.has(at))
|
|
235
|
+
return next;
|
|
236
|
+
visited.add(at);
|
|
237
|
+
at = hops.get(at);
|
|
238
|
+
}
|
|
239
|
+
return at;
|
|
240
|
+
};
|
|
241
|
+
for (const { entry, filename, response, duration, redirect } of pendingRedirects) {
|
|
242
|
+
const html = redirectStub(destination(entry.path));
|
|
243
|
+
const emitted = redirectStubs && shouldEmit(entry);
|
|
244
|
+
if (emitted)
|
|
245
|
+
await writeOutput(outDir, filename, html);
|
|
246
|
+
const page = {
|
|
247
|
+
path: entry.path,
|
|
248
|
+
referrers: referrersOf(entry.path),
|
|
249
|
+
filename,
|
|
250
|
+
emitted,
|
|
251
|
+
response,
|
|
252
|
+
duration,
|
|
253
|
+
html,
|
|
254
|
+
redirect
|
|
255
|
+
};
|
|
256
|
+
rendered.push(page);
|
|
257
|
+
if (onRendered)
|
|
258
|
+
await onRendered(page);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
172
261
|
try {
|
|
173
262
|
for (const integration of integrations)
|
|
174
263
|
await integration.setup?.(context);
|
|
@@ -199,6 +288,7 @@ export async function runPrerender(options) {
|
|
|
199
288
|
});
|
|
200
289
|
for (const miss of skipped)
|
|
201
290
|
miss.referrers = referrersOf(miss.path);
|
|
291
|
+
await finalizeRedirects();
|
|
202
292
|
for (const integration of integrations)
|
|
203
293
|
await integration.teardown?.(context);
|
|
204
294
|
for (const file of emitted) {
|
|
@@ -208,15 +298,15 @@ export async function runPrerender(options) {
|
|
|
208
298
|
finally {
|
|
209
299
|
await transport.close?.();
|
|
210
300
|
}
|
|
211
|
-
return { pages: rendered, files: emitted, skipped };
|
|
301
|
+
return { pages: rendered, redirects, files: emitted, skipped };
|
|
212
302
|
}
|
|
213
303
|
const describe = (error) => (error instanceof Error ? error.message : String(error));
|
|
214
304
|
function redirectStub(location) {
|
|
215
|
-
const target =
|
|
305
|
+
const target = location.replace(/&/g, "&").replace(/"/g, """);
|
|
216
306
|
return `<!DOCTYPE html><html><head><meta http-equiv="refresh" content="0; url=${target}"><link rel="canonical" href="${target}"></head></html>`;
|
|
217
307
|
}
|
|
218
308
|
async function writeOutput(outDir, filename, contents) {
|
|
219
|
-
const target =
|
|
309
|
+
const target = resolve(outDir, filename);
|
|
220
310
|
await mkdir(dirname(target), { recursive: true });
|
|
221
311
|
await writeFile(target, contents);
|
|
222
312
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
export { runPrerender } from "./crawl.ts";
|
|
2
2
|
export type { RunOptions } from "./crawl.ts";
|
|
3
|
-
export { extractLinks, normalizeLink, normalizePath } from "./links.ts";
|
|
3
|
+
export { extractLinks, normalizeLink, normalizePath, normalizeRoute, splitRoute } from "./links.ts";
|
|
4
|
+
export type { LinkOptions } from "./links.ts";
|
|
4
5
|
export { outputFilename } from "./output.ts";
|
|
5
|
-
export
|
|
6
|
+
export { formatRedirectsFile, redirects } from "./redirects.ts";
|
|
7
|
+
export type { RedirectsIntegrationOptions } from "./redirects.ts";
|
|
8
|
+
export { report } from "./report.ts";
|
|
9
|
+
export { HINT_HEADER, announcePages } from "./announce.ts";
|
|
10
|
+
export type { AnnounceOptions } from "./announce.ts";
|
|
11
|
+
export type { PrerenderReport, ReportIntegrationOptions, ReportPage, ReportSkip } from "./report.ts";
|
|
12
|
+
export { formatSitemap, indexable, sitemap } from "./sitemap.ts";
|
|
13
|
+
export type { SitemapChangeFrequency, SitemapEntry, SitemapIntegrationOptions } from "./sitemap.ts";
|
|
14
|
+
export { httpTransport, loadHandler, moduleTransport } from "./transports.ts";
|
|
15
|
+
export type { HttpTransportOptions, RequestHandler } from "./transports.ts";
|
|
16
|
+
export type { EmittedFile, PageEntry, PagesSource, PrerenderContext, PrerenderIntegration, PrerenderMode, PrerenderOptions, PrerenderResult, RedirectRecord, RenderedPage, SkippedPage, Transport } from "./types.ts";
|
package/dist/index.js
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
1
|
export { runPrerender } from "./crawl.js";
|
|
2
|
-
export { extractLinks, normalizeLink, normalizePath } from "./links.js";
|
|
2
|
+
export { extractLinks, normalizeLink, normalizePath, normalizeRoute, splitRoute } from "./links.js";
|
|
3
3
|
export { outputFilename } from "./output.js";
|
|
4
|
+
export { formatRedirectsFile, redirects } from "./redirects.js";
|
|
5
|
+
export { report } from "./report.js";
|
|
6
|
+
export { HINT_HEADER, announcePages } from "./announce.js";
|
|
7
|
+
export { formatSitemap, indexable, sitemap } from "./sitemap.js";
|
|
8
|
+
export { httpTransport, loadHandler, moduleTransport } from "./transports.js";
|
package/dist/links.d.ts
CHANGED
|
@@ -2,23 +2,39 @@
|
|
|
2
2
|
* Link discovery: which hrefs in a rendered page name more pages of this
|
|
3
3
|
* site. The rules here are correctness fixes other prerenderers earned one
|
|
4
4
|
* bug report at a time — resolve relative hrefs against the PAGE's URL
|
|
5
|
-
* (not the origin), honor <base href>, strip queries
|
|
6
|
-
* dedupe, and never leave the origin.
|
|
5
|
+
* (not the origin), honor <base href>, strip queries (unless asked to keep
|
|
6
|
+
* them) and fragments before dedupe, and never leave the origin.
|
|
7
7
|
*/
|
|
8
|
+
export interface LinkOptions {
|
|
9
|
+
/** Keep the query string as part of the page's identity. @default false */
|
|
10
|
+
keepQuery?: boolean;
|
|
11
|
+
}
|
|
8
12
|
/**
|
|
9
13
|
* Extracts the crawlable same-origin route paths from a page's HTML.
|
|
10
|
-
* Returned paths are normalized (`/about`, no
|
|
11
|
-
*
|
|
14
|
+
* Returned paths are normalized (`/about`, no fragment, no trailing slash
|
|
15
|
+
* except the root, query only with `keepQuery`) and deduped.
|
|
12
16
|
*/
|
|
13
|
-
export declare function extractLinks(html: string, pageUrl: URL): string[];
|
|
17
|
+
export declare function extractLinks(html: string, pageUrl: URL, options?: LinkOptions): string[];
|
|
14
18
|
/**
|
|
15
19
|
* Resolves one href to a normalized same-origin path, or undefined when it
|
|
16
20
|
* is not a page of this site (foreign origin, unparseable).
|
|
17
21
|
*/
|
|
18
|
-
export declare function normalizeLink(href: string, base: URL, origin: string): string | undefined;
|
|
22
|
+
export declare function normalizeLink(href: string, base: URL, origin: string, options?: LinkOptions): string | undefined;
|
|
19
23
|
/**
|
|
20
|
-
* One spelling per page:
|
|
21
|
-
*
|
|
22
|
-
*
|
|
24
|
+
* One spelling per page: the fragment never reaches here (URL parsing split
|
|
25
|
+
* it off), the trailing slash is dropped (except the root), percent-encoding
|
|
26
|
+
* is left exactly as the URL parser produced it, and the query is kept only
|
|
27
|
+
* on request — with its parameters sorted, so `?a=1&b=2` and `?b=2&a=1`
|
|
28
|
+
* are the one page they are.
|
|
29
|
+
*/
|
|
30
|
+
export declare function normalizeRoute(url: URL, options?: LinkOptions): string;
|
|
31
|
+
/**
|
|
32
|
+
* The pathname half of normalization: the trailing slash is dropped (except
|
|
33
|
+
* the root) and nothing else is touched.
|
|
23
34
|
*/
|
|
24
35
|
export declare function normalizePath(pathname: string): string;
|
|
36
|
+
/** Splits a normalized route into its pathname and its (possibly empty) query. */
|
|
37
|
+
export declare function splitRoute(route: string): {
|
|
38
|
+
pathname: string;
|
|
39
|
+
search: string;
|
|
40
|
+
};
|
package/dist/links.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* Link discovery: which hrefs in a rendered page name more pages of this
|
|
3
3
|
* site. The rules here are correctness fixes other prerenderers earned one
|
|
4
4
|
* bug report at a time — resolve relative hrefs against the PAGE's URL
|
|
5
|
-
* (not the origin), honor <base href>, strip queries
|
|
6
|
-
* dedupe, and never leave the origin.
|
|
5
|
+
* (not the origin), honor <base href>, strip queries (unless asked to keep
|
|
6
|
+
* them) and fragments before dedupe, and never leave the origin.
|
|
7
7
|
*/
|
|
8
8
|
const LINK_PATTERN = /<a\s[^>]*?href\s*=\s*(?:"([^"]*)"|'([^']*)')/gis;
|
|
9
9
|
const BASE_PATTERN = /<base\s[^>]*?href\s*=\s*(?:"([^"]*)"|'([^']*)')/is;
|
|
@@ -11,10 +11,10 @@ const BASE_PATTERN = /<base\s[^>]*?href\s*=\s*(?:"([^"]*)"|'([^']*)')/is;
|
|
|
11
11
|
const NON_PAGE_SCHEME = /^(?:mailto|tel|javascript|data|blob|about):/i;
|
|
12
12
|
/**
|
|
13
13
|
* Extracts the crawlable same-origin route paths from a page's HTML.
|
|
14
|
-
* Returned paths are normalized (`/about`, no
|
|
15
|
-
*
|
|
14
|
+
* Returned paths are normalized (`/about`, no fragment, no trailing slash
|
|
15
|
+
* except the root, query only with `keepQuery`) and deduped.
|
|
16
16
|
*/
|
|
17
|
-
export function extractLinks(html, pageUrl) {
|
|
17
|
+
export function extractLinks(html, pageUrl, options = {}) {
|
|
18
18
|
// <base href> shifts what relative hrefs resolve against, exactly as the
|
|
19
19
|
// browser would resolve them.
|
|
20
20
|
const baseMatch = BASE_PATTERN.exec(html);
|
|
@@ -33,7 +33,7 @@ export function extractLinks(html, pageUrl) {
|
|
|
33
33
|
const href = (match[1] ?? match[2] ?? "").trim();
|
|
34
34
|
if (!href || NON_PAGE_SCHEME.test(href))
|
|
35
35
|
continue;
|
|
36
|
-
const path = normalizeLink(href, base, pageUrl.origin);
|
|
36
|
+
const path = normalizeLink(href, base, pageUrl.origin, options);
|
|
37
37
|
if (path !== undefined)
|
|
38
38
|
found.add(path);
|
|
39
39
|
}
|
|
@@ -43,7 +43,7 @@ export function extractLinks(html, pageUrl) {
|
|
|
43
43
|
* Resolves one href to a normalized same-origin path, or undefined when it
|
|
44
44
|
* is not a page of this site (foreign origin, unparseable).
|
|
45
45
|
*/
|
|
46
|
-
export function normalizeLink(href, base, origin) {
|
|
46
|
+
export function normalizeLink(href, base, origin, options = {}) {
|
|
47
47
|
let url;
|
|
48
48
|
try {
|
|
49
49
|
url = new URL(href, base);
|
|
@@ -53,15 +53,37 @@ export function normalizeLink(href, base, origin) {
|
|
|
53
53
|
}
|
|
54
54
|
if (url.origin !== origin)
|
|
55
55
|
return undefined;
|
|
56
|
-
return
|
|
56
|
+
return normalizeRoute(url, options);
|
|
57
57
|
}
|
|
58
58
|
/**
|
|
59
|
-
* One spelling per page:
|
|
60
|
-
*
|
|
61
|
-
*
|
|
59
|
+
* One spelling per page: the fragment never reaches here (URL parsing split
|
|
60
|
+
* it off), the trailing slash is dropped (except the root), percent-encoding
|
|
61
|
+
* is left exactly as the URL parser produced it, and the query is kept only
|
|
62
|
+
* on request — with its parameters sorted, so `?a=1&b=2` and `?b=2&a=1`
|
|
63
|
+
* are the one page they are.
|
|
64
|
+
*/
|
|
65
|
+
export function normalizeRoute(url, options = {}) {
|
|
66
|
+
const path = normalizePath(url.pathname);
|
|
67
|
+
if (!options.keepQuery || !url.search)
|
|
68
|
+
return path;
|
|
69
|
+
const params = new URLSearchParams(url.search);
|
|
70
|
+
params.sort();
|
|
71
|
+
const query = params.toString();
|
|
72
|
+
return query ? `${path}?${query}` : path;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* The pathname half of normalization: the trailing slash is dropped (except
|
|
76
|
+
* the root) and nothing else is touched.
|
|
62
77
|
*/
|
|
63
78
|
export function normalizePath(pathname) {
|
|
64
79
|
if (pathname === "" || pathname === "/")
|
|
65
80
|
return "/";
|
|
66
81
|
return pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
|
|
67
82
|
}
|
|
83
|
+
/** Splits a normalized route into its pathname and its (possibly empty) query. */
|
|
84
|
+
export function splitRoute(route) {
|
|
85
|
+
const at = route.indexOf("?");
|
|
86
|
+
return at === -1
|
|
87
|
+
? { pathname: route, search: "" }
|
|
88
|
+
: { pathname: route.slice(0, at), search: route.slice(at) };
|
|
89
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { PrerenderIntegration, RedirectRecord } from "./types.ts";
|
|
2
|
+
export interface RedirectsIntegrationOptions {
|
|
3
|
+
/** Output file, relative to the output directory. @default "_redirects" */
|
|
4
|
+
filename?: string;
|
|
5
|
+
/**
|
|
6
|
+
* Produces the file's contents from the run's redirects — for hosts with
|
|
7
|
+
* their own format. Defaults to the `_redirects` line format.
|
|
8
|
+
*/
|
|
9
|
+
format?(redirects: readonly RedirectRecord[]): string;
|
|
10
|
+
/**
|
|
11
|
+
* Netlify only: append `!` to force each rule past an existing file at
|
|
12
|
+
* its path. Unneeded when the engine writes no stubs (the default with
|
|
13
|
+
* this integration active), and Cloudflare Pages rejects the syntax.
|
|
14
|
+
* @default false
|
|
15
|
+
*/
|
|
16
|
+
force?: boolean;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Emits the crawl's redirects as host rules — by default a `_redirects` file
|
|
20
|
+
* (Netlify, Cloudflare Pages). Declares `handlesRedirects`, so the engine
|
|
21
|
+
* writes no meta-refresh stubs at redirected paths.
|
|
22
|
+
*
|
|
23
|
+
* ```ts
|
|
24
|
+
* import { redirects } from "prerender-crawler";
|
|
25
|
+
* prerender({ integrations: [redirects()] })
|
|
26
|
+
* ```
|
|
27
|
+
*/
|
|
28
|
+
export declare function redirects(options?: RedirectsIntegrationOptions): PrerenderIntegration;
|
|
29
|
+
/**
|
|
30
|
+
* The `_redirects` line format: `/from /to status`, one rule per line,
|
|
31
|
+
* sorted by source for a stable file across builds.
|
|
32
|
+
*/
|
|
33
|
+
export declare function formatRedirectsFile(redirects: readonly RedirectRecord[], force?: boolean): string;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Emits the crawl's redirects as host rules — by default a `_redirects` file
|
|
3
|
+
* (Netlify, Cloudflare Pages). Declares `handlesRedirects`, so the engine
|
|
4
|
+
* writes no meta-refresh stubs at redirected paths.
|
|
5
|
+
*
|
|
6
|
+
* ```ts
|
|
7
|
+
* import { redirects } from "prerender-crawler";
|
|
8
|
+
* prerender({ integrations: [redirects()] })
|
|
9
|
+
* ```
|
|
10
|
+
*/
|
|
11
|
+
export function redirects(options = {}) {
|
|
12
|
+
const { filename = "_redirects", force = false } = options;
|
|
13
|
+
const format = options.format ?? (records => formatRedirectsFile(records, force));
|
|
14
|
+
return {
|
|
15
|
+
name: "redirects",
|
|
16
|
+
handlesRedirects: true,
|
|
17
|
+
teardown(context) {
|
|
18
|
+
if (context.redirects.length === 0)
|
|
19
|
+
return;
|
|
20
|
+
context.emitFile({ filename, contents: format(context.redirects) });
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* The `_redirects` line format: `/from /to status`, one rule per line,
|
|
26
|
+
* sorted by source for a stable file across builds.
|
|
27
|
+
*/
|
|
28
|
+
export function formatRedirectsFile(redirects, force = false) {
|
|
29
|
+
const lines = [...redirects]
|
|
30
|
+
.sort((a, b) => (a.from < b.from ? -1 : a.from > b.from ? 1 : 0))
|
|
31
|
+
.map(({ from, to, status }) => `${from} ${to} ${status}${force ? "!" : ""}`);
|
|
32
|
+
return lines.join("\n") + "\n";
|
|
33
|
+
}
|
package/dist/report.d.ts
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { PrerenderIntegration, RedirectRecord } from "./types.ts";
|
|
2
|
+
/**
|
|
3
|
+
* The report integration: what the crawl did, as JSON — every page with
|
|
4
|
+
* its status, timing, output file and the pages that linked to it; every
|
|
5
|
+
* redirect; every skipped page and why; every file other integrations
|
|
6
|
+
* emitted. The answers to "why was this page crawled", "which page is
|
|
7
|
+
* slow" and "what did the build actually produce" are all in here, and
|
|
8
|
+
* they are otherwise gone the moment the process exits.
|
|
9
|
+
*/
|
|
10
|
+
export interface ReportIntegrationOptions {
|
|
11
|
+
/**
|
|
12
|
+
* Where to write, resolved against the output directory. The default
|
|
13
|
+
* lands inside it and so deploys with the site; point it outside
|
|
14
|
+
* (`"../prerender-report.json"`) to keep it a build artifact.
|
|
15
|
+
* @default "prerender-report.json"
|
|
16
|
+
*/
|
|
17
|
+
filename?: string;
|
|
18
|
+
}
|
|
19
|
+
export interface PrerenderReport {
|
|
20
|
+
generatedAt: string;
|
|
21
|
+
mode: string;
|
|
22
|
+
origin: string;
|
|
23
|
+
totals: {
|
|
24
|
+
pages: number;
|
|
25
|
+
written: number;
|
|
26
|
+
redirects: number;
|
|
27
|
+
skipped: number;
|
|
28
|
+
files: number;
|
|
29
|
+
/** Sum of page durations, milliseconds — render cost, not wall time. */
|
|
30
|
+
duration: number;
|
|
31
|
+
};
|
|
32
|
+
pages: ReportPage[];
|
|
33
|
+
redirects: RedirectRecord[];
|
|
34
|
+
skipped: ReportSkip[];
|
|
35
|
+
/** Files emitted by integrations that ran before this one, this run's pages excluded. */
|
|
36
|
+
files: string[];
|
|
37
|
+
}
|
|
38
|
+
export interface ReportPage {
|
|
39
|
+
path: string;
|
|
40
|
+
status: number;
|
|
41
|
+
contentType: string | null;
|
|
42
|
+
/** Milliseconds, the successful attempt. */
|
|
43
|
+
duration: number;
|
|
44
|
+
filename: string;
|
|
45
|
+
/** Whether the HTML was written (`false` in hybrid mode, for query spellings, ...). */
|
|
46
|
+
written: boolean;
|
|
47
|
+
referrers: string[];
|
|
48
|
+
redirect?: RedirectRecord;
|
|
49
|
+
}
|
|
50
|
+
export interface ReportSkip {
|
|
51
|
+
path: string;
|
|
52
|
+
error: string;
|
|
53
|
+
referrers: string[];
|
|
54
|
+
}
|
|
55
|
+
export declare function report(options?: ReportIntegrationOptions): PrerenderIntegration;
|
package/dist/report.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
export function report(options = {}) {
|
|
2
|
+
const { filename = "prerender-report.json" } = options;
|
|
3
|
+
return {
|
|
4
|
+
name: "report",
|
|
5
|
+
teardown(context) {
|
|
6
|
+
const pages = context.pages
|
|
7
|
+
.map(page => ({
|
|
8
|
+
path: page.path,
|
|
9
|
+
status: page.response.status,
|
|
10
|
+
contentType: page.response.headers.get("content-type"),
|
|
11
|
+
duration: Math.round(page.duration * 100) / 100,
|
|
12
|
+
filename: page.filename,
|
|
13
|
+
written: page.emitted,
|
|
14
|
+
referrers: [...page.referrers].sort(),
|
|
15
|
+
...(page.redirect ? { redirect: page.redirect } : {})
|
|
16
|
+
}))
|
|
17
|
+
.sort(byPath);
|
|
18
|
+
const skipped = context.skipped
|
|
19
|
+
.map(miss => ({
|
|
20
|
+
path: miss.path,
|
|
21
|
+
error: miss.error instanceof Error ? miss.error.message : String(miss.error),
|
|
22
|
+
referrers: [...miss.referrers].sort()
|
|
23
|
+
}))
|
|
24
|
+
.sort(byPath);
|
|
25
|
+
const summary = {
|
|
26
|
+
generatedAt: new Date().toISOString(),
|
|
27
|
+
mode: context.mode,
|
|
28
|
+
origin: context.origin,
|
|
29
|
+
totals: {
|
|
30
|
+
pages: pages.length,
|
|
31
|
+
written: pages.filter(page => page.written).length,
|
|
32
|
+
redirects: context.redirects.length,
|
|
33
|
+
skipped: skipped.length,
|
|
34
|
+
files: context.files.length,
|
|
35
|
+
duration: Math.round(pages.reduce((sum, page) => sum + page.duration, 0))
|
|
36
|
+
},
|
|
37
|
+
pages,
|
|
38
|
+
redirects: [...context.redirects],
|
|
39
|
+
skipped,
|
|
40
|
+
files: context.files.map(file => file.filename)
|
|
41
|
+
};
|
|
42
|
+
context.emitFile({ filename, contents: JSON.stringify(summary, null, 2) + "\n" });
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
const byPath = (a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0;
|