polite-crawler 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +293 -2
  3. package/build/browser-gJ_h4AhB.mjs +511 -0
  4. package/build/browser-gJ_h4AhB.mjs.map +1 -0
  5. package/build/classify-CrfyXcwo.d.mts +51 -0
  6. package/build/classify.d.mts +2 -0
  7. package/build/classify.mjs +116 -0
  8. package/build/classify.mjs.map +1 -0
  9. package/build/cli.mjs +348 -0
  10. package/build/cli.mjs.map +1 -0
  11. package/build/hours.d.mts +113 -0
  12. package/build/hours.mjs +905 -0
  13. package/build/hours.mjs.map +1 -0
  14. package/build/html-nWYFVsRU.d.mts +98 -0
  15. package/build/html-oNpGAHwi.mjs +1038 -0
  16. package/build/html-oNpGAHwi.mjs.map +1 -0
  17. package/build/html.d.mts +2 -0
  18. package/build/html.mjs +1 -0
  19. package/build/index.d.mts +131 -0
  20. package/build/index.mjs +8 -0
  21. package/build/mcp.d.mts +40 -0
  22. package/build/mcp.mjs +480 -0
  23. package/build/mcp.mjs.map +1 -0
  24. package/build/open-CJR4b44v.mjs +4888 -0
  25. package/build/open-CJR4b44v.mjs.map +1 -0
  26. package/build/open-CZqwY7GH.d.mts +1461 -0
  27. package/build/politeness-C5bZ1OKM.d.mts +274 -0
  28. package/build/politeness.d.mts +2 -0
  29. package/build/politeness.mjs +372 -0
  30. package/build/politeness.mjs.map +1 -0
  31. package/build/robots-R-qaEBOb.d.mts +144 -0
  32. package/build/robots.d.mts +2 -0
  33. package/build/robots.mjs +369 -0
  34. package/build/robots.mjs.map +1 -0
  35. package/build/signals-BHpy2xb2.d.mts +46 -0
  36. package/build/signals.d.mts +2 -0
  37. package/build/signals.mjs +113 -0
  38. package/build/signals.mjs.map +1 -0
  39. package/build/version-rVx8_y1_.mjs +11 -0
  40. package/build/version-rVx8_y1_.mjs.map +1 -0
  41. package/package.json +131 -3
  42. package/record.schema.json +198 -0
@@ -0,0 +1,4888 @@
1
+ import { a as RenderRefused, o as playwrightRenderer, r as checkedRenderer, s as __exportAll } from "./browser-gJ_h4AhB.mjs";
2
+ import { classify, isClientError, parseHttpDate } from "./classify.mjs";
3
+ import { DEFAULT_POLICY, HostScheduler, HostStopped, MAX_STOP_MS, MemoryHostStateStore, checkPolicy, equalJitterMs, policyFor, realClock, restAfter } from "./politeness.mjs";
4
+ import { decide, isProductToken, robotsFromResponse, robotsFromStatus, signalsFor, unreachableRobots } from "./robots.mjs";
5
+ import { NO_SIGNALS, metaSignals, readSignals } from "./signals.mjs";
6
+ import { n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-rVx8_y1_.mjs";
7
+ import { c as canonicalUrl, o as decodeBody, r as typesOf, s as cacheKeyOf, t as extractPage } from "./html-oNpGAHwi.mjs";
8
+ import { createHash, randomUUID } from "node:crypto";
9
+ import { mkdir, open, readFile, readdir, rename, rm, stat } from "node:fs/promises";
10
+ import { dirname, join } from "node:path";
11
+ import { createReadStream, existsSync, lstatSync, mkdirSync, realpathSync, statSync } from "node:fs";
12
+ import { Transform } from "node:stream";
13
+ import { constants, createBrotliDecompress, createGunzip, createInflate, createInflateRaw, inflateSync } from "node:zlib";
14
+ import { Agent, request } from "undici";
15
+ import { lookup } from "node:dns";
16
+ import { isIP } from "node:net";
17
+ import { DatabaseSync } from "node:sqlite";
18
+ import { getDocumentProxy, getMeta } from "unpdf";
19
+ //#region src/blobs.ts
20
+ const HASH = /^[0-9a-f]{64}$/;
21
+ /** A rename can fail with these codes when a different writer puts the same blob at the same time. */
22
+ const RACE_CODES = /* @__PURE__ */ new Set([
23
+ "EPERM",
24
+ "EEXIST",
25
+ "EBUSY",
26
+ "EACCES",
27
+ "ENOTEMPTY"
28
+ ]);
29
+ const blobPath = (root, hash) => join(root, hash.slice(0, 2), hash.slice(2, 4), hash);
30
+ const hashOf = (bytes) => createHash("sha256").update(bytes).digest("hex");
31
+ const codeOf = (error) => typeof error === "object" && error !== null && "code" in error ? String(error.code) : "";
32
+ const statOf = (path) => stat(path).catch(() => void 0);
33
+ /** This function writes a folder to the disk, so a rename in it stays after a power cut. Windows cannot open a folder, so it skips this step. */
34
+ const syncFolder = async (path) => {
35
+ if (process.platform === "win32") return;
36
+ try {
37
+ const handle = await open(path, "r");
38
+ try {
39
+ await handle.sync();
40
+ } finally {
41
+ await handle.close();
42
+ }
43
+ } catch {}
44
+ };
45
+ /**
46
+ * This store keeps each blob in a file: `<dir>/ab/cd/abcd…`, where the name is the SHA-256 hash
47
+ * of the bytes. These are its rules:
48
+ *
49
+ * - Equal bytes are written one time only. A file of the wrong size (a write that a crash cut) is
50
+ * written again.
51
+ * - A write goes to a temporary file with a random name, goes to the disk, and then gets its name.
52
+ * A crash thus cannot leave half a blob under its name.
53
+ * - Two writers can put the same bytes at the same time. When the rename of one fails because the
54
+ * other was first, the put succeeds if the stored file has the same hash.
55
+ * - The method `get` examines the hash of the bytes. A damaged file gets a new name
56
+ * (`.damaged`), and `get` gives no bytes, so the crawler fetches the body again.
57
+ */
58
+ var FileBlobStore = class FileBlobStore {
59
+ dir;
60
+ writing = /* @__PURE__ */ new Map();
61
+ sync;
62
+ constructor(dir, options = {}) {
63
+ this.dir = dir;
64
+ this.sync = options.sync ?? true;
65
+ }
66
+ put(bytes) {
67
+ const hash = hashOf(bytes);
68
+ const running = this.writing.get(hash);
69
+ if (running !== void 0) return running;
70
+ const work = this.write(hash, bytes).finally(() => this.writing.delete(hash));
71
+ this.writing.set(hash, work);
72
+ return work;
73
+ }
74
+ /** This method tells if a file holds exactly the bytes of a hash. */
75
+ static async holds(path, hash) {
76
+ try {
77
+ return hashOf(await readFile(path)) === hash;
78
+ } catch {
79
+ return false;
80
+ }
81
+ }
82
+ async write(hash, bytes) {
83
+ const target = blobPath(this.dir, hash);
84
+ const existing = await statOf(target);
85
+ if (existing?.isFile() === true && existing.size === bytes.length) return {
86
+ hash,
87
+ existed: true
88
+ };
89
+ const folder = dirname(target);
90
+ await mkdir(folder, {
91
+ recursive: true,
92
+ mode: 448
93
+ });
94
+ const temporary = `${target}.${randomUUID()}.tmp`;
95
+ try {
96
+ const handle = await open(temporary, "wx", 384);
97
+ try {
98
+ await handle.writeFile(bytes);
99
+ if (this.sync) await handle.sync();
100
+ } finally {
101
+ await handle.close();
102
+ }
103
+ try {
104
+ await rename(temporary, target);
105
+ } catch (error) {
106
+ if (!RACE_CODES.has(codeOf(error)) || !await FileBlobStore.holds(target, hash)) throw error;
107
+ return {
108
+ hash,
109
+ existed: true
110
+ };
111
+ }
112
+ if (this.sync) await syncFolder(folder);
113
+ return {
114
+ hash,
115
+ existed: false
116
+ };
117
+ } finally {
118
+ await rm(temporary, { force: true }).catch(() => void 0);
119
+ }
120
+ }
121
+ async get(hash) {
122
+ if (!HASH.test(hash)) return void 0;
123
+ const target = blobPath(this.dir, hash);
124
+ let bytes;
125
+ try {
126
+ bytes = await readFile(target);
127
+ } catch {
128
+ return;
129
+ }
130
+ if (hashOf(bytes) !== hash) {
131
+ await rename(target, `${target}.${randomUUID()}.damaged`).catch(() => void 0);
132
+ return;
133
+ }
134
+ return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
135
+ }
136
+ };
137
+ //#endregion
138
+ //#region src/egress.ts
139
+ /** The connection hook gives this error when the policy refuses each address of a host. */
140
+ var EgressRefused = class extends Error {
141
+ address;
142
+ kind;
143
+ code = "EGRESS_REFUSED";
144
+ constructor(address, kind) {
145
+ super(egressNote(address, kind));
146
+ this.address = address;
147
+ this.kind = kind;
148
+ this.name = "EgressRefused";
149
+ }
150
+ };
151
+ /** This function gives the note of a refusal, with the address and the kind of address. */
152
+ const egressNote = (address, kind) => `egress policy: ${address} is ${/^[aeiou]/i.test(kind) ? "an" : "a"} ${kind} address; set allowPrivate to fetch it`;
153
+ /**
154
+ * This function reads an IPv4 address the way the URL parser reads a host. The address has one to
155
+ * four parts, and each part is decimal, octal (`0177`) or hexadecimal (`0x7f`). Thus
156
+ * `2130706433` and `0x7f.1` both give 127.0.0.1. A text that is not an IPv4 address gives no value.
157
+ */
158
+ const parseIPv4 = (text) => {
159
+ const parts = text.split(".");
160
+ if (parts.length > 1 && parts[parts.length - 1] === "") parts.pop();
161
+ if (parts.length === 0 || parts.length > 4) return void 0;
162
+ const numbers = [];
163
+ for (const part of parts) {
164
+ let value;
165
+ if (/^0[xX][0-9a-fA-F]*$/.test(part)) value = part.length === 2 ? 0 : Number.parseInt(part.slice(2), 16);
166
+ else if (/^0[0-7]+$/.test(part)) value = Number.parseInt(part.slice(1), 8);
167
+ else if (/^(0|[1-9][0-9]*)$/.test(part)) value = Number(part);
168
+ else return void 0;
169
+ numbers.push(value);
170
+ }
171
+ const last = numbers.pop();
172
+ if (numbers.some((value) => value > 255)) return void 0;
173
+ if (last >= 256 ** (4 - numbers.length)) return void 0;
174
+ let rest = last;
175
+ const tail = [];
176
+ for (let index = numbers.length; index < 4; index += 1) {
177
+ tail.unshift(rest % 256);
178
+ rest = Math.floor(rest / 256);
179
+ }
180
+ return [...numbers, ...tail];
181
+ };
182
+ /** This function reads an IPv6 address, with or without brackets and a zone. It gives 16 bytes. */
183
+ const parseIPv6 = (text) => {
184
+ let value = text.startsWith("[") && text.endsWith("]") ? text.slice(1, -1) : text;
185
+ const zone = value.indexOf("%");
186
+ if (zone !== -1) value = value.slice(0, zone);
187
+ if (isIP(value) !== 6) return void 0;
188
+ const lastColon = value.lastIndexOf(":");
189
+ const tail = value.slice(lastColon + 1);
190
+ if (tail.includes(".")) {
191
+ const v4 = parseIPv4(tail);
192
+ if (v4 === void 0) return void 0;
193
+ const high = v4[0] << 8 | v4[1];
194
+ const low = v4[2] << 8 | v4[3];
195
+ value = `${value.slice(0, lastColon + 1)}${high.toString(16)}:${low.toString(16)}`;
196
+ }
197
+ const [head = "", rest] = value.split("::");
198
+ const groupsOf = (part) => part.length === 0 ? [] : part.split(":").map((group) => Number.parseInt(group, 16));
199
+ const front = groupsOf(head);
200
+ const back = rest === void 0 ? [] : groupsOf(rest);
201
+ const groups = [
202
+ ...front,
203
+ ...Array(8 - front.length - back.length).fill(0),
204
+ ...back
205
+ ];
206
+ if (groups.length !== 8) return void 0;
207
+ return groups.flatMap((group) => [group >> 8, group & 255]);
208
+ };
209
+ const v4 = (text) => parseIPv4(text);
210
+ const v6 = (text) => parseIPv6(text);
211
+ /** This function tells if the first `bits` bits of two addresses agree. */
212
+ const inPrefix = (address, prefix, bits) => {
213
+ if (address.length !== prefix.length) return false;
214
+ for (let index = 0; index < address.length; index += 1) {
215
+ const take = Math.max(0, Math.min(8, bits - index * 8));
216
+ if (take === 0) return true;
217
+ const mask = 255 << 8 - take & 255;
218
+ if ((address[index] & mask) !== (prefix[index] & mask)) return false;
219
+ }
220
+ return true;
221
+ };
222
+ const V4_RANGES = [
223
+ [
224
+ v4("169.254.169.254"),
225
+ 32,
226
+ "metadata"
227
+ ],
228
+ [
229
+ v4("168.63.129.16"),
230
+ 32,
231
+ "metadata"
232
+ ],
233
+ [
234
+ v4("100.100.100.200"),
235
+ 32,
236
+ "metadata"
237
+ ],
238
+ [
239
+ v4("0.0.0.0"),
240
+ 8,
241
+ "unspecified"
242
+ ],
243
+ [
244
+ v4("10.0.0.0"),
245
+ 8,
246
+ "private"
247
+ ],
248
+ [
249
+ v4("100.64.0.0"),
250
+ 10,
251
+ "carrier-grade NAT"
252
+ ],
253
+ [
254
+ v4("127.0.0.0"),
255
+ 8,
256
+ "loopback"
257
+ ],
258
+ [
259
+ v4("169.254.0.0"),
260
+ 16,
261
+ "link-local"
262
+ ],
263
+ [
264
+ v4("172.16.0.0"),
265
+ 12,
266
+ "private"
267
+ ],
268
+ [
269
+ v4("192.0.0.0"),
270
+ 24,
271
+ "reserved"
272
+ ],
273
+ [
274
+ v4("192.0.2.0"),
275
+ 24,
276
+ "documentation"
277
+ ],
278
+ [
279
+ v4("192.168.0.0"),
280
+ 16,
281
+ "private"
282
+ ],
283
+ [
284
+ v4("198.18.0.0"),
285
+ 15,
286
+ "benchmark"
287
+ ],
288
+ [
289
+ v4("198.51.100.0"),
290
+ 24,
291
+ "documentation"
292
+ ],
293
+ [
294
+ v4("203.0.113.0"),
295
+ 24,
296
+ "documentation"
297
+ ],
298
+ [
299
+ v4("224.0.0.0"),
300
+ 4,
301
+ "multicast"
302
+ ],
303
+ [
304
+ v4("240.0.0.0"),
305
+ 4,
306
+ "reserved"
307
+ ]
308
+ ];
309
+ const V6_RANGES = [
310
+ [
311
+ v6("fd00:ec2::254"),
312
+ 128,
313
+ "metadata"
314
+ ],
315
+ [
316
+ v6("::"),
317
+ 128,
318
+ "unspecified"
319
+ ],
320
+ [
321
+ v6("::1"),
322
+ 128,
323
+ "loopback"
324
+ ],
325
+ [
326
+ v6("64:ff9b:1::"),
327
+ 48,
328
+ "reserved"
329
+ ],
330
+ [
331
+ v6("100::"),
332
+ 64,
333
+ "reserved"
334
+ ],
335
+ [
336
+ v6("2001::"),
337
+ 32,
338
+ "reserved"
339
+ ],
340
+ [
341
+ v6("2001:db8::"),
342
+ 32,
343
+ "documentation"
344
+ ],
345
+ [
346
+ v6("fc00::"),
347
+ 7,
348
+ "unique local"
349
+ ],
350
+ [
351
+ v6("fe80::"),
352
+ 10,
353
+ "link-local"
354
+ ],
355
+ [
356
+ v6("fec0::"),
357
+ 10,
358
+ "private"
359
+ ],
360
+ [
361
+ v6("ff00::"),
362
+ 8,
363
+ "multicast"
364
+ ]
365
+ ];
366
+ const kindOfV4 = (address) => V4_RANGES.find(([prefix, bits]) => inPrefix(address, prefix, bits))?.[2];
367
+ const kindOfV6 = (address) => {
368
+ const zeroes = (from, to) => address.slice(from, to).every((byte) => byte === 0);
369
+ const embedded = (from) => kindOfV4(address.slice(from, from + 4));
370
+ if (zeroes(0, 10) && address[10] === 255 && address[11] === 255) return embedded(12);
371
+ if (inPrefix(address, v6("64:ff9b::"), 96)) return embedded(12);
372
+ if (address[0] === 32 && address[1] === 2) return embedded(2);
373
+ const listed = V6_RANGES.find(([prefix, bits]) => inPrefix(address, prefix, bits))?.[2];
374
+ if (listed !== void 0) return listed;
375
+ if (zeroes(0, 12)) return embedded(12) ?? "reserved";
376
+ if (address[0] === 0) return "reserved";
377
+ };
378
+ /**
379
+ * This function gives the kind of an address that is not public, for example `loopback` or
380
+ * `private`. It reads each IPv4 spelling and each IPv6 form. A public address, or a text that is not
381
+ * an address, gives no value.
382
+ */
383
+ const addressKind = (address) => {
384
+ const four = parseIPv4(address);
385
+ if (four !== void 0) return kindOfV4(four);
386
+ const six = parseIPv6(address);
387
+ return six === void 0 ? void 0 : kindOfV6(six);
388
+ };
389
+ const nameOf = (host) => host.trim().toLowerCase().replace(/\.$/, "");
390
+ /** These names have a fixed meaning. They do not go to DNS. */
391
+ const NAME_KINDS = [
392
+ [/(^|\.)localhost$/, "loopback"],
393
+ [/^metadata\.google\.internal$/, "metadata"],
394
+ [/^metadata\.goog$/, "metadata"]
395
+ ];
396
+ const literalOf = (host) => parseIPv6(host) ?? parseIPv4(host);
397
+ const inRange = (entry, address) => {
398
+ const slash = entry.indexOf("/");
399
+ const base = literalOf(slash === -1 ? entry : entry.slice(0, slash));
400
+ if (base === void 0 || base.length !== address.length) return false;
401
+ const bits = slash === -1 ? base.length * 8 : Number(entry.slice(slash + 1));
402
+ return Number.isInteger(bits) && bits >= 0 && bits <= base.length * 8 && inPrefix(address, base, bits);
403
+ };
404
+ /** This function tells if an entry of `allowHosts` permits this host name or this address. */
405
+ const permitted = (policy, host, address) => {
406
+ if (policy.allowPrivate === true) return true;
407
+ const name = nameOf(host);
408
+ const bytes = address === void 0 ? literalOf(name) : literalOf(address);
409
+ for (const raw of policy.allowHosts ?? []) {
410
+ const entry = nameOf(raw);
411
+ if (entry === name) return true;
412
+ if (entry.startsWith("*.") && name.endsWith(entry.slice(1))) return true;
413
+ if (bytes !== void 0 && inRange(entry, bytes)) return true;
414
+ }
415
+ return false;
416
+ };
417
+ /** This function gives a short form of an address for a note: IPv6 in brackets, IPv4 with dots. */
418
+ const shown = (host) => {
419
+ const bytes = literalOf(host);
420
+ if (bytes === void 0) return host;
421
+ if (bytes.length === 4) return bytes.join(".");
422
+ const groups = [];
423
+ for (let index = 0; index < 16; index += 2) groups.push((bytes[index] << 8 | bytes[index + 1]).toString(16));
424
+ return groups.join(":").replace(/(^|:)0(:0)+(:|$)/, "::");
425
+ };
426
+ /**
427
+ * This function examines a host before DNS: a literal IP address in any spelling, and the names
428
+ * with a fixed meaning (`localhost`). It gives the note of a refusal, or no value. A host name that
429
+ * needs DNS gets its check in the connection hook.
430
+ */
431
+ const egressRefusal = (policy, host) => {
432
+ if (policy.allowPrivate === true) return void 0;
433
+ const name = nameOf(host);
434
+ if (literalOf(name) !== void 0) {
435
+ const kind = addressKind(name);
436
+ return kind === void 0 || permitted(policy, name, void 0) ? void 0 : egressNote(shown(name), kind);
437
+ }
438
+ for (const [pattern, kind] of NAME_KINDS) if (pattern.test(name) && !permitted(policy, name, void 0)) return egressNote(name, kind);
439
+ };
440
+ const resolveAll = (host, options, callback) => lookup(host, options, callback);
441
+ /** This function keeps the addresses that the policy permits. When it keeps none, it gives the refusal. */
442
+ const screen = (policy, host, addresses) => {
443
+ const usable = [];
444
+ let refusal;
445
+ for (const entry of addresses) {
446
+ const kind = addressKind(entry.address);
447
+ if (kind === void 0 || permitted(policy, host, entry.address)) usable.push(entry);
448
+ else refusal ??= new EgressRefused(shown(entry.address), kind);
449
+ }
450
+ return {
451
+ usable,
452
+ refusal: usable.length === 0 ? refusal ?? new EgressRefused(host, "unresolved") : void 0
453
+ };
454
+ };
455
+ /**
456
+ * This function gives a `lookup` hook for the connection options of an undici `Agent`. The hook
457
+ * resolves the name, removes each address that the policy refuses, and gives the rest to the
458
+ * socket. When no address stays, the connection fails with `EgressRefused`.
459
+ */
460
+ const egressLookup = (policy, resolve = resolveAll) => (host, options, callback) => {
461
+ resolve(host, {
462
+ ...options,
463
+ all: true
464
+ }, (error, addresses) => {
465
+ if (error !== null) {
466
+ callback(error, []);
467
+ return;
468
+ }
469
+ const { usable, refusal } = policy.allowPrivate === true ? {
470
+ usable: addresses,
471
+ refusal: void 0
472
+ } : screen(policy, host, addresses);
473
+ if (refusal !== void 0) {
474
+ callback(refusal, []);
475
+ return;
476
+ }
477
+ if (options.all === true) callback(null, usable);
478
+ else callback(null, usable[0].address, usable[0].family);
479
+ });
480
+ };
481
+ /**
482
+ * This function examines a URL with DNS, for a client that does its own DNS, for example the
483
+ * browser. It refuses a host when one of its addresses is not public. A name that DNS cannot
484
+ * resolve gets no refusal, because the client cannot connect to it either. The function gives
485
+ * the note of a refusal, or no value.
486
+ *
487
+ * The client resolves the name again, so a DNS answer that changes between the two lookups can
488
+ * pass. The connection hook (`egressLookup`) does not have this gap.
489
+ */
490
+ const checkEgress = async (policy, url, resolve = resolveAll) => {
491
+ if (policy.allowPrivate === true) return void 0;
492
+ const before = egressRefusal(policy, url.hostname);
493
+ if (before !== void 0) return before;
494
+ const name = nameOf(url.hostname);
495
+ if (literalOf(name) !== void 0 || permitted(policy, name, void 0)) return void 0;
496
+ const addresses = await new Promise((settle) => resolve(name, { all: true }, (error, found) => settle(error ?? found)));
497
+ if (addresses instanceof Error) return void 0;
498
+ for (const entry of addresses) {
499
+ const kind = addressKind(entry.address);
500
+ if (kind !== void 0 && !permitted(policy, name, entry.address)) return egressNote(shown(entry.address), kind);
501
+ }
502
+ };
503
+ /** This function finds an `EgressRefused` in an error, its cause or its group of errors. */
504
+ const egressError = (error) => {
505
+ for (let current = error, depth = 0; current !== void 0 && current !== null && depth < 5; depth += 1) {
506
+ if (current instanceof EgressRefused) return current;
507
+ if (current instanceof AggregateError) {
508
+ const inner = current.errors.find((item) => item instanceof EgressRefused);
509
+ if (inner !== void 0) return inner;
510
+ }
511
+ current = current.cause;
512
+ }
513
+ };
514
+ //#endregion
515
+ //#region src/record.ts
516
+ /**
517
+ * This function writes a value as JSON with the keys of each object in sorted order. Equal values
518
+ * thus always give equal text, and equal text gives an equal hash.
519
+ */
520
+ const stableStringify = (value) => {
521
+ if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
522
+ if (Array.isArray(value)) return `[${value.map((item) => stableStringify(item)).join(",")}]`;
523
+ return `{${Object.entries(value).filter(([, item]) => item !== void 0).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => `${JSON.stringify(key)}:${stableStringify(item)}`).join(",")}}`;
524
+ };
525
+ /** This function gives the SHA-256 hash of a text as hexadecimal digits. */
526
+ const sha256 = (text) => createHash("sha256").update(text, "utf8").digest("hex");
527
+ /**
528
+ * This function makes a capture record. The hash covers the source, the kind, the external id
529
+ * and the payload. The id is the source and the first 12 digits of the hash. The times, `scope`
530
+ * and `labels` are not part of the hash.
531
+ */
532
+ const makeRecord = (input) => {
533
+ const content_hash = sha256(stableStringify({
534
+ source: input.source,
535
+ kind: input.kind,
536
+ external_id: input.external_id ?? null,
537
+ payload: input.payload
538
+ }));
539
+ return {
540
+ id: `${input.source}.${content_hash.slice(0, 12)}`,
541
+ source: input.source,
542
+ kind: input.kind,
543
+ received_at: input.received_at,
544
+ payload: input.payload,
545
+ content_hash,
546
+ ...input.external_id !== void 0 ? { external_id: input.external_id } : {},
547
+ ...input.occurred_at !== void 0 ? { occurred_at: input.occurred_at } : {},
548
+ ...input.scope !== void 0 ? { scope: input.scope } : {},
549
+ ...input.labels !== void 0 && Object.keys(input.labels).length > 0 ? { labels: { ...input.labels } } : {}
550
+ };
551
+ };
552
+ //#endregion
553
+ //#region src/crawler.ts
554
+ /**
555
+ * This function gives the key of the cache entry of a rendered page in the form of version 0.1.
556
+ * The index keeps the variant in a column of its own, and it reads this form too.
557
+ */
558
+ const renderedKey = (url) => `${url}#rendered`;
559
+ /** This value is the version of the payload of the records that the crawler writes. A record with no `schema` is version 1. */
560
+ const CAPTURE_SCHEMA = 2;
561
+ /** The content type of a rendered page: the browser gives its text as UTF-8. */
562
+ const RENDERED_TYPE = "text/html; charset=utf-8";
563
+ /**
564
+ * These query parameters hold secrets in many APIs. The crawler writes `REDACTED` in place of
565
+ * their values in each capture, in addition to the names of `secretParams`.
566
+ */
567
+ const DEFAULT_SECRET_PARAMS = [
568
+ "access_token",
569
+ "api-key",
570
+ "api_key",
571
+ "apikey",
572
+ "auth",
573
+ "auth_token",
574
+ "client_secret",
575
+ "id_token",
576
+ "passwd",
577
+ "password",
578
+ "pwd",
579
+ "refresh_token",
580
+ "secret",
581
+ "session",
582
+ "session_id",
583
+ "sessionid",
584
+ "sig",
585
+ "signature",
586
+ "token",
587
+ "x-amz-credential",
588
+ "x-amz-security-token",
589
+ "x-amz-signature",
590
+ "x-goog-credential",
591
+ "x-goog-signature"
592
+ ];
593
+ const ERROR_BODY_BYTES = 65536;
594
+ const ROBOTS_BYTES = 524288;
595
+ const LAST_DATE_MS = 864e13;
596
+ const WEB_PROTOCOLS = /* @__PURE__ */ new Set(["http:", "https:"]);
597
+ const HEADER_TEXT = /^[\t -~]*$/;
598
+ const PERMANENT_REDIRECTS = /* @__PURE__ */ new Set([301, 308]);
599
+ const HEURISTIC_STATUSES = /* @__PURE__ */ new Set([
600
+ 200,
601
+ 203,
602
+ 204,
603
+ 206,
604
+ 300,
605
+ 301,
606
+ 308,
607
+ 404,
608
+ 405,
609
+ 410,
610
+ 414,
611
+ 501
612
+ ]);
613
+ /** A caller cannot set these headers. The crawler sets them, or HTTP does. */
614
+ const RESERVED_HEADERS = /* @__PURE__ */ new Set([
615
+ "user-agent",
616
+ "from",
617
+ "accept-encoding",
618
+ "if-none-match",
619
+ "if-modified-since",
620
+ "if-match",
621
+ "if-unmodified-since",
622
+ "if-range",
623
+ "range",
624
+ "host",
625
+ "connection",
626
+ "content-length",
627
+ "transfer-encoding",
628
+ "te",
629
+ "upgrade",
630
+ "keep-alive"
631
+ ]);
632
+ /** A redirect to a different origin removes these headers, as the Fetch standard does. */
633
+ const CREDENTIAL_HEADERS = /* @__PURE__ */ new Set([
634
+ "authorization",
635
+ "cookie",
636
+ "proxy-authorization"
637
+ ]);
638
+ /** The crawler sends the same value of these headers in each request, so a `Vary` on them changes nothing. */
639
+ const CONSTANT_HEADERS = /* @__PURE__ */ new Set([
640
+ "accept-encoding",
641
+ "user-agent",
642
+ "from"
643
+ ]);
644
+ /**
645
+ * These content codings change nothing in the body. A server that stored an object with
646
+ * `aws-chunked` can send the name with the decoded body.
647
+ */
648
+ const IDENTITY_CODINGS = /* @__PURE__ */ new Set(["identity", "aws-chunked"]);
649
+ /**
650
+ * This function tells if an answer is a web page that the request did not ask for. The answer is
651
+ * HTML, and the `Accept` of the request admits no HTML (`application/json`). A login or an error
652
+ * page in place of the data of an API is such an answer. A media range with `q=0` admits nothing.
653
+ */
654
+ const unaskedPage = (accept, contentType) => {
655
+ const type = (contentType ?? "").split(";")[0].trim().toLowerCase();
656
+ if (accept === void 0 || accept.trim() === "" || type !== "text/html" && type !== "application/xhtml+xml") return false;
657
+ return !accept.split(",").some((range) => {
658
+ const [media = "", ...params] = range.split(";").map((part) => part.trim().toLowerCase());
659
+ const q = params.find((param) => param.startsWith("q="));
660
+ if (q !== void 0 && Number(q.slice(2)) === 0) return false;
661
+ return media === "*/*" || media === type || media.endsWith("/*") && type.startsWith(media.slice(0, -1));
662
+ });
663
+ };
664
+ /** This function reads the TDMRep `<meta>` tags at the start of an HTML body. */
665
+ const pageSignals = (body, contentType) => body === void 0 || !/html/i.test(contentType ?? "") ? [] : metaSignals(new TextDecoder("latin1").decode(body.subarray(0, 65536)));
666
+ /** A count of slots: `max` holders at one time at most, and the others wait in sequence. */
667
+ var Slots = class {
668
+ max;
669
+ used = 0;
670
+ waiting = [];
671
+ constructor(max) {
672
+ this.max = max;
673
+ }
674
+ /** This method gives a function that frees the slot. It rejects when the signal aborts before a slot is free. */
675
+ async acquire(signal) {
676
+ if (this.used < this.max) {
677
+ this.used += 1;
678
+ return this.releaser();
679
+ }
680
+ await new Promise((resolve, reject) => {
681
+ const stop = () => {
682
+ const at = this.waiting.indexOf(go);
683
+ if (at !== -1) this.waiting.splice(at, 1);
684
+ reject(signal.reason instanceof Error ? signal.reason : /* @__PURE__ */ new Error("cancelled"));
685
+ };
686
+ const go = () => {
687
+ signal.removeEventListener("abort", stop);
688
+ resolve();
689
+ };
690
+ if (signal.aborted) {
691
+ stop();
692
+ return;
693
+ }
694
+ this.waiting.push(go);
695
+ signal.addEventListener("abort", stop, { once: true });
696
+ });
697
+ return this.releaser();
698
+ }
699
+ releaser() {
700
+ let freed = false;
701
+ return () => {
702
+ if (freed) return;
703
+ freed = true;
704
+ const next = this.waiting.shift();
705
+ if (next !== void 0) next();
706
+ else this.used -= 1;
707
+ };
708
+ }
709
+ };
710
+ /** This function tells if an error of a renderer says that it has no browser: no `playwright-core`, or no browser on the disk. */
711
+ const missingBrowser = (message) => message.startsWith("rendering needs the package playwright-core and a browser: npm install playwright-core, then npx playwright-core install chromium-headless-shell. For the MCP server with npx, start it with npx -y -p polite-crawler -p playwright-core polite-crawler mcp") || /Executable doesn't exist|Cannot find (?:module|package) 'playwright-core'/.test(message);
712
+ /** This text tells why the crawler did not fetch a page: a rule of the site, or no robots.txt to read. */
713
+ const robotsRefusal = (robots, by) => robots.unreachable === void 0 ? `robots.txt disallows it for ${by}` : `robots.txt could not be read (${robots.unreachable}), so nothing on the site is fetched`;
714
+ const messageOf$2 = (error) => error instanceof Error ? error.message.split(/\r?\n/)[0] ?? "" : String(error);
715
+ const isoOf = (ms) => Number.isFinite(ms) && Math.abs(ms) < LAST_DATE_MS ? new Date(ms).toISOString() : void 0;
716
+ const stopNote = (reason, until) => {
717
+ const end = isoOf(until);
718
+ return end === void 0 ? `host stopped: ${reason}` : `host stopped until ${end}: ${reason}`;
719
+ };
720
+ const secretSet = (secretParams) => new Set([...DEFAULT_SECRET_PARAMS, ...secretParams].map((name) => name.toLowerCase()));
721
+ const nameOfParam = (raw) => {
722
+ try {
723
+ return decodeURIComponent(raw.replace(/\+/g, " ")).toLowerCase();
724
+ } catch {
725
+ return raw.toLowerCase();
726
+ }
727
+ };
728
+ /**
729
+ * This function gives the URL of a capture. It removes a user name and a password, and it writes
730
+ * `REDACTED` in place of the value of each secret parameter. The other parameters stay as they
731
+ * were written.
732
+ */
733
+ const redact = (url, secretParams = []) => {
734
+ const copy = new URL(url.href);
735
+ copy.username = "";
736
+ copy.password = "";
737
+ if (copy.search.length > 1) {
738
+ const secrets = secretSet(secretParams);
739
+ copy.search = `?${copy.search.slice(1).split("&").map((part) => {
740
+ const equals = part.indexOf("=");
741
+ if (equals === -1) return part;
742
+ const name = part.slice(0, equals);
743
+ return secrets.has(nameOfParam(name)) ? `${name}=REDACTED` : part;
744
+ }).join("&")}`;
745
+ }
746
+ return copy.href;
747
+ };
748
+ /** This function tells if a URL holds a secret: a user name, a password or a secret parameter. */
749
+ const holdsSecrets = (url, secretParams) => {
750
+ if (url.username !== "" || url.password !== "") return true;
751
+ const secrets = secretSet(secretParams);
752
+ return url.search.slice(1).split("&").some((part) => part.includes("=") && secrets.has(nameOfParam(part.slice(0, part.indexOf("=")))));
753
+ };
754
+ /** The pace, the stops and the robots.txt of a host do not change with a trailing dot or the case of the name. */
755
+ const hostKey = (url) => `${url.hostname.toLowerCase().replace(/\.$/, "")}${url.port === "" ? "" : `:${url.port}`}`;
756
+ const originKey = (url) => `${url.protocol}//${hostKey(url)}`;
757
+ const headerOf = (headers) => (name) => {
758
+ const value = headers[name.toLowerCase()];
759
+ return Array.isArray(value) ? value.join(", ") : value;
760
+ };
761
+ var Lru = class {
762
+ max;
763
+ map = /* @__PURE__ */ new Map();
764
+ constructor(max) {
765
+ this.max = max;
766
+ }
767
+ get(key) {
768
+ const value = this.map.get(key);
769
+ if (value !== void 0) {
770
+ this.map.delete(key);
771
+ this.map.set(key, value);
772
+ }
773
+ return value;
774
+ }
775
+ peek(key) {
776
+ return this.map.get(key);
777
+ }
778
+ set(key, value) {
779
+ this.map.delete(key);
780
+ this.map.set(key, value);
781
+ if (this.map.size > this.max) this.map.delete(this.map.keys().next().value);
782
+ }
783
+ delete(key) {
784
+ this.map.delete(key);
785
+ }
786
+ keys() {
787
+ return [...this.map.keys()];
788
+ }
789
+ get size() {
790
+ return this.map.size;
791
+ }
792
+ };
793
+ /** A body failed in transport (a reset, a timeout) or in decompression. */
794
+ var BodyError = class extends Error {
795
+ kind;
796
+ constructor(kind, cause) {
797
+ super(messageOf$2(cause), { cause });
798
+ this.kind = kind;
799
+ }
800
+ };
801
+ /** RFC 9110 says that `deflate` is the zlib format, but some servers send raw deflate. This stream reads both. */
802
+ const inflateAny = () => {
803
+ let inner;
804
+ const outer = new Transform({
805
+ transform(chunk, _encoding, done) {
806
+ if (inner === void 0) {
807
+ inner = chunk.length >= 2 && (chunk[0] & 15) === 8 && (chunk[0] << 8 | chunk[1]) % 31 === 0 ? createInflate() : createInflateRaw();
808
+ inner.on("data", (data) => outer.push(data));
809
+ inner.on("error", (error) => outer.destroy(error));
810
+ }
811
+ inner.write(chunk, () => done());
812
+ },
813
+ flush(done) {
814
+ if (inner === void 0) {
815
+ done();
816
+ return;
817
+ }
818
+ inner.once("end", () => done());
819
+ inner.end();
820
+ },
821
+ destroy(error, callback) {
822
+ inner?.destroy();
823
+ callback(error);
824
+ }
825
+ });
826
+ return outer;
827
+ };
828
+ const decoderFor = (coding) => {
829
+ if (coding === "gzip" || coding === "x-gzip") return createGunzip();
830
+ if (coding === "br") return createBrotliDecompress();
831
+ if (coding === "deflate") return inflateAny();
832
+ throw new BodyError("decode", /* @__PURE__ */ new Error(`unsupported content-encoding "${coding}"`));
833
+ };
834
+ /** This function tells if a decoder stopped because the compressed body ended early: the server cut it. */
835
+ const endedEarly = (error) => typeof error === "object" && error !== null && "code" in error && (error.code === "Z_BUF_ERROR" || /unexpected end of file/i.test(messageOf$2(error)));
836
+ /**
837
+ * This function reads a body as a stream and decodes each content coding in sequence. It counts
838
+ * the decoded bytes and keeps at most `maxBytes`. Thus a decompression bomb stops at the
839
+ * maximum, and a long page keeps its first part. A body that fails gives a `BodyError`. A
840
+ * compressed body that ends early gives the part that it decoded, marked `cut`.
841
+ */
842
+ const readBody = async (body, encoding, maxBytes) => {
843
+ const codings = (encoding ?? "").split(",").map((coding) => coding.trim().toLowerCase()).filter((coding) => coding.length > 0 && !IDENTITY_CODINGS.has(coding)).reverse();
844
+ body.on("error", () => void 0);
845
+ let decoders;
846
+ try {
847
+ decoders = codings.map(decoderFor);
848
+ } catch (error) {
849
+ body.destroy();
850
+ throw error;
851
+ }
852
+ const rawCap = maxBytes * 4 + 65536;
853
+ let rawBytes = 0;
854
+ let stream = body;
855
+ if (decoders.length > 0) {
856
+ body.on("data", (chunk) => {
857
+ rawBytes += chunk.length;
858
+ });
859
+ for (const decoder of decoders) {
860
+ const source = stream;
861
+ source.on("error", (error) => decoder.destroy(error));
862
+ stream = source.pipe(decoder);
863
+ }
864
+ }
865
+ const chunks = [];
866
+ let size = 0;
867
+ let truncated = false;
868
+ let cut = false;
869
+ try {
870
+ for await (const chunk of stream) {
871
+ const room = maxBytes - size;
872
+ if (chunk.length > room) {
873
+ if (room > 0) chunks.push(chunk.subarray(0, room));
874
+ size += Math.max(0, room);
875
+ truncated = true;
876
+ break;
877
+ }
878
+ chunks.push(chunk);
879
+ size += chunk.length;
880
+ if (rawBytes > rawCap) {
881
+ truncated = true;
882
+ break;
883
+ }
884
+ }
885
+ } catch (error) {
886
+ const transport = body.errored !== null || decoders.length === 0;
887
+ if (!transport && size > 0 && endedEarly(error)) {
888
+ cut = true;
889
+ truncated = true;
890
+ } else throw new BodyError(transport ? "transport" : "decode", error);
891
+ } finally {
892
+ body.destroy();
893
+ for (const decoder of decoders) decoder.destroy();
894
+ }
895
+ const bytes = new Uint8Array(size);
896
+ let at = 0;
897
+ for (const chunk of chunks) {
898
+ bytes.set(chunk, at);
899
+ at += chunk.length;
900
+ }
901
+ return {
902
+ bytes,
903
+ truncated,
904
+ cut
905
+ };
906
+ };
907
+ const directives = (value) => {
908
+ const out = /* @__PURE__ */ new Map();
909
+ for (const part of (value ?? "").split(",")) {
910
+ const equals = part.indexOf("=");
911
+ const name = (equals === -1 ? part : part.slice(0, equals)).trim().toLowerCase();
912
+ if (name.length === 0) continue;
913
+ const raw = equals === -1 ? "" : part.slice(equals + 1).trim();
914
+ out.set(name, raw.length >= 2 && raw.startsWith("\"") && raw.endsWith("\"") ? raw.slice(1, -1) : raw);
915
+ }
916
+ return out;
917
+ };
918
+ const TEN_YEARS_MS = 31536e7;
919
+ /**
920
+ * This function gives the end of the freshness of a response, as RFC 9111 §4.2 calculates it.
921
+ * The sources in sequence are `max-age`, `Expires`, and 10 percent of the age of
922
+ * `Last-Modified`, with a maximum. A response with `no-store` or `no-cache` is stale at once. A
923
+ * response with no freshness gives null.
924
+ */
925
+ const freshUntil = (header, status, fetchedAt, lastModified, capMs) => {
926
+ const control = directives(header("cache-control"));
927
+ if (control.has("no-store") || control.has("no-cache")) return isoOf(fetchedAt) ?? null;
928
+ const ageText = header("age")?.trim() ?? "";
929
+ const age = /^\d+$/.test(ageText) ? Number(ageText) * 1e3 : 0;
930
+ const date = parseHttpDate(header("date")) ?? fetchedAt;
931
+ let lifetime;
932
+ const maxAge = control.get("max-age");
933
+ const expires = header("expires");
934
+ if (maxAge !== void 0 && /^\d+$/.test(maxAge)) lifetime = Number(maxAge) * 1e3;
935
+ else if (expires !== void 0) {
936
+ const at = parseHttpDate(expires);
937
+ lifetime = at === void 0 ? 0 : at - date;
938
+ } else if (HEURISTIC_STATUSES.has(status)) {
939
+ const modified = parseHttpDate(lastModified ?? void 0);
940
+ if (modified !== void 0 && modified < date) lifetime = Math.min(capMs, (date - modified) / 10);
941
+ else if (PERMANENT_REDIRECTS.has(status)) lifetime = capMs;
942
+ }
943
+ if (lifetime === void 0) return null;
944
+ return isoOf(fetchedAt + Math.max(0, Math.min(lifetime, TEN_YEARS_MS) - age)) ?? null;
945
+ };
946
+ /** This function gives the request headers that a response varies on, with their values. A `Vary: *` gives `*`. */
947
+ const varyKeyOf = (vary, sent) => {
948
+ if (vary === void 0) return null;
949
+ const names = vary.split(",").map((name) => name.trim().toLowerCase()).filter((name) => name.length > 0 && !CONSTANT_HEADERS.has(name));
950
+ if (names.includes("*")) return "*";
951
+ if (names.length === 0) return null;
952
+ return [...new Set(names)].sort().map((name) => `${name}=${sent[name] ?? ""}`).join("\n");
953
+ };
954
+ const varyMatches = (stored, sent) => {
955
+ if (stored === void 0 || stored === null) return true;
956
+ if (stored === "*") return false;
957
+ return stored.split("\n").every((line) => {
958
+ const equals = line.indexOf("=");
959
+ return (sent[line.slice(0, equals)] ?? "") === line.slice(equals + 1);
960
+ });
961
+ };
962
+ const withoutCredentials = (headers) => headers === void 0 ? void 0 : Object.fromEntries(Object.entries(headers).filter(([name]) => !CREDENTIAL_HEADERS.has(name.toLowerCase())));
963
+ const invalidNumber = (name, value, auto) => {
964
+ if (value === void 0 || auto && value === "auto") return void 0;
965
+ return typeof value === "number" && value >= 0 ? void 0 : `invalid option: ${name} is ${String(value)}`;
966
+ };
967
+ const closedResult = (url) => ({
968
+ url,
969
+ status: 0,
970
+ outcome: "skipped",
971
+ fromCache: false,
972
+ capture: void 0,
973
+ blob: void 0,
974
+ body: void 0,
975
+ contentType: void 0,
976
+ signals: NO_SIGNALS,
977
+ note: "crawler closed"
978
+ });
979
+ const robotsCheck = (verdict, tokens, decision) => ({
980
+ capture: verdict.capture ?? null,
981
+ tokens,
982
+ rule: decision.rule === void 0 ? null : {
983
+ line: decision.rule.line ?? null,
984
+ pattern: decision.rule.pattern,
985
+ allow: decision.rule.allow
986
+ },
987
+ group_line: decision.groups?.[0]?.line ?? null,
988
+ ...verdict.robots.unreachable !== void 0 ? { unreachable: verdict.robots.unreachable } : {},
989
+ ...verdict.stale !== void 0 ? { stale: verdict.stale } : {}
990
+ });
991
+ const raceAbort = (work, signal) => {
992
+ if (signal.aborted) {
993
+ work.catch(() => void 0);
994
+ return Promise.reject(signal.reason);
995
+ }
996
+ return new Promise((resolve, reject) => {
997
+ const cancel = () => reject(signal.reason);
998
+ signal.addEventListener("abort", cancel, { once: true });
999
+ work.then((value) => {
1000
+ signal.removeEventListener("abort", cancel);
1001
+ resolve(value);
1002
+ }, (error) => {
1003
+ signal.removeEventListener("abort", cancel);
1004
+ reject(error);
1005
+ });
1006
+ });
1007
+ };
1008
+ var Crawler = class {
1009
+ options;
1010
+ clock;
1011
+ random;
1012
+ scheduler;
1013
+ dispatcher;
1014
+ ownsDispatcher;
1015
+ egress;
1016
+ robots = new Lru(4096);
1017
+ /** The fields of a cache entry that the index does not keep, and the raw target of a redirect. */
1018
+ meta = new Lru(1e4);
1019
+ inFlight = /* @__PURE__ */ new Map();
1020
+ running = /* @__PURE__ */ new Set();
1021
+ closing = new AbortController();
1022
+ closed = false;
1023
+ closingDone;
1024
+ /** This note tells why the renderer has no browser. After the first page that finds it, the crawler renders nothing. */
1025
+ renderOff;
1026
+ /** The renders that run at one time (`maxRenders`). */
1027
+ renders;
1028
+ /** These tokens apply because the `User-Agent` links the bot page (`obeyDefaultToken`). */
1029
+ pageTokens;
1030
+ constructor(options) {
1031
+ this.options = options;
1032
+ if (!isProductToken(options.token)) throw new Error(`the token "${options.token}" is not a product token: use only letters, "_" and "-" (RFC 9309)`);
1033
+ for (const [name, value] of [["userAgent", options.userAgent], ["from", options.from]]) if (value !== void 0 && !HEADER_TEXT.test(value)) throw new Error(`${name} "${value}" has a character that an HTTP header cannot hold: use printable ASCII`);
1034
+ checkPolicy(options.politeness);
1035
+ this.clock = options.clock ?? realClock;
1036
+ this.random = options.random ?? Math.random;
1037
+ this.egress = options.egress ?? {};
1038
+ this.scheduler = new HostScheduler(this.clock, options.maxActive ?? 16, { store: options.hostState });
1039
+ this.pageTokens = options.obeyDefaultToken !== false && options.token.toLowerCase() !== "PoliteCrawler".toLowerCase() && options.userAgent.includes("https://mark1russell7.github.io/crawler/bot/") ? [DEFAULT_TOKEN] : [];
1040
+ const maxRenders = options.maxRenders ?? 2;
1041
+ if (!Number.isInteger(maxRenders) || maxRenders < 1) throw new RangeError(`maxRenders must be an integer of 1 or more, not ${String(maxRenders)}`);
1042
+ this.renders = new Slots(maxRenders);
1043
+ this.ownsDispatcher = options.dispatcher === void 0;
1044
+ this.dispatcher = options.dispatcher ?? new Agent({
1045
+ keepAliveTimeout: 6e4,
1046
+ keepAliveMaxTimeout: 12e4,
1047
+ allowH2: true,
1048
+ connect: {
1049
+ timeout: 1e4,
1050
+ lookup: egressLookup(this.egress)
1051
+ },
1052
+ headersTimeout: 3e4,
1053
+ bodyTimeout: 6e4
1054
+ });
1055
+ }
1056
+ iso() {
1057
+ return new Date(this.clock.now()).toISOString();
1058
+ }
1059
+ policyOf(url) {
1060
+ return policyFor(this.options.politeness, url.hostname);
1061
+ }
1062
+ emit(event) {
1063
+ try {
1064
+ this.options.onEvent?.(event);
1065
+ } catch {}
1066
+ }
1067
+ track(work) {
1068
+ this.running.add(work);
1069
+ work.then(() => this.running.delete(work), () => this.running.delete(work));
1070
+ return work;
1071
+ }
1072
+ stopHost(host, until, reason) {
1073
+ this.scheduler.store.stop(host, until, reason);
1074
+ this.emit({
1075
+ type: "stop",
1076
+ host,
1077
+ until,
1078
+ reason
1079
+ });
1080
+ }
1081
+ async record(kind, payload, url, labels = {}) {
1082
+ if (this.closed) return void 0;
1083
+ const at = this.iso();
1084
+ const scope = labels.scope ?? this.options.scope;
1085
+ const record = makeRecord({
1086
+ source: "fetch",
1087
+ kind,
1088
+ payload: {
1089
+ schema: 2,
1090
+ ...payload,
1091
+ fetched_at: at
1092
+ },
1093
+ received_at: at,
1094
+ occurred_at: at,
1095
+ external_id: url,
1096
+ ...scope !== void 0 ? { scope } : {},
1097
+ ...labels.labels !== void 0 ? { labels: labels.labels } : {}
1098
+ });
1099
+ await this.options.log.append(record);
1100
+ return record.id;
1101
+ }
1102
+ async skip(base, url, note, extra = {}, labels = {}) {
1103
+ this.emit({
1104
+ type: "skip",
1105
+ url,
1106
+ note
1107
+ });
1108
+ return {
1109
+ url,
1110
+ status: 0,
1111
+ outcome: "skipped",
1112
+ fromCache: false,
1113
+ capture: await this.record("skipped", {
1114
+ ...base,
1115
+ url,
1116
+ note,
1117
+ ...extra
1118
+ }, url, labels),
1119
+ blob: void 0,
1120
+ body: void 0,
1121
+ contentType: void 0,
1122
+ signals: NO_SIGNALS,
1123
+ note
1124
+ };
1125
+ }
1126
+ /** This method reads the cache entry of a URL, with the fields that the crawler keeps in memory. */
1127
+ lookup(url, variant = "plain") {
1128
+ const key = cacheKeyOf(url);
1129
+ const row = this.options.index.latest(key, variant);
1130
+ if (row === void 0) return void 0;
1131
+ const meta = this.meta.get(`${variant} ${key}`);
1132
+ if (meta === void 0 || meta.entry.capture !== row.capture) return row;
1133
+ const kept = Object.fromEntries(Object.entries(row).filter(([, value]) => value !== void 0));
1134
+ return {
1135
+ ...meta.entry,
1136
+ ...kept
1137
+ };
1138
+ }
1139
+ store(entry, location) {
1140
+ const variant = entry.variant ?? "plain";
1141
+ const keyed = {
1142
+ ...entry,
1143
+ url: cacheKeyOf(entry.url),
1144
+ variant
1145
+ };
1146
+ this.options.index.setLatest(keyed);
1147
+ this.meta.set(`${variant} ${keyed.url}`, {
1148
+ entry: keyed,
1149
+ ...location !== void 0 ? { location } : {}
1150
+ });
1151
+ }
1152
+ /** This method gives the target of a stored redirect, when the crawler can follow it. */
1153
+ followable(entry, secretParams) {
1154
+ const meta = this.meta.peek(`${entry.variant ?? "plain"} ${cacheKeyOf(entry.url)}`);
1155
+ if (meta?.location !== void 0 && meta.entry.capture === entry.capture) return meta.location;
1156
+ if (entry.location === void 0 || entry.location === null) return void 0;
1157
+ try {
1158
+ const target = new URL(entry.location);
1159
+ return holdsSecrets(target, secretParams) ? void 0 : target.href;
1160
+ } catch {
1161
+ return;
1162
+ }
1163
+ }
1164
+ /** This method answers from the cache, when the stored entry is young or fresh enough. */
1165
+ async fromCache(url, variant, maxAge, sent, secretParams) {
1166
+ if (maxAge === void 0) return void 0;
1167
+ const entry = this.lookup(url, variant);
1168
+ if (entry === void 0 || !varyMatches(entry.vary_key, sent)) return void 0;
1169
+ const now = this.clock.now();
1170
+ const fetchedAt = Date.parse(entry.fetched_at);
1171
+ if (!Number.isFinite(fetchedAt)) return void 0;
1172
+ const age = now - fetchedAt;
1173
+ const until = entry.fresh_until === void 0 || entry.fresh_until === null ? NaN : Date.parse(entry.fresh_until);
1174
+ const fresh = Number.isFinite(until) && until > now;
1175
+ const redirect = entry.status >= 300 && entry.status < 400;
1176
+ if (!(maxAge === "auto" ? fresh : redirect && !PERMANENT_REDIRECTS.has(entry.status) ? fresh && age < maxAge : age < maxAge)) return void 0;
1177
+ if (redirect) {
1178
+ const next = this.followable(entry, secretParams);
1179
+ if (next === void 0) return void 0;
1180
+ return {
1181
+ fetched: {
1182
+ url,
1183
+ status: entry.status,
1184
+ outcome: "redirect",
1185
+ fromCache: true,
1186
+ capture: entry.capture,
1187
+ blob: void 0,
1188
+ body: void 0,
1189
+ contentType: void 0,
1190
+ signals: NO_SIGNALS,
1191
+ note: `location ${entry.location ?? redact(new URL(next), secretParams)}`,
1192
+ fetchedAt: entry.fetched_at
1193
+ },
1194
+ next
1195
+ };
1196
+ }
1197
+ if (entry.blob === null || entry.status < 200 || entry.status >= 300) return void 0;
1198
+ const body = await this.options.blobs.get(entry.blob);
1199
+ if (body === void 0) return void 0;
1200
+ return { fetched: {
1201
+ url,
1202
+ status: entry.status,
1203
+ outcome: "ok",
1204
+ fromCache: true,
1205
+ capture: entry.capture,
1206
+ blob: entry.blob,
1207
+ body,
1208
+ contentType: entry.content_type ?? void 0,
1209
+ signals: {
1210
+ ai_input: entry.ai_input ?? "unset",
1211
+ raw: []
1212
+ },
1213
+ note: entry.truncated === true ? "a truncated copy" : void 0,
1214
+ ...entry.truncated === true ? { truncated: true } : {},
1215
+ fetchedAt: entry.fetched_at
1216
+ } };
1217
+ }
1218
+ requestHeaders(request_) {
1219
+ const headers = {};
1220
+ for (const [name, value] of Object.entries(request_.headers ?? {})) {
1221
+ const lower = name.toLowerCase();
1222
+ if (!RESERVED_HEADERS.has(lower)) headers[lower] = value;
1223
+ }
1224
+ headers["accept"] = request_.accept ?? headers["accept"] ?? "*/*";
1225
+ headers["user-agent"] = this.options.userAgent;
1226
+ headers["accept-encoding"] = "gzip, deflate, br";
1227
+ if (this.options.from !== void 0) headers["from"] = this.options.from;
1228
+ return headers;
1229
+ }
1230
+ /**
1231
+ * This method records the result of a request for the host, and it gives the rest of the host.
1232
+ * The rest comes from `restAfter`. A 429 or 503 waits, an error counts toward a stop, and a
1233
+ * refusal stops the host for a time.
1234
+ */
1235
+ settle(host, policy, step) {
1236
+ const store = this.scheduler.store;
1237
+ const now = this.clock.now();
1238
+ const delayMs = restAfter(policy, step.rest).ms;
1239
+ const waited = (ms, reason) => {
1240
+ if (ms > delayMs) this.emit({
1241
+ type: "wait",
1242
+ host,
1243
+ ms,
1244
+ reason
1245
+ });
1246
+ return ms;
1247
+ };
1248
+ if (step.result !== void 0) {
1249
+ store.record(host, step.result);
1250
+ return delayMs;
1251
+ }
1252
+ const outcome = step.outcome;
1253
+ if (outcome === "rate-limited" || outcome === "unavailable") {
1254
+ const { waits } = store.record(host, "wait");
1255
+ const asked = step.retryAfterMs;
1256
+ if (asked !== void 0 && asked > policy.maxRetryAfterMs) {
1257
+ this.stopHost(host, now + Math.min(asked, MAX_STOP_MS), `asked to wait ${String(Math.round(asked / 1e3))} s`);
1258
+ return delayMs;
1259
+ }
1260
+ return waited(Math.max(delayMs, asked ?? equalJitterMs(waits, policy.backoffBaseMs, policy.backoffCapMs, this.random)), asked === void 0 ? "backoff" : "retry-after");
1261
+ }
1262
+ if (outcome === "error") {
1263
+ if (isClientError(step.status)) {
1264
+ store.record(host, "neutral");
1265
+ return delayMs;
1266
+ }
1267
+ const { failures } = store.record(host, "failure");
1268
+ if (failures >= policy.maxFailures) this.stopHost(host, now + policy.failureStopMs, `${String(failures)} failures in a row`);
1269
+ return waited(Math.max(delayMs, equalJitterMs(failures, policy.backoffBaseMs, policy.backoffCapMs, this.random)), "backoff");
1270
+ }
1271
+ if (outcome === "challenge") this.stopHost(host, now + policy.challengeStopMs, outcome);
1272
+ else if (outcome === "payment-required") this.stopHost(host, now + policy.paymentStopMs, outcome);
1273
+ else if (outcome === "blocked" && !step.robotsFile) this.stopHost(host, now + policy.blockedStopMs, outcome);
1274
+ store.record(host, "success");
1275
+ return delayMs;
1276
+ }
1277
+ /**
1278
+ * This method stops the host of a URL as a challenge does: for the `challengeStopMs` of the policy.
1279
+ * The runtime uses it when a page that answered 200 is a bot check (`meta.blocked`).
1280
+ */
1281
+ reportBlocked(url, what) {
1282
+ let target;
1283
+ try {
1284
+ target = new URL(url);
1285
+ } catch {
1286
+ return;
1287
+ }
1288
+ const policy = this.policyOf(target);
1289
+ this.stopHost(hostKey(target), this.clock.now() + policy.challengeStopMs, `challenge (${what})`);
1290
+ }
1291
+ /**
1292
+ * This method applies the rules that a fetch and a render share. The sequence is the URL, the
1293
+ * options, the egress policy, the product tokens and the rules for an agent. Then come the
1294
+ * cache, the stops of the host and robots.txt. It gives the answer when a rule decides, or the
1295
+ * facts that the transport needs.
1296
+ */
1297
+ async admit(spec, extra) {
1298
+ const marks = {
1299
+ scope: spec.scope,
1300
+ labels: spec.labels
1301
+ };
1302
+ let target;
1303
+ try {
1304
+ target = new URL(spec.url);
1305
+ } catch {
1306
+ return { answer: { fetched: await this.skip(spec.base, spec.url, "not a valid URL", {}, marks) } };
1307
+ }
1308
+ if (!WEB_PROTOCOLS.has(target.protocol)) return { answer: { fetched: await this.skip(spec.base, redact(target, spec.secretParams), "not an http(s) URL", {}, marks) } };
1309
+ target.hash = "";
1310
+ const url = redact(target, spec.secretParams);
1311
+ const skip = (note, extra_) => this.skip(spec.base, url, note, extra_, marks);
1312
+ const answer = async (note, extra_) => ({ answer: { fetched: await skip(note, extra_) } });
1313
+ if (spec.invalid !== void 0) return answer(spec.invalid);
1314
+ const refused = egressRefusal(this.egress, target.hostname);
1315
+ if (refused !== void 0) return answer(refused);
1316
+ const tokens = [
1317
+ this.options.token,
1318
+ ...spec.onBehalfOf ?? [],
1319
+ ...this.pageTokens
1320
+ ];
1321
+ const badToken = tokens.find((token) => !isProductToken(token));
1322
+ if (spec.mode === "page" && badToken !== void 0) return answer(`not a valid product token: "${badToken}"`);
1323
+ const host = hostKey(target);
1324
+ const policy = this.policyOf(target);
1325
+ const path = `${target.pathname}${target.search}`;
1326
+ let verdict;
1327
+ if (spec.mode === "page") {
1328
+ verdict = tokens.length > 1 ? await this.robotsFor(target, policy, spec) : this.knownRobots(target);
1329
+ if (verdict !== void 0) {
1330
+ const decision = decide(verdict.robots, tokens, path);
1331
+ if (!decision.allowed) return answer(robotsRefusal(verdict.robots, decision.by ?? this.options.token), { robots_check: robotsCheck(verdict, tokens, decision) });
1332
+ }
1333
+ }
1334
+ const cached = await spec.cached(url);
1335
+ if (cached !== void 0) return { answer: cached };
1336
+ const stop = this.scheduler.stopOf(host);
1337
+ if (stop !== void 0) return answer(stopNote(stop.reason, stop.until));
1338
+ const early = await extra?.({
1339
+ target,
1340
+ url,
1341
+ host,
1342
+ policy,
1343
+ tokens,
1344
+ skip
1345
+ });
1346
+ if (early !== void 0) return { answer: { fetched: early } };
1347
+ let robotsLines = [];
1348
+ let crawlDelaySeconds = 0;
1349
+ let check;
1350
+ if (spec.mode === "page") {
1351
+ verdict ??= await this.robotsFor(target, policy, spec);
1352
+ const decision = decide(verdict.robots, tokens, path);
1353
+ check = robotsCheck(verdict, tokens, decision);
1354
+ if (!decision.allowed) return answer(robotsRefusal(verdict.robots, decision.by ?? this.options.token), { robots_check: check });
1355
+ robotsLines = signalsFor(verdict.robots, tokens, path);
1356
+ crawlDelaySeconds = decision.crawlDelaySeconds ?? 0;
1357
+ }
1358
+ if (this.closed) return { answer: { fetched: closedResult(url) } };
1359
+ return { admitted: {
1360
+ target,
1361
+ url,
1362
+ host,
1363
+ policy,
1364
+ tokens,
1365
+ verdict,
1366
+ check,
1367
+ robotsLines,
1368
+ crawlDelaySeconds,
1369
+ skip
1370
+ } };
1371
+ }
1372
+ /** This method fetches a URL. It follows the redirects up to the maximum, and each step is a separate decision. */
1373
+ async fetch(request_) {
1374
+ if (this.closed) return closedResult(request_.url);
1375
+ return this.track(this.follow(request_));
1376
+ }
1377
+ async follow(request_) {
1378
+ const signal = request_.signal === void 0 ? this.closing.signal : AbortSignal.any([request_.signal, this.closing.signal]);
1379
+ let url = request_.url;
1380
+ let headers = request_.headers;
1381
+ let origin;
1382
+ let maxRedirects = DEFAULT_POLICY.maxRedirects;
1383
+ try {
1384
+ const first = new URL(url);
1385
+ origin = first.origin;
1386
+ maxRedirects = this.policyOf(first).maxRedirects;
1387
+ } catch {}
1388
+ try {
1389
+ for (let hop = 0;; hop += 1) {
1390
+ const step = await this.hop({
1391
+ ...request_,
1392
+ url,
1393
+ headers
1394
+ }, signal);
1395
+ if (step.next === void 0) return step.fetched;
1396
+ if (hop >= maxRedirects) return {
1397
+ ...step.fetched,
1398
+ outcome: "error",
1399
+ note: `more than ${String(maxRedirects)} redirects`
1400
+ };
1401
+ const next = new URL(step.next);
1402
+ if (next.origin !== origin) headers = withoutCredentials(headers);
1403
+ origin = next.origin;
1404
+ url = step.next;
1405
+ }
1406
+ } catch (error) {
1407
+ if (this.closed) return closedResult(request_.url);
1408
+ return {
1409
+ ...closedResult(request_.url),
1410
+ outcome: "error",
1411
+ note: `crawler error: ${messageOf$2(error)}`
1412
+ };
1413
+ }
1414
+ }
1415
+ /** This method joins a request to an equal request in progress, so one URL gets one request at a time. */
1416
+ hop(request_, signal) {
1417
+ if (request_.signal !== void 0) return this.fetchOnce(request_, signal);
1418
+ const key = JSON.stringify([
1419
+ cacheKeyOf(request_.url.split("#")[0] ?? request_.url),
1420
+ request_.mode,
1421
+ request_.recipe,
1422
+ request_.accept ?? null,
1423
+ request_.headers ?? null,
1424
+ request_.secretParams ?? null,
1425
+ request_.minIntervalMs ?? null,
1426
+ request_.maxAgeMs ?? null,
1427
+ request_.maxBytes ?? null,
1428
+ request_.onBehalfOf ?? null,
1429
+ request_.job ?? null,
1430
+ request_.scope ?? null,
1431
+ request_.labels ?? null
1432
+ ]);
1433
+ const running = this.inFlight.get(key);
1434
+ if (running !== void 0) return running;
1435
+ const work = this.fetchOnce(request_, signal).finally(() => this.inFlight.delete(key));
1436
+ this.inFlight.set(key, work);
1437
+ return work;
1438
+ }
1439
+ knownRobots(target) {
1440
+ const entry = this.robots.peek(originKey(target));
1441
+ return entry !== void 0 && this.clock.now() - entry.at < entry.ttlMs ? entry.value : void 0;
1442
+ }
1443
+ /**
1444
+ * This method gives the robots.txt of the origin of a URL. A good copy stays for the TTL of the
1445
+ * policy. An unreachable verdict stays for some minutes, with a backoff and the `Retry-After`
1446
+ * of the server. During that time the last good copy applies, when there is one (RFC 9309 §2.4).
1447
+ */
1448
+ robotsFor(target, policy, marks = {}) {
1449
+ const key = originKey(target);
1450
+ const now = this.clock.now();
1451
+ const cached = this.robots.get(key);
1452
+ if (cached !== void 0 && now - cached.at < cached.ttlMs) {
1453
+ if (cached.value === void 0) this.join(cached.waiting, marks);
1454
+ return cached.verdict;
1455
+ }
1456
+ const waiting = {
1457
+ scope: void 0,
1458
+ labels: void 0
1459
+ };
1460
+ this.join(waiting, marks);
1461
+ const entry = {
1462
+ at: now,
1463
+ waiting,
1464
+ ttlMs: policy.robotsTtlMs,
1465
+ verdict: Promise.resolve({
1466
+ robots: unreachableRobots("not read yet"),
1467
+ capture: void 0
1468
+ }),
1469
+ value: void 0,
1470
+ misses: cached?.misses ?? 0,
1471
+ good: cached?.good
1472
+ };
1473
+ entry.verdict = this.fetchRobots(target, policy, waiting).catch((error) => ({
1474
+ robots: unreachableRobots(`robots.txt failed: ${messageOf$2(error)}`),
1475
+ capture: void 0,
1476
+ retryAfterMs: void 0
1477
+ })).then(async (read) => {
1478
+ let verdict = {
1479
+ robots: read.robots,
1480
+ capture: read.capture
1481
+ };
1482
+ if (read.robots.unreachable === void 0) {
1483
+ entry.misses = 0;
1484
+ entry.good = {
1485
+ verdict,
1486
+ at: this.clock.now()
1487
+ };
1488
+ } else {
1489
+ entry.misses += 1;
1490
+ const backoff = equalJitterMs(entry.misses, policy.robotsRetryMs, policy.robotsRetryCapMs, this.random);
1491
+ entry.ttlMs = Math.min(policy.robotsRetryCapMs, Math.max(read.retryAfterMs ?? 0, backoff));
1492
+ const good = entry.good !== void 0 && this.clock.now() - entry.good.at <= policy.robotsStaleMs ? entry.good.verdict : await this.storedRobots(target, policy);
1493
+ if (good !== void 0) verdict = {
1494
+ ...good,
1495
+ stale: read.robots.unreachable
1496
+ };
1497
+ }
1498
+ entry.value = verdict;
1499
+ return verdict;
1500
+ });
1501
+ this.robots.set(key, entry);
1502
+ return entry.verdict;
1503
+ }
1504
+ /**
1505
+ * This method gives the robots.txt rules that apply to a URL, as the crawler applies them. They
1506
+ * come from its memory, or from a fetch of `/robots.txt` with its redirects. A robots.txt that
1507
+ * the crawler cannot read gives rules that permit nothing, and `unreachable` gives the reason.
1508
+ */
1509
+ async readRobots(url) {
1510
+ const target = new URL(url);
1511
+ if (!WEB_PROTOCOLS.has(target.protocol)) return {
1512
+ robots: unreachableRobots("not an http(s) URL"),
1513
+ capture: void 0
1514
+ };
1515
+ return this.robotsFor(target, this.policyOf(target));
1516
+ }
1517
+ /** This method reads the stored copy of a robots.txt, when it is younger than the limit of the policy. */
1518
+ async storedRobots(target, policy) {
1519
+ try {
1520
+ const row = this.lookup(redact(new URL("/robots.txt", target.origin)));
1521
+ if (row === void 0 || row.blob === null || row.status < 200 || row.status >= 300) return void 0;
1522
+ if (this.clock.now() - Date.parse(row.fetched_at) > policy.robotsStaleMs) return void 0;
1523
+ const body = await this.options.blobs.get(row.blob);
1524
+ if (body === void 0) return void 0;
1525
+ const robots = robotsFromResponse(200, row.content_type, body);
1526
+ return robots.unreachable === void 0 ? {
1527
+ robots,
1528
+ capture: row.capture
1529
+ } : void 0;
1530
+ } catch {
1531
+ return;
1532
+ }
1533
+ }
1534
+ /** This method adds the scope and the labels of one request to the requests that wait for a robots.txt. The most private scope wins. */
1535
+ join(waiting, marks) {
1536
+ const scope = marks.scope ?? this.options.scope;
1537
+ const rank = this.options.scopeRank;
1538
+ if (waiting.scope === void 0) waiting.scope = scope;
1539
+ else if (scope !== void 0 && rank !== void 0 && rank(scope) > rank(waiting.scope)) waiting.scope = scope;
1540
+ if (marks.labels !== void 0) waiting.labels = {
1541
+ ...marks.labels,
1542
+ ...waiting.labels
1543
+ };
1544
+ }
1545
+ async fetchRobots(target, policy, waiting) {
1546
+ let url = new URL("/robots.txt", target.origin);
1547
+ const unreachable = (reason, capture, retryAfterMs) => ({
1548
+ robots: unreachableRobots(reason),
1549
+ capture,
1550
+ retryAfterMs
1551
+ });
1552
+ for (let hop = 0; hop <= policy.maxRedirects; hop += 1) {
1553
+ const step = await this.fetchOnce({
1554
+ url: url.href,
1555
+ mode: "api",
1556
+ recipe: "robots.txt",
1557
+ accept: "text/plain",
1558
+ maxBytes: ROBOTS_BYTES,
1559
+ minIntervalMs: policy.apiIntervalMs,
1560
+ get scope() {
1561
+ return waiting.scope;
1562
+ },
1563
+ get labels() {
1564
+ return waiting.labels;
1565
+ }
1566
+ }, this.closing.signal);
1567
+ const result = step.fetched;
1568
+ if (result.outcome === "redirect") {
1569
+ if (step.next === void 0) return unreachable(result.note ?? "a redirect with no Location header", result.capture);
1570
+ url = new URL(step.next);
1571
+ continue;
1572
+ }
1573
+ if (result.outcome === "skipped" || result.status === 0) return unreachable(result.note ?? "no response", result.capture);
1574
+ if (result.outcome === "error" && result.status >= 200 && result.status < 300) return unreachable(result.note ?? "the body could not be read", result.capture);
1575
+ if (result.note?.startsWith("the compressed body ended early") === true) return unreachable(result.note, result.capture);
1576
+ if (result.outcome === "not-modified") return result.body === void 0 ? unreachable("status 304, and no stored copy", result.capture) : {
1577
+ robots: robotsFromResponse(200, result.contentType ?? null, result.body),
1578
+ capture: result.capture,
1579
+ retryAfterMs: void 0
1580
+ };
1581
+ return {
1582
+ robots: result.status >= 200 && result.status < 300 ? robotsFromResponse(result.status, result.contentType ?? null, result.body ?? /* @__PURE__ */ new Uint8Array()) : robotsFromStatus(result.status, void 0),
1583
+ capture: result.capture,
1584
+ retryAfterMs: step.retryAfterMs
1585
+ };
1586
+ }
1587
+ return unreachable(`more than ${String(policy.maxRedirects)} redirects`, void 0);
1588
+ }
1589
+ async fetchOnce(request_, signal) {
1590
+ const base = {
1591
+ recipe: request_.recipe,
1592
+ mode: request_.mode,
1593
+ ...request_.job ? { job: request_.job } : {}
1594
+ };
1595
+ const secretParams = request_.secretParams ?? [];
1596
+ const sent = this.requestHeaders(request_);
1597
+ const marks = {
1598
+ get scope() {
1599
+ return request_.scope;
1600
+ },
1601
+ get labels() {
1602
+ return request_.labels;
1603
+ }
1604
+ };
1605
+ const admission = await this.admit({
1606
+ transport: "http",
1607
+ url: request_.url,
1608
+ mode: request_.mode,
1609
+ base,
1610
+ secretParams,
1611
+ invalid: invalidNumber("minIntervalMs", request_.minIntervalMs, false) ?? invalidNumber("maxBytes", request_.maxBytes, false) ?? invalidNumber("maxAgeMs", request_.maxAgeMs, true),
1612
+ onBehalfOf: request_.onBehalfOf,
1613
+ scope: request_.scope,
1614
+ labels: request_.labels,
1615
+ cached: (url) => this.fromCache(url, "plain", request_.maxAgeMs, sent, secretParams)
1616
+ });
1617
+ if ("answer" in admission) return admission.answer;
1618
+ const { target, url, host, policy, check, robotsLines, crawlDelaySeconds, skip } = admission.admitted;
1619
+ const maxBytes = request_.maxBytes ?? policy.maxBytes;
1620
+ const rest = (elapsedMs) => ({
1621
+ kind: request_.mode,
1622
+ elapsedMs,
1623
+ crawlDelaySeconds,
1624
+ minIntervalMs: request_.minIntervalMs
1625
+ });
1626
+ const robotsFile = request_.recipe === "robots.txt";
1627
+ try {
1628
+ return await this.scheduler.run(host, async () => {
1629
+ const again = await this.fromCache(url, "plain", request_.maxAgeMs, sent, secretParams);
1630
+ if (again !== void 0) return {
1631
+ value: again,
1632
+ delayMs: 0
1633
+ };
1634
+ const latest = this.lookup(url);
1635
+ const usable = latest !== void 0 && latest.truncated !== true && varyMatches(latest.vary_key, sent) ? latest : void 0;
1636
+ let stored;
1637
+ if (usable?.blob && (usable.etag || usable.last_modified)) stored = await this.options.blobs.get(usable.blob);
1638
+ const headers = { ...sent };
1639
+ if (stored !== void 0 && usable?.etag) headers["if-none-match"] = usable.etag;
1640
+ if (stored !== void 0 && usable?.last_modified) headers["if-modified-since"] = usable.last_modified;
1641
+ const started = this.clock.now();
1642
+ const timeout = AbortSignal.timeout(policy.requestTimeoutMs);
1643
+ const abort = AbortSignal.any([timeout, signal]);
1644
+ const cancelled = () => request_.signal?.aborted === true;
1645
+ const failedNote = (prefix, error) => timeout.aborted ? `${prefix}: timed out after ${String(Math.round(policy.requestTimeoutMs / 1e3))} s` : `${prefix}: ${messageOf$2(error)}`;
1646
+ this.emit({
1647
+ type: "request",
1648
+ url,
1649
+ host,
1650
+ transport: "http"
1651
+ });
1652
+ let response;
1653
+ try {
1654
+ response = await request(target, {
1655
+ method: "GET",
1656
+ headers,
1657
+ dispatcher: this.dispatcher,
1658
+ signal: abort
1659
+ });
1660
+ } catch (error) {
1661
+ const egress = egressError(error);
1662
+ if (egress !== void 0) return {
1663
+ value: { fetched: await skip(egress.message) },
1664
+ delayMs: 0
1665
+ };
1666
+ if (this.closed) return {
1667
+ value: { fetched: closedResult(url) },
1668
+ delayMs: 0
1669
+ };
1670
+ const elapsedMs = this.clock.now() - started;
1671
+ const note = cancelled() ? "cancelled by the caller" : failedNote("no response", error);
1672
+ const delayMs = this.settle(host, policy, {
1673
+ outcome: "error",
1674
+ status: 0,
1675
+ retryAfterMs: void 0,
1676
+ rest: rest(elapsedMs),
1677
+ robotsFile,
1678
+ ...cancelled() ? { result: "neutral" } : {}
1679
+ });
1680
+ this.emit({
1681
+ type: "response",
1682
+ url,
1683
+ host,
1684
+ status: 0,
1685
+ outcome: "error",
1686
+ elapsedMs
1687
+ });
1688
+ const capture = await this.record("capture", {
1689
+ ...base,
1690
+ url,
1691
+ status: 0,
1692
+ outcome: "error",
1693
+ note,
1694
+ elapsed_ms: elapsedMs,
1695
+ ...check ? { robots_check: check } : {}
1696
+ }, url, marks);
1697
+ return {
1698
+ value: { fetched: {
1699
+ url,
1700
+ status: 0,
1701
+ outcome: "error",
1702
+ fromCache: false,
1703
+ capture,
1704
+ blob: void 0,
1705
+ body: void 0,
1706
+ contentType: void 0,
1707
+ signals: NO_SIGNALS,
1708
+ note,
1709
+ fetchedAt: new Date(started).toISOString(),
1710
+ ...marks
1711
+ } },
1712
+ delayMs
1713
+ };
1714
+ }
1715
+ const header = headerOf(response.headers);
1716
+ const ttfbMs = this.clock.now() - started;
1717
+ const classified = classify(response.statusCode, header, this.clock.now());
1718
+ let outcome = classified.outcome;
1719
+ let note = classified.note;
1720
+ let body;
1721
+ let blob;
1722
+ let truncated = false;
1723
+ let cut = false;
1724
+ let next;
1725
+ let result;
1726
+ if (outcome === "ok") {
1727
+ try {
1728
+ const read = await readBody(response.body, header("content-encoding"), maxBytes);
1729
+ body = read.bytes;
1730
+ truncated = read.truncated;
1731
+ cut = read.cut;
1732
+ if (read.cut) note = `the compressed body ended early, so the crawler keeps the ${String(read.bytes.length)} bytes before the end`;
1733
+ else if (truncated) note = `truncated at ${String(maxBytes)} bytes`;
1734
+ } catch (error) {
1735
+ if (this.closed) return {
1736
+ value: { fetched: closedResult(url) },
1737
+ delayMs: 0
1738
+ };
1739
+ outcome = "error";
1740
+ if (cancelled()) {
1741
+ note = "cancelled by the caller";
1742
+ result = "neutral";
1743
+ } else note = failedNote(error instanceof BodyError && error.kind === "decode" ? "undecodable body" : "body failed", error);
1744
+ }
1745
+ if (body !== void 0) try {
1746
+ blob = (await this.options.blobs.put(body)).hash;
1747
+ } catch (error) {
1748
+ outcome = "error";
1749
+ note = `blob store: ${messageOf$2(error)}`;
1750
+ body = void 0;
1751
+ result = "neutral";
1752
+ }
1753
+ } else try {
1754
+ await readBody(response.body, void 0, ERROR_BODY_BYTES);
1755
+ } catch {}
1756
+ const elapsedMs = this.clock.now() - started;
1757
+ if (outcome === "not-modified") {
1758
+ if (stored !== void 0 && usable?.blob) {
1759
+ body = stored;
1760
+ blob = usable.blob;
1761
+ } else note = "not modified, but the crawler has no stored copy";
1762
+ }
1763
+ let location;
1764
+ if (outcome === "redirect") {
1765
+ const raw = header("location");
1766
+ if (raw === void 0) note = "a redirect with no Location header";
1767
+ else try {
1768
+ const resolved = new URL(raw, target);
1769
+ next = resolved.href;
1770
+ location = redact(resolved, secretParams);
1771
+ note = `location ${location}`;
1772
+ } catch {
1773
+ outcome = "error";
1774
+ note = "bad redirect target";
1775
+ }
1776
+ }
1777
+ const revisit = outcome === "not-modified" && usable !== void 0;
1778
+ let signals = readSignals(robotsLines, header, pageSignals(body, header("content-type")));
1779
+ if (revisit) {
1780
+ const merged = signals.ai_input === "no" || usable.ai_input === "no" ? "no" : signals.ai_input !== "unset" ? signals.ai_input : usable.ai_input;
1781
+ signals = {
1782
+ ...signals,
1783
+ ai_input: merged
1784
+ };
1785
+ }
1786
+ const contentType = header("content-type") ?? (revisit ? usable.content_type ?? void 0 : void 0);
1787
+ const unasked = request_.mode === "api" && outcome === "ok" && unaskedPage(request_.accept, contentType);
1788
+ if (unasked) {
1789
+ const said = `the server sent ${contentType ?? ""}, and the request accepts ${request_.accept ?? ""}, so the crawler keeps no stored copy of it`;
1790
+ note = note === void 0 ? said : `${note}; ${said}`;
1791
+ }
1792
+ const etag = header("etag") ?? (revisit ? usable.etag : null) ?? null;
1793
+ const lastModified = header("last-modified") ?? (revisit ? usable.last_modified : null) ?? null;
1794
+ const fetchedAt = this.clock.now();
1795
+ const freshness = freshUntil(header, response.statusCode, fetchedAt, lastModified, policy.heuristicFreshnessCapMs);
1796
+ const varyKey = header("vary") !== void 0 ? varyKeyOf(header("vary"), sent) : revisit ? usable.vary_key ?? null : null;
1797
+ const cacheHeaders = Object.fromEntries([
1798
+ "cache-control",
1799
+ "expires",
1800
+ "age",
1801
+ "date",
1802
+ "vary"
1803
+ ].flatMap((name) => {
1804
+ const value = header(name);
1805
+ return value === void 0 ? [] : [[name.replace("-", "_"), value]];
1806
+ }));
1807
+ this.emit({
1808
+ type: "response",
1809
+ url,
1810
+ host,
1811
+ status: response.statusCode,
1812
+ outcome,
1813
+ elapsedMs
1814
+ });
1815
+ const capture = await this.record(revisit ? "revisit" : "capture", {
1816
+ ...base,
1817
+ url,
1818
+ status: response.statusCode,
1819
+ outcome,
1820
+ elapsed_ms: elapsedMs,
1821
+ ttfb_ms: ttfbMs,
1822
+ content_type: contentType ?? null,
1823
+ bytes: body?.length ?? 0,
1824
+ blob: blob ?? null,
1825
+ etag,
1826
+ last_modified: lastModified,
1827
+ ...revisit ? { revisit_of: usable.capture } : {},
1828
+ robots: request_.mode === "page" ? "allowed" : "not-checked",
1829
+ ...check !== void 0 ? { robots_check: check } : {},
1830
+ signals,
1831
+ ...truncated ? { truncated: true } : {},
1832
+ ...cut ? { cut: true } : {},
1833
+ ...unasked ? { unasked: true } : {},
1834
+ ...cacheHeaders,
1835
+ ...freshness !== null ? { fresh_until: freshness } : {},
1836
+ ...varyKey !== null ? { vary_key: varyKey } : {},
1837
+ ...location !== void 0 ? { location } : {},
1838
+ ...note !== void 0 ? { note } : {}
1839
+ }, url, marks);
1840
+ const fetchedIso = new Date(fetchedAt).toISOString();
1841
+ if (capture !== void 0 && (outcome === "ok" || outcome === "not-modified") && blob !== void 0 && !cut && !unasked) this.store({
1842
+ url,
1843
+ capture,
1844
+ blob,
1845
+ etag: truncated ? null : etag,
1846
+ last_modified: truncated ? null : lastModified,
1847
+ content_type: contentType ?? null,
1848
+ fetched_at: fetchedIso,
1849
+ status: revisit ? usable.status === 304 ? 200 : usable.status : response.statusCode,
1850
+ ai_input: signals.ai_input,
1851
+ truncated: truncated ? true : void 0,
1852
+ fresh_until: freshness,
1853
+ vary_key: varyKey
1854
+ });
1855
+ if (capture !== void 0 && outcome === "redirect" && next !== void 0 && (PERMANENT_REDIRECTS.has(response.statusCode) || freshness !== null)) this.store({
1856
+ url,
1857
+ capture,
1858
+ blob: null,
1859
+ etag: null,
1860
+ last_modified: null,
1861
+ content_type: null,
1862
+ fetched_at: fetchedIso,
1863
+ status: response.statusCode,
1864
+ ai_input: "unset",
1865
+ location: location ?? null,
1866
+ fresh_until: freshness,
1867
+ vary_key: varyKey
1868
+ }, next);
1869
+ const delayMs = this.settle(host, policy, {
1870
+ outcome,
1871
+ status: response.statusCode,
1872
+ retryAfterMs: classified.retryAfterMs,
1873
+ rest: rest(elapsedMs),
1874
+ robotsFile,
1875
+ ...result !== void 0 ? { result } : {}
1876
+ });
1877
+ return {
1878
+ value: {
1879
+ fetched: {
1880
+ url,
1881
+ status: response.statusCode,
1882
+ outcome,
1883
+ fromCache: false,
1884
+ capture,
1885
+ blob,
1886
+ body,
1887
+ contentType,
1888
+ signals,
1889
+ note,
1890
+ ...truncated ? { truncated: true } : {},
1891
+ fetchedAt: fetchedIso,
1892
+ ...marks
1893
+ },
1894
+ next,
1895
+ retryAfterMs: classified.retryAfterMs
1896
+ },
1897
+ delayMs
1898
+ };
1899
+ }, signal);
1900
+ } catch (error) {
1901
+ if (error instanceof HostStopped) return { fetched: await skip(stopNote(error.reason, error.until)) };
1902
+ if (this.closed) return { fetched: closedResult(url) };
1903
+ if (request_.signal?.aborted === true) return { fetched: await skip("cancelled by the caller") };
1904
+ throw error;
1905
+ }
1906
+ }
1907
+ /**
1908
+ * This method gives the function that decides each request of a rendered page. It refuses an
1909
+ * address that the egress policy refuses, and a URL of the page's host that robots.txt
1910
+ * disallows. For a different host, it uses a robots.txt that the crawler holds, when there is one.
1911
+ */
1912
+ allowRequestFor(page, robots, tokens, signal, tally) {
1913
+ const pageOrigin = originKey(page);
1914
+ const checked = /* @__PURE__ */ new Map();
1915
+ const refuse = () => {
1916
+ tally.refused += 1;
1917
+ return false;
1918
+ };
1919
+ return async (raw) => {
1920
+ if (signal.aborted) return false;
1921
+ let target;
1922
+ try {
1923
+ target = new URL(raw);
1924
+ } catch {
1925
+ return refuse();
1926
+ }
1927
+ if (target.protocol === "data:" || target.protocol === "blob:" || target.protocol === "about:") return true;
1928
+ const web = WEB_PROTOCOLS.has(target.protocol);
1929
+ if (!web && target.protocol !== "ws:" && target.protocol !== "wss:") return refuse();
1930
+ let check = checked.get(target.hostname);
1931
+ if (check === void 0) {
1932
+ check = checkEgress(this.egress, target);
1933
+ checked.set(target.hostname, check);
1934
+ }
1935
+ if (await check !== void 0) return refuse();
1936
+ if (web) {
1937
+ const rules = originKey(target) === pageOrigin ? robots : this.knownRobots(target)?.robots;
1938
+ if (rules !== void 0 && !decide(rules, tokens, `${target.pathname}${target.search}`).allowed) return refuse();
1939
+ }
1940
+ tally.allowed += 1;
1941
+ if (tally.hosts.size < 50) tally.hosts.add(hostKey(target));
1942
+ return true;
1943
+ };
1944
+ }
1945
+ /**
1946
+ * This method tells why the browser did not go on to a URL that the page moved to: the egress
1947
+ * policy or robots.txt. For a different site, it uses a robots.txt that the crawler holds. It
1948
+ * gives undefined for the page itself, or for a URL that it cannot read.
1949
+ */
1950
+ async movedRefusal(raw, page, robots, tokens) {
1951
+ let target;
1952
+ try {
1953
+ target = new URL(raw);
1954
+ } catch {
1955
+ return;
1956
+ }
1957
+ target.hash = "";
1958
+ if (target.href === page.href) return void 0;
1959
+ const url = redact(target);
1960
+ const moved = `the page moved to ${url}`;
1961
+ if (!WEB_PROTOCOLS.has(target.protocol)) return {
1962
+ note: `${moved}, which is not an http(s) URL`,
1963
+ url
1964
+ };
1965
+ const egress = await checkEgress(this.egress, target);
1966
+ if (egress !== void 0) return {
1967
+ note: `${moved}: ${egress}`,
1968
+ url
1969
+ };
1970
+ const rules = originKey(target) === originKey(page) ? robots : this.knownRobots(target)?.robots;
1971
+ if (rules !== void 0) {
1972
+ const decision = decide(rules, tokens, `${target.pathname}${target.search}`);
1973
+ if (!decision.allowed) return {
1974
+ note: `${moved}: ${robotsRefusal(rules, decision.by ?? this.options.token)}`,
1975
+ url
1976
+ };
1977
+ }
1978
+ }
1979
+ /**
1980
+ * This method renders a page in the headless browser and captures what the page shows. The rules
1981
+ * of a fetch apply:
1982
+ *
1983
+ * - the egress policy and robots.txt, for the page, its final URL and each request of the page
1984
+ * - one request at a time for each host
1985
+ * - the backoff after a 429 or a 503
1986
+ * - a capture for each decision
1987
+ *
1988
+ * A render costs the site more than a fetch, so the host rests longer after one. A render that
1989
+ * the browser refuses (`RenderRefused`) does not count against the host.
1990
+ */
1991
+ async render(request_) {
1992
+ if (this.closed) return closedResult(request_.url);
1993
+ return this.track(this.renderOnce(request_).catch((error) => this.closed ? closedResult(request_.url) : {
1994
+ ...closedResult(request_.url),
1995
+ outcome: "error",
1996
+ note: `crawler error: ${messageOf$2(error)}`
1997
+ }));
1998
+ }
1999
+ async renderOnce(request_) {
2000
+ const base = {
2001
+ recipe: request_.recipe,
2002
+ mode: "page",
2003
+ rendered: true,
2004
+ ...request_.job ? { job: request_.job } : {}
2005
+ };
2006
+ const marks = {
2007
+ scope: request_.scope,
2008
+ labels: request_.labels
2009
+ };
2010
+ const renderer = this.options.renderer;
2011
+ const admission = await this.admit({
2012
+ transport: "browser",
2013
+ url: request_.url,
2014
+ mode: "page",
2015
+ base,
2016
+ secretParams: [],
2017
+ invalid: invalidNumber("maxAgeMs", request_.maxAgeMs, true),
2018
+ onBehalfOf: request_.onBehalfOf,
2019
+ scope: request_.scope,
2020
+ labels: request_.labels,
2021
+ cached: async (url) => {
2022
+ const cached = await this.fromCache(url, "rendered", request_.maxAgeMs, {}, []);
2023
+ if (cached === void 0 || cached.fetched.body === void 0) return void 0;
2024
+ return { fetched: {
2025
+ ...cached.fetched,
2026
+ url,
2027
+ contentType: RENDERED_TYPE
2028
+ } };
2029
+ }
2030
+ }, async ({ target, skip }) => {
2031
+ if (renderer === void 0) return skip("no browser configured for rendering");
2032
+ if (this.renderOff !== void 0) return skip(this.renderOff);
2033
+ const resolved = await checkEgress(this.egress, target);
2034
+ return resolved === void 0 ? void 0 : skip(resolved);
2035
+ });
2036
+ if ("answer" in admission) return admission.answer.fetched;
2037
+ const { target, url, host, policy, tokens, check, crawlDelaySeconds, skip } = admission.admitted;
2038
+ const verdict = admission.admitted.verdict;
2039
+ const signal = request_.signal === void 0 ? this.closing.signal : AbortSignal.any([request_.signal, this.closing.signal]);
2040
+ const tally = {
2041
+ allowed: 0,
2042
+ refused: 0,
2043
+ hosts: /* @__PURE__ */ new Set()
2044
+ };
2045
+ const allowRequest = this.allowRequestFor(target, verdict.robots, tokens, signal, tally);
2046
+ const rest = (elapsedMs) => ({
2047
+ kind: "render",
2048
+ elapsedMs,
2049
+ crawlDelaySeconds
2050
+ });
2051
+ let release;
2052
+ try {
2053
+ release = await this.renders.acquire(signal);
2054
+ } catch {
2055
+ if (this.closed) return closedResult(url);
2056
+ return skip("cancelled by the caller");
2057
+ }
2058
+ let done;
2059
+ try {
2060
+ done = await this.scheduler.run(host, async () => {
2061
+ const started = this.clock.now();
2062
+ this.emit({
2063
+ type: "request",
2064
+ url,
2065
+ host,
2066
+ transport: "browser"
2067
+ });
2068
+ try {
2069
+ const rendered = await raceAbort(renderer.render(target.href, {
2070
+ userAgent: this.options.userAgent,
2071
+ timeoutMs: policy.renderTimeoutMs,
2072
+ ...this.options.from !== void 0 ? { headers: { from: this.options.from } } : {},
2073
+ allowRequest,
2074
+ maxRedirects: policy.maxRedirects,
2075
+ signal
2076
+ }), signal);
2077
+ const header = (name) => rendered.headers[name.toLowerCase()];
2078
+ const classified = rendered.status === null ? {
2079
+ outcome: "ok",
2080
+ retryAfterMs: void 0,
2081
+ note: void 0
2082
+ } : classify(rendered.status, header, this.clock.now());
2083
+ const elapsedMs = this.clock.now() - started;
2084
+ const delayMs = this.settle(host, policy, {
2085
+ outcome: classified.outcome,
2086
+ status: rendered.status ?? 200,
2087
+ retryAfterMs: classified.retryAfterMs,
2088
+ rest: rest(elapsedMs),
2089
+ robotsFile: false
2090
+ });
2091
+ return {
2092
+ value: {
2093
+ kind: "rendered",
2094
+ rendered,
2095
+ classified,
2096
+ elapsedMs
2097
+ },
2098
+ delayMs
2099
+ };
2100
+ } catch (error) {
2101
+ const elapsedMs = this.clock.now() - started;
2102
+ if (this.closed) return {
2103
+ value: { kind: "closed" },
2104
+ delayMs: 0
2105
+ };
2106
+ if (request_.signal?.aborted === true) return {
2107
+ value: {
2108
+ kind: "cancelled",
2109
+ elapsedMs
2110
+ },
2111
+ delayMs: restAfter(policy, rest(0)).ms
2112
+ };
2113
+ const refused = error instanceof RenderRefused ? error : missingBrowser(messageOf$2(error)) ? new RenderRefused("browser", `rendering needs a browser: ${messageOf$2(error)}`) : void 0;
2114
+ if (refused !== void 0) {
2115
+ if (refused.reason === "browser") this.renderOff = refused.message;
2116
+ const delayMs = this.settle(host, policy, {
2117
+ outcome: "skipped",
2118
+ status: 0,
2119
+ retryAfterMs: void 0,
2120
+ rest: rest(0),
2121
+ robotsFile: false,
2122
+ result: "neutral"
2123
+ });
2124
+ return {
2125
+ value: {
2126
+ kind: "refused",
2127
+ reason: refused.reason,
2128
+ note: refused.message,
2129
+ url: refused.url,
2130
+ elapsedMs
2131
+ },
2132
+ delayMs
2133
+ };
2134
+ }
2135
+ const delayMs = this.settle(host, policy, {
2136
+ outcome: "error",
2137
+ status: 0,
2138
+ retryAfterMs: void 0,
2139
+ rest: rest(elapsedMs),
2140
+ robotsFile: false
2141
+ });
2142
+ return {
2143
+ value: {
2144
+ kind: "failed",
2145
+ note: `render failed: ${messageOf$2(error)}`,
2146
+ elapsedMs
2147
+ },
2148
+ delayMs
2149
+ };
2150
+ }
2151
+ }, signal);
2152
+ } catch (error) {
2153
+ if (error instanceof HostStopped) return skip(stopNote(error.reason, error.until));
2154
+ if (this.closed) return closedResult(url);
2155
+ if (request_.signal?.aborted === true) return skip("cancelled by the caller");
2156
+ throw error;
2157
+ } finally {
2158
+ release();
2159
+ }
2160
+ if (done.kind === "closed" || this.closed) return closedResult(url);
2161
+ const subrequests = {
2162
+ allowed: tally.allowed,
2163
+ refused: tally.refused,
2164
+ hosts: [...tally.hosts]
2165
+ };
2166
+ if (done.kind === "refused") {
2167
+ this.emit({
2168
+ type: "response",
2169
+ url,
2170
+ host,
2171
+ status: 0,
2172
+ outcome: done.reason === "download" ? "not-html" : "skipped",
2173
+ elapsedMs: done.elapsedMs
2174
+ });
2175
+ if (done.reason !== "download") {
2176
+ const moved = done.url === void 0 ? void 0 : await this.movedRefusal(done.url, target, verdict.robots, tokens);
2177
+ return skip(moved?.note ?? done.note, {
2178
+ robots_check: check,
2179
+ subrequests,
2180
+ ...moved === void 0 ? {} : { final_url: moved.url }
2181
+ });
2182
+ }
2183
+ const capture = await this.record("capture", {
2184
+ ...base,
2185
+ url,
2186
+ status: 0,
2187
+ outcome: "not-html",
2188
+ note: done.note,
2189
+ elapsed_ms: done.elapsedMs,
2190
+ robots_check: check,
2191
+ subrequests
2192
+ }, url, marks);
2193
+ return {
2194
+ ...closedResult(url),
2195
+ outcome: "not-html",
2196
+ capture,
2197
+ note: done.note,
2198
+ ...marks
2199
+ };
2200
+ }
2201
+ if (done.kind === "failed" || done.kind === "cancelled") {
2202
+ const note = done.kind === "failed" ? done.note : "cancelled by the caller";
2203
+ this.emit({
2204
+ type: "response",
2205
+ url,
2206
+ host,
2207
+ status: 0,
2208
+ outcome: "error",
2209
+ elapsedMs: done.elapsedMs
2210
+ });
2211
+ return {
2212
+ url,
2213
+ status: 0,
2214
+ outcome: "error",
2215
+ fromCache: false,
2216
+ capture: await this.record("capture", {
2217
+ ...base,
2218
+ url,
2219
+ status: 0,
2220
+ outcome: "error",
2221
+ note,
2222
+ elapsed_ms: done.elapsedMs,
2223
+ robots_check: check,
2224
+ subrequests
2225
+ }, url, marks),
2226
+ blob: void 0,
2227
+ body: void 0,
2228
+ contentType: void 0,
2229
+ signals: NO_SIGNALS,
2230
+ note,
2231
+ ...marks
2232
+ };
2233
+ }
2234
+ const { rendered, classified, elapsedMs } = done;
2235
+ const header = (name) => rendered.headers[name.toLowerCase()];
2236
+ let finalTarget = target;
2237
+ let finalVerdict = verdict;
2238
+ try {
2239
+ finalTarget = new URL(rendered.finalUrl);
2240
+ finalTarget.hash = "";
2241
+ } catch {
2242
+ finalTarget = target;
2243
+ }
2244
+ const finalUrl = redact(finalTarget);
2245
+ const finalPath = `${finalTarget.pathname}${finalTarget.search}`;
2246
+ if (finalTarget.href !== target.href) {
2247
+ const moved = `the page moved to ${finalUrl}`;
2248
+ if (!WEB_PROTOCOLS.has(finalTarget.protocol)) return skip(`${moved}, which is not an http(s) URL`, { final_url: finalUrl });
2249
+ const refusal = await checkEgress(this.egress, finalTarget);
2250
+ if (refusal !== void 0) return skip(`${moved}: ${refusal}`, { final_url: finalUrl });
2251
+ if (originKey(finalTarget) !== originKey(target)) finalVerdict = await this.robotsFor(finalTarget, this.policyOf(finalTarget), request_);
2252
+ const finalDecision = decide(finalVerdict.robots, tokens, finalPath);
2253
+ if (!finalDecision.allowed) return skip(`${moved}: ${robotsRefusal(finalVerdict.robots, finalDecision.by ?? this.options.token)}`, {
2254
+ final_url: finalUrl,
2255
+ robots_check: robotsCheck(finalVerdict, tokens, finalDecision)
2256
+ });
2257
+ }
2258
+ let outcome = classified.outcome;
2259
+ let note = classified.note;
2260
+ let body;
2261
+ let blob;
2262
+ if (outcome === "ok") {
2263
+ body = new TextEncoder().encode(rendered.html);
2264
+ try {
2265
+ blob = (await this.options.blobs.put(body)).hash;
2266
+ } catch (error) {
2267
+ outcome = "error";
2268
+ note = `blob store: ${messageOf$2(error)}`;
2269
+ body = void 0;
2270
+ }
2271
+ }
2272
+ const signals = readSignals(signalsFor(finalVerdict.robots, tokens, finalPath), header, metaSignals(rendered.html));
2273
+ const status = rendered.status ?? 200;
2274
+ const fetchedAt = this.clock.now();
2275
+ const freshness = freshUntil(header, status, fetchedAt, header("last-modified"), policy.heuristicFreshnessCapMs);
2276
+ this.emit({
2277
+ type: "response",
2278
+ url,
2279
+ host,
2280
+ status,
2281
+ outcome,
2282
+ elapsedMs
2283
+ });
2284
+ const capture = await this.record("capture", {
2285
+ ...base,
2286
+ url,
2287
+ final_url: finalUrl,
2288
+ status,
2289
+ outcome,
2290
+ elapsed_ms: elapsedMs,
2291
+ content_type: RENDERED_TYPE,
2292
+ bytes: body?.length ?? 0,
2293
+ blob: blob ?? null,
2294
+ robots: "allowed",
2295
+ robots_check: check,
2296
+ signals,
2297
+ subrequests,
2298
+ ...freshness !== null ? { fresh_until: freshness } : {},
2299
+ ...note !== void 0 ? { note } : {}
2300
+ }, url, marks);
2301
+ const fetchedIso = new Date(fetchedAt).toISOString();
2302
+ if (capture !== void 0 && blob !== void 0) this.store({
2303
+ url,
2304
+ variant: "rendered",
2305
+ capture,
2306
+ blob,
2307
+ etag: null,
2308
+ last_modified: null,
2309
+ content_type: RENDERED_TYPE,
2310
+ fetched_at: fetchedIso,
2311
+ status,
2312
+ ai_input: signals.ai_input,
2313
+ fresh_until: freshness
2314
+ });
2315
+ return {
2316
+ url,
2317
+ status,
2318
+ outcome,
2319
+ fromCache: false,
2320
+ capture,
2321
+ blob,
2322
+ body,
2323
+ contentType: RENDERED_TYPE,
2324
+ signals,
2325
+ note,
2326
+ fetchedAt: fetchedIso,
2327
+ ...marks
2328
+ };
2329
+ }
2330
+ /**
2331
+ * This method removes the stops and the failure counts of one host, or of each host. It also
2332
+ * forgets the robots.txt verdicts that came from an unreachable robots.txt. The pace of each host
2333
+ * stays. A process that runs for a long time uses it, for example after the network comes back.
2334
+ */
2335
+ resetHosts(host) {
2336
+ const name = host?.toLowerCase().replace(/\.$/, "");
2337
+ this.scheduler.reset(name);
2338
+ for (const key of this.robots.keys()) {
2339
+ const entry = this.robots.peek(key);
2340
+ if ((entry?.value?.robots.unreachable !== void 0 || entry?.value?.stale !== void 0) && (name === void 0 || key.endsWith(`//${name}`))) this.robots.delete(key);
2341
+ }
2342
+ }
2343
+ /**
2344
+ * This method closes the crawler. Work in the queue ends as `skipped` with the note
2345
+ * `crawler closed`, and the crawler writes no capture after the close. Then it closes the
2346
+ * browser, the log and its own dispatcher. A second close waits for the first.
2347
+ */
2348
+ close() {
2349
+ if (this.closingDone === void 0) {
2350
+ this.closed = true;
2351
+ this.closing.abort(/* @__PURE__ */ new Error("crawler closed"));
2352
+ this.closingDone = (async () => {
2353
+ await Promise.allSettled(this.running);
2354
+ await this.options.renderer?.close();
2355
+ await this.options.log.close?.();
2356
+ if (this.ownsDispatcher) await this.dispatcher.close();
2357
+ })();
2358
+ }
2359
+ return this.closingDone;
2360
+ }
2361
+ };
2362
+ //#endregion
2363
+ //#region src/fold.ts
2364
+ const MARK = /\p{M}/u;
2365
+ const NONSPACING = /[\p{Mn}\p{Me}]/gu;
2366
+ const IGNORABLE = /^\p{Default_Ignorable_Code_Point}$/u;
2367
+ const SPACE = /^\s$/u;
2368
+ const SEGMENTED = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u;
2369
+ /** These characters become ASCII: typographic quotes, dashes and the minus sign. */
2370
+ const TYPOGRAPHIC = /* @__PURE__ */ new Map([
2371
+ [8216, 39],
2372
+ [8217, 39],
2373
+ [8218, 39],
2374
+ [8219, 39],
2375
+ [8242, 39],
2376
+ [8220, 34],
2377
+ [8221, 34],
2378
+ [8222, 34],
2379
+ [8223, 34],
2380
+ [8243, 34],
2381
+ [8208, 45],
2382
+ [8209, 45],
2383
+ [8210, 45],
2384
+ [8211, 45],
2385
+ [8212, 45],
2386
+ [8213, 45],
2387
+ [8722, 45],
2388
+ [65112, 45],
2389
+ [65123, 45]
2390
+ ]);
2391
+ const isAsciiSpace = (code) => code === 32 || code >= 9 && code <= 13;
2392
+ /**
2393
+ * This function folds a text, and it keeps the position in the original text of each unit of the
2394
+ * result. Its time is linear. An ASCII character with no combining mark after it takes a short path.
2395
+ */
2396
+ const foldText = (text, mode) => {
2397
+ let codes = new Uint16Array(Math.max(16, text.length + 16));
2398
+ let starts = new Int32Array(codes.length);
2399
+ let ends = new Int32Array(codes.length);
2400
+ let size = 0;
2401
+ let space = true;
2402
+ const grow = (need) => {
2403
+ if (size + need <= codes.length) return;
2404
+ const length = Math.max(codes.length * 2, size + need);
2405
+ const nextCodes = new Uint16Array(length);
2406
+ nextCodes.set(codes.subarray(0, size));
2407
+ const nextStarts = new Int32Array(length);
2408
+ nextStarts.set(starts.subarray(0, size));
2409
+ const nextEnds = new Int32Array(length);
2410
+ nextEnds.set(ends.subarray(0, size));
2411
+ codes = nextCodes;
2412
+ starts = nextStarts;
2413
+ ends = nextEnds;
2414
+ };
2415
+ const push = (code, from, to) => {
2416
+ grow(1);
2417
+ codes[size] = code;
2418
+ starts[size] = from;
2419
+ ends[size] = to;
2420
+ size += 1;
2421
+ };
2422
+ const pushSpace = (from, to) => {
2423
+ if (space) return;
2424
+ push(32, from, to);
2425
+ space = true;
2426
+ };
2427
+ const length = text.length;
2428
+ let index = 0;
2429
+ while (index < length) {
2430
+ const code = text.charCodeAt(index);
2431
+ const next = index + 1 < length ? text.charCodeAt(index + 1) : 0;
2432
+ if (code < 128 && next < 768) {
2433
+ if (isAsciiSpace(code)) pushSpace(index, index + 1);
2434
+ else {
2435
+ push(code >= 65 && code <= 90 ? code + 32 : code, index, index + 1);
2436
+ space = false;
2437
+ }
2438
+ index += 1;
2439
+ continue;
2440
+ }
2441
+ const point = text.codePointAt(index);
2442
+ let end = index + (point > 65535 ? 2 : 1);
2443
+ while (end < length) {
2444
+ const mark = text.codePointAt(end);
2445
+ if (!MARK.test(String.fromCodePoint(mark))) break;
2446
+ end += mark > 65535 ? 2 : 1;
2447
+ }
2448
+ const cluster = text.slice(index, end);
2449
+ const first = String.fromCodePoint(point);
2450
+ if (IGNORABLE.test(first) && end === index + first.length) {
2451
+ index = end;
2452
+ continue;
2453
+ }
2454
+ if (SPACE.test(first)) {
2455
+ pushSpace(index, end);
2456
+ index = end;
2457
+ continue;
2458
+ }
2459
+ let folded = mode === "quote" ? cluster.normalize("NFKC").toLowerCase() : cluster.normalize("NFKD").replace(NONSPACING, "").normalize("NFC").toUpperCase().toLowerCase();
2460
+ if (mode === "search" && SEGMENTED.test(first)) folded = ` ${folded} `;
2461
+ for (let at = 0; at < folded.length; at += 1) {
2462
+ let unit = folded.charCodeAt(at);
2463
+ unit = TYPOGRAPHIC.get(unit) ?? unit;
2464
+ if (unit === 32 || unit < 128 && isAsciiSpace(unit) || unit === 160) {
2465
+ pushSpace(index, end);
2466
+ continue;
2467
+ }
2468
+ push(unit, index, end);
2469
+ space = false;
2470
+ }
2471
+ index = end;
2472
+ }
2473
+ if (size > 0 && codes[size - 1] === 32) size -= 1;
2474
+ let result = "";
2475
+ for (let at = 0; at < size; at += 8192) result += String.fromCharCode(...codes.subarray(at, Math.min(size, at + 8192)));
2476
+ return {
2477
+ text: result,
2478
+ start: starts.subarray(0, size),
2479
+ end: ends.subarray(0, size)
2480
+ };
2481
+ };
2482
+ /** This function gives the folded text alone, for a text with no need of positions. */
2483
+ const fold = (text, mode) => foldText(text, mode).text;
2484
+ const NOT_FOUND = {
2485
+ found: false,
2486
+ start: null,
2487
+ end: null,
2488
+ excerpt: null,
2489
+ count: 0
2490
+ };
2491
+ /** This function moves a position off the second half of a surrogate pair, so a slice does not cut a character. */
2492
+ const atBoundary = (text, position, forward) => {
2493
+ const code = text.charCodeAt(position);
2494
+ if (position > 0 && position < text.length && code >= 56320 && code <= 57343) return forward ? position + 1 : position - 1;
2495
+ return position;
2496
+ };
2497
+ /**
2498
+ * This function finds a quote in a text, in the `quote` mode of `foldText`. It ignores case, runs
2499
+ * of spaces, typographic punctuation, soft hyphens, zero-width characters and the differences of
2500
+ * NFKC, NFC and NFD. The positions are in the original text.
2501
+ */
2502
+ const locate = (text, quote) => {
2503
+ const needle = fold(quote, "quote");
2504
+ if (needle.length === 0) return NOT_FOUND;
2505
+ const haystack = foldText(text, "quote");
2506
+ const hit = haystack.text.indexOf(needle);
2507
+ if (hit === -1) return NOT_FOUND;
2508
+ let count = 0;
2509
+ for (let at = hit; at !== -1; at = haystack.text.indexOf(needle, at + needle.length)) count += 1;
2510
+ const start = haystack.start[hit];
2511
+ const end = haystack.end[hit + needle.length - 1];
2512
+ const from = atBoundary(text, Math.max(0, start - 80), false);
2513
+ const to = atBoundary(text, Math.min(text.length, end + 80), true);
2514
+ return {
2515
+ found: true,
2516
+ start,
2517
+ end,
2518
+ excerpt: `${from > 0 ? "…" : ""}${text.slice(from, to)}${to < text.length ? "…" : ""}`,
2519
+ count
2520
+ };
2521
+ };
2522
+ /** A token character of the search index: a letter, a digit, a private-use character or a spacing mark. */
2523
+ const TOKEN_SPLIT = /[^\p{L}\p{N}\p{Co}\p{Mc}]+/u;
2524
+ /** This function gives the tokens of a text, as the search index reads them. */
2525
+ const tokensOf = (text) => fold(text, "search").split(TOKEN_SPLIT).filter((token) => token.length > 0);
2526
+ const MAX_PARTS = 16;
2527
+ /**
2528
+ * These English words are in almost every document. In the form that needs one part, they match
2529
+ * each document and cost the most time. Thus that form leaves them out when other words remain.
2530
+ */
2531
+ const STOP_WORDS = /* @__PURE__ */ new Set([
2532
+ "a",
2533
+ "an",
2534
+ "and",
2535
+ "are",
2536
+ "as",
2537
+ "at",
2538
+ "be",
2539
+ "by",
2540
+ "for",
2541
+ "from",
2542
+ "how",
2543
+ "in",
2544
+ "is",
2545
+ "it",
2546
+ "of",
2547
+ "on",
2548
+ "or",
2549
+ "that",
2550
+ "the",
2551
+ "this",
2552
+ "to",
2553
+ "was",
2554
+ "what",
2555
+ "when",
2556
+ "where",
2557
+ "which",
2558
+ "who",
2559
+ "with"
2560
+ ]);
2561
+ /**
2562
+ * This function reads a search query. The syntax:
2563
+ *
2564
+ * - A word is a term. The index folds case, accents and Unicode forms, so `cafe` finds `Café`. It
2565
+ * reduces English words to their stems, so `baking` finds `bake`.
2566
+ * - Text in double quotes is a phrase: its words must be next to each other.
2567
+ * - A `*` at the end of a word asks for a prefix: `bake*` finds `bakery`.
2568
+ * - A word of Chinese, Japanese or Thai is a phrase of its characters, so it matches in the text.
2569
+ * - Other punctuation is not syntax. The words `AND`, `OR`, `NOT` and `NEAR` are words.
2570
+ *
2571
+ * The function gives no query for a text with no word. It reads 16 parts or fewer. Each part is
2572
+ * quoted, so the text of a user cannot become FTS5 syntax. The form that needs one part (`any`)
2573
+ * leaves out the common English words (`STOP_WORDS`) when other words remain.
2574
+ */
2575
+ const parseQuery = (q) => {
2576
+ const parts = [];
2577
+ for (const match of q.matchAll(/"([^"]*)"(\*?)|([^\s"]+)/g)) {
2578
+ if (parts.length >= MAX_PARTS) break;
2579
+ const phrase = match[1] !== void 0;
2580
+ const raw = phrase ? match[1] : match[3];
2581
+ const prefix = phrase ? match[2] === "*" : raw.endsWith("*");
2582
+ const tokens = tokensOf(phrase ? raw : raw.replace(/\*+$/, ""));
2583
+ if (tokens.length === 0) continue;
2584
+ parts.push({
2585
+ text: `"${tokens.join(" ")}"${prefix ? " *" : ""}`,
2586
+ term: tokens[0],
2587
+ stop: !phrase && !prefix && tokens.length === 1 && STOP_WORDS.has(tokens[0])
2588
+ });
2589
+ }
2590
+ if (parts.length === 0) return void 0;
2591
+ const words = parts.filter((part) => !part.stop);
2592
+ const any = words.length > 0 ? words : parts;
2593
+ return {
2594
+ all: parts.map((part) => part.text).join(" AND "),
2595
+ any: any.map((part) => part.text).join(" OR "),
2596
+ terms: any.map((part) => part.term),
2597
+ parts: parts.length
2598
+ };
2599
+ };
2600
+ /**
2601
+ * This function gives a short part of a text around the first match of a term, with the match in
2602
+ * square brackets. With no match, it gives the start of the text.
2603
+ */
2604
+ const snippetOf = (text, terms, width = 160) => {
2605
+ if (text.length === 0) return "";
2606
+ let start = -1;
2607
+ let end = -1;
2608
+ const lower = text.toLowerCase();
2609
+ if (lower.length === text.length) for (const term of terms) {
2610
+ const at = lower.indexOf(term);
2611
+ if (at !== -1 && (start === -1 || at < start)) {
2612
+ start = at;
2613
+ end = at + term.length;
2614
+ }
2615
+ }
2616
+ if (start === -1) {
2617
+ const folded = foldText(text, "search");
2618
+ for (const term of terms) {
2619
+ const at = folded.text.indexOf(term);
2620
+ if (at !== -1) {
2621
+ const from = folded.start[at];
2622
+ if (start === -1 || from < start) {
2623
+ start = from;
2624
+ end = folded.end[at + term.length - 1];
2625
+ }
2626
+ }
2627
+ }
2628
+ }
2629
+ if (start === -1) {
2630
+ const cut = atBoundary(text, Math.min(text.length, width), false);
2631
+ return `${text.slice(0, cut)}${cut < text.length ? "…" : ""}`;
2632
+ }
2633
+ const from = atBoundary(text, Math.max(0, start - Math.floor(width / 3)), false);
2634
+ const to = atBoundary(text, Math.min(text.length, end + Math.ceil(width * 2 / 3)), true);
2635
+ const clean = (part) => part.replace(/\s+/g, " ");
2636
+ return `${from > 0 ? "…" : ""}${clean(text.slice(from, start))}[${clean(text.slice(start, end))}]${clean(text.slice(end, to))}${to < text.length ? "…" : ""}`;
2637
+ };
2638
+ /** The search index reads this many characters of each text, at most. */
2639
+ const MAX_INDEX_CHARS = 2e5;
2640
+ const namesOf = (generation) => ({
2641
+ docs: `docs_${String(generation)}`,
2642
+ fts: `fts_${String(generation)}`,
2643
+ versions: `versions_${String(generation)}`,
2644
+ texts: `texts_${String(generation)}`,
2645
+ latest: `latest_${String(generation)}`
2646
+ });
2647
+ const GENERATION_TABLE = /^(docs|fts|versions|texts|latest)_(\d+)$/;
2648
+ /** A build that started this long ago and did not finish belongs to a process that stopped. */
2649
+ const ABANDONED_MS = 864e5;
2650
+ /** The FTS5 tokenizer: folded text, English stems, and spacing marks kept in the words. */
2651
+ const TOKENIZER = `tokenize = "porter unicode61 remove_diacritics 0 categories 'L* N* Co Mc'"`;
2652
+ const ddlOf = (n) => `
2653
+ CREATE TABLE IF NOT EXISTS ${n.texts} (
2654
+ sha TEXT PRIMARY KEY,
2655
+ body TEXT NOT NULL
2656
+ );
2657
+ CREATE TABLE IF NOT EXISTS ${n.versions} (
2658
+ doc TEXT NOT NULL,
2659
+ capture TEXT NOT NULL,
2660
+ recipe TEXT NOT NULL,
2661
+ rendered INTEGER NOT NULL,
2662
+ fetched_at TEXT NOT NULL,
2663
+ blob TEXT,
2664
+ kind TEXT NOT NULL,
2665
+ title TEXT NOT NULL,
2666
+ url TEXT,
2667
+ meta_sha TEXT NOT NULL,
2668
+ text_sha TEXT NOT NULL,
2669
+ text_chars INTEGER NOT NULL,
2670
+ full_sha TEXT,
2671
+ ai_input TEXT NOT NULL,
2672
+ merge TEXT NOT NULL,
2673
+ scope TEXT,
2674
+ labels TEXT,
2675
+ PRIMARY KEY (doc, capture)
2676
+ );
2677
+ CREATE INDEX IF NOT EXISTS ${n.versions}_time ON ${n.versions} (doc, recipe, rendered, fetched_at);
2678
+ CREATE INDEX IF NOT EXISTS ${n.versions}_capture ON ${n.versions} (capture);
2679
+ CREATE INDEX IF NOT EXISTS ${n.versions}_fetched ON ${n.versions} (fetched_at);
2680
+ CREATE INDEX IF NOT EXISTS ${n.versions}_recipe ON ${n.versions} (recipe);
2681
+ CREATE TABLE IF NOT EXISTS ${n.docs} (
2682
+ doc TEXT PRIMARY KEY,
2683
+ kind TEXT NOT NULL,
2684
+ title TEXT NOT NULL,
2685
+ url TEXT,
2686
+ meta TEXT NOT NULL,
2687
+ text_sha TEXT NOT NULL,
2688
+ full_sha TEXT,
2689
+ captures TEXT NOT NULL,
2690
+ capture_count INTEGER NOT NULL,
2691
+ capture TEXT NOT NULL,
2692
+ blob TEXT,
2693
+ fetched_at TEXT NOT NULL,
2694
+ recipe TEXT NOT NULL,
2695
+ rendered INTEGER NOT NULL,
2696
+ ai_input TEXT NOT NULL,
2697
+ scope TEXT,
2698
+ labels TEXT,
2699
+ updated_at TEXT NOT NULL
2700
+ );
2701
+ CREATE VIRTUAL TABLE IF NOT EXISTS ${n.fts} USING fts5(
2702
+ title, text, content = '', contentless_delete = 1, ${TOKENIZER}
2703
+ );
2704
+ CREATE TABLE IF NOT EXISTS ${n.latest} (
2705
+ url TEXT NOT NULL,
2706
+ variant TEXT NOT NULL,
2707
+ capture TEXT NOT NULL,
2708
+ blob TEXT,
2709
+ etag TEXT,
2710
+ last_modified TEXT,
2711
+ content_type TEXT,
2712
+ fetched_at TEXT NOT NULL,
2713
+ status INTEGER NOT NULL,
2714
+ ai_input TEXT NOT NULL,
2715
+ location TEXT,
2716
+ truncated INTEGER,
2717
+ fresh_until TEXT,
2718
+ vary_key TEXT,
2719
+ PRIMARY KEY (url, variant)
2720
+ );
2721
+ CREATE INDEX IF NOT EXISTS ${n.latest}_fetched ON ${n.latest} (fetched_at);
2722
+ `;
2723
+ const RENDERED_MARK$1 = "#rendered";
2724
+ /** This function splits a cache key into its URL and its variant. A key in the form of version 0.1 (`…#rendered`) is a rendered variant. */
2725
+ const keyOf = (url, variant) => {
2726
+ if (url.endsWith(RENDERED_MARK$1)) return {
2727
+ url: cacheKeyOf(url.slice(0, -9)),
2728
+ variant: "rendered"
2729
+ };
2730
+ return {
2731
+ url: cacheKeyOf(url),
2732
+ variant: variant ?? "plain"
2733
+ };
2734
+ };
2735
+ const sha = (text) => createHash("sha256").update(text, "utf8").digest("hex");
2736
+ const parseJson = (text, fallback) => {
2737
+ if (text === null || text === void 0) return fallback;
2738
+ try {
2739
+ return JSON.parse(text);
2740
+ } catch {
2741
+ return fallback;
2742
+ }
2743
+ };
2744
+ const toLatest = (row) => ({
2745
+ url: row.url,
2746
+ capture: row.capture,
2747
+ blob: row.blob,
2748
+ etag: row.etag,
2749
+ last_modified: row.last_modified,
2750
+ content_type: row.content_type,
2751
+ fetched_at: row.fetched_at,
2752
+ status: row.status,
2753
+ ai_input: row.ai_input,
2754
+ location: row.location,
2755
+ truncated: row.truncated === 1 ? true : void 0,
2756
+ fresh_until: row.fresh_until,
2757
+ vary_key: row.vary_key,
2758
+ variant: row.variant
2759
+ });
2760
+ const toVersion = (row) => ({
2761
+ doc: row.doc,
2762
+ capture: row.capture,
2763
+ recipe: row.recipe,
2764
+ rendered: row.rendered === 1,
2765
+ fetched_at: row.fetched_at,
2766
+ blob: row.blob,
2767
+ kind: row.kind,
2768
+ title: row.title,
2769
+ url: row.url,
2770
+ ai_input: row.ai_input,
2771
+ chars: row.text_chars,
2772
+ cut: row.full_sha !== null
2773
+ });
2774
+ const messageOf$1 = (error) => error instanceof Error ? error.message : String(error);
2775
+ var DocIndex = class {
2776
+ path;
2777
+ db;
2778
+ statements = /* @__PURE__ */ new Map();
2779
+ depth = 0;
2780
+ closed = false;
2781
+ empty;
2782
+ /**
2783
+ * This value is true when the file had a different schema, or no tables, so the index opened
2784
+ * empty. A store with captures then needs `reindex`. The commit of a rebuild makes it false.
2785
+ */
2786
+ get stale() {
2787
+ return this.empty;
2788
+ }
2789
+ constructor(path, options = {}) {
2790
+ this.path = path;
2791
+ if (path !== ":memory:") mkdirSync(dirname(path), {
2792
+ recursive: true,
2793
+ mode: 448
2794
+ });
2795
+ this.db = new DatabaseSync(path);
2796
+ const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 1e4));
2797
+ this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);
2798
+ try {
2799
+ this.db.exec("CREATE VIRTUAL TABLE temp.polite_fts5_probe USING fts5(x, content = '', contentless_delete = 1); DROP TABLE temp.polite_fts5_probe;");
2800
+ } catch (cause) {
2801
+ this.db.close();
2802
+ throw new Error(`polite-crawler needs SQLite FTS5 with contentless_delete, and the SQLite of Node.js ${process.version} does not have it: use Node.js 22.19 or later`, { cause });
2803
+ }
2804
+ if (path !== ":memory:") this.db.exec("PRAGMA journal_mode = WAL;");
2805
+ this.db.exec("PRAGMA synchronous = NORMAL;");
2806
+ this.empty = this.write(() => this.prepareSchema());
2807
+ }
2808
+ /** This method makes the tables of the schema. It gives true when the index opened empty. */
2809
+ prepareSchema() {
2810
+ const version = this.db.prepare("PRAGMA user_version").get().user_version;
2811
+ const tables = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all();
2812
+ if (version === 2 && tables.some((table) => table.name === "index_meta")) {
2813
+ this.dropAbandoned();
2814
+ return false;
2815
+ }
2816
+ const virtual = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'").all();
2817
+ for (const { name } of virtual) this.db.exec(`DROP TABLE IF EXISTS "${name}"`);
2818
+ const rest = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all();
2819
+ for (const { name } of rest) this.db.exec(`DROP TABLE IF EXISTS "${name}"`);
2820
+ this.db.exec("CREATE TABLE index_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)");
2821
+ this.db.exec(ddlOf(namesOf(1)));
2822
+ this.statements.clear();
2823
+ this.setMeta("generation", "1");
2824
+ this.setMeta("next_generation", "2");
2825
+ this.db.exec(`PRAGMA user_version = ${String(2)}`);
2826
+ return true;
2827
+ }
2828
+ setMeta(key, value) {
2829
+ this.stmt("INSERT INTO index_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(key, value);
2830
+ }
2831
+ getMeta(key) {
2832
+ return this.stmt("SELECT value FROM index_meta WHERE key = ?").get(key)?.value;
2833
+ }
2834
+ stmt(sql) {
2835
+ let statement = this.statements.get(sql);
2836
+ if (statement === void 0) {
2837
+ statement = this.db.prepare(sql);
2838
+ this.statements.set(sql, statement);
2839
+ }
2840
+ return statement;
2841
+ }
2842
+ /**
2843
+ * This method does a write in one transaction. A transaction in progress takes the write in,
2844
+ * also a transaction that the caller opened on the connection.
2845
+ */
2846
+ write(work) {
2847
+ if (this.closed) throw new Error("the index is closed");
2848
+ if (this.depth > 0 || this.db.isTransaction) {
2849
+ this.depth += 1;
2850
+ try {
2851
+ return work();
2852
+ } finally {
2853
+ this.depth -= 1;
2854
+ }
2855
+ }
2856
+ this.db.exec("BEGIN IMMEDIATE");
2857
+ this.depth = 1;
2858
+ try {
2859
+ const result = work();
2860
+ this.db.exec("COMMIT");
2861
+ return result;
2862
+ } catch (error) {
2863
+ if (this.db.isTransaction) this.db.exec("ROLLBACK");
2864
+ throw error;
2865
+ } finally {
2866
+ this.depth = 0;
2867
+ }
2868
+ }
2869
+ generation() {
2870
+ return Number(this.getMeta("generation") ?? "1");
2871
+ }
2872
+ /**
2873
+ * This method does a step on the tables of the current generation. A rebuild of a different
2874
+ * process can remove those tables in the meantime. Then the method reads the generation again
2875
+ * and does the step once more.
2876
+ */
2877
+ live(step) {
2878
+ if (this.closed) throw new Error("the index is closed");
2879
+ try {
2880
+ return step(namesOf(this.generation()));
2881
+ } catch (error) {
2882
+ if (!/no such table/i.test(messageOf$1(error)) || this.depth > 0) throw error;
2883
+ this.statements.clear();
2884
+ return step(namesOf(this.generation()));
2885
+ }
2886
+ }
2887
+ /** This method removes the tables of a generation. */
2888
+ dropGeneration(generation) {
2889
+ const n = namesOf(generation);
2890
+ for (const table of [
2891
+ n.fts,
2892
+ n.docs,
2893
+ n.versions,
2894
+ n.texts,
2895
+ n.latest
2896
+ ]) this.db.exec(`DROP TABLE IF EXISTS ${table}`);
2897
+ this.db.prepare("DELETE FROM index_meta WHERE key = ?").run(`building:${String(generation)}`);
2898
+ this.statements.clear();
2899
+ }
2900
+ /** This method removes the generations of builds that a stopped process left. */
2901
+ dropAbandoned() {
2902
+ const current = this.generation();
2903
+ const generations = /* @__PURE__ */ new Set();
2904
+ for (const { name } of this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all()) {
2905
+ const match = GENERATION_TABLE.exec(name);
2906
+ if (match !== null) generations.add(Number(match[2]));
2907
+ }
2908
+ for (const generation of generations) {
2909
+ if (generation === current) continue;
2910
+ const started = this.getMeta(`building:${String(generation)}`);
2911
+ if (!((started === void 0 ? Number.POSITIVE_INFINITY : Date.now() - Date.parse(started)) < ABANDONED_MS)) this.dropGeneration(generation);
2912
+ }
2913
+ }
2914
+ putText(n, body) {
2915
+ const key = sha(body);
2916
+ this.stmt(`INSERT OR IGNORE INTO ${n.texts} (sha, body) VALUES (?, ?)`).run(key, body);
2917
+ return key;
2918
+ }
2919
+ textOf(n, key) {
2920
+ if (key === null) return void 0;
2921
+ return this.stmt(`SELECT body FROM ${n.texts} WHERE sha = ?`).get(key)?.body;
2922
+ }
2923
+ putVersion(n, input, now) {
2924
+ const full = input.full_text !== void 0 && input.full_text !== input.text ? this.putText(n, input.full_text) : null;
2925
+ this.stmt(`INSERT INTO ${n.versions} (doc, capture, recipe, rendered, fetched_at, blob, kind, title, url,
2926
+ meta_sha, text_sha, text_chars, full_sha, ai_input, merge, scope, labels)
2927
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
2928
+ ON CONFLICT(doc, capture) DO UPDATE SET recipe = excluded.recipe, rendered = excluded.rendered,
2929
+ fetched_at = excluded.fetched_at, blob = excluded.blob, kind = excluded.kind, title = excluded.title,
2930
+ url = excluded.url, meta_sha = excluded.meta_sha, text_sha = excluded.text_sha,
2931
+ text_chars = excluded.text_chars, full_sha = excluded.full_sha, ai_input = excluded.ai_input,
2932
+ merge = excluded.merge, scope = excluded.scope, labels = excluded.labels`).run(input.doc, input.capture ?? input.captures[0] ?? "", input.recipe ?? "", input.rendered === true ? 1 : 0, input.fetched_at ?? now, input.blob ?? null, input.kind, input.title, input.url, this.putText(n, JSON.stringify(input.meta ?? {})), this.putText(n, input.text), input.text.length, full, input.ai_input ?? "unset", input.merge ?? "richest", input.scope ?? null, input.labels === void 0 ? null : JSON.stringify(input.labels));
2933
+ }
2934
+ newest(n, doc, recipe, rendered) {
2935
+ return this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = ?
2936
+ ORDER BY fetched_at DESC, capture DESC LIMIT 1`).get(doc, recipe, rendered);
2937
+ }
2938
+ /** This method chooses the current version of a document from its versions, and writes it with its search entry. */
2939
+ recompute(n, doc, now) {
2940
+ const groups = this.stmt(`SELECT DISTINCT recipe, rendered FROM ${n.versions} WHERE doc = ?`).all(doc);
2941
+ const before = this.stmt(`SELECT rowid, text_sha, title FROM ${n.docs} WHERE doc = ?`).get(doc);
2942
+ if (groups.length === 0) {
2943
+ if (before !== void 0) {
2944
+ this.stmt(`DELETE FROM ${n.fts} WHERE rowid = ?`).run(before.rowid);
2945
+ this.stmt(`DELETE FROM ${n.docs} WHERE doc = ?`).run(doc);
2946
+ }
2947
+ return;
2948
+ }
2949
+ const byRecipe = /* @__PURE__ */ new Map();
2950
+ for (const group of groups) {
2951
+ const row = this.newest(n, doc, group.recipe, group.rendered);
2952
+ if (row === void 0) continue;
2953
+ const entry = byRecipe.get(group.recipe) ?? {};
2954
+ if (group.rendered === 1) entry.rendered = row;
2955
+ else entry.plain = row;
2956
+ byRecipe.set(group.recipe, entry);
2957
+ }
2958
+ const candidates = [];
2959
+ for (const [recipe, { plain, rendered }] of byRecipe) {
2960
+ let pick;
2961
+ if (rendered === void 0) pick = plain;
2962
+ else if (plain === void 0 || rendered.fetched_at >= plain.fetched_at) pick = rendered;
2963
+ else {
2964
+ const base = this.stmt(`SELECT blob FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = 0 AND fetched_at <= ?
2965
+ ORDER BY fetched_at DESC, capture DESC LIMIT 1`).get(doc, recipe, rendered.fetched_at);
2966
+ pick = base !== void 0 && base.blob !== null && base.blob === plain.blob ? rendered : plain;
2967
+ }
2968
+ const newest = plain === void 0 ? rendered : rendered === void 0 || plain.fetched_at > rendered.fetched_at ? plain : rendered;
2969
+ const others = [plain, rendered].filter((row) => row !== void 0 && row !== pick);
2970
+ candidates.push({
2971
+ row: pick,
2972
+ ai: newest.ai_input,
2973
+ others
2974
+ });
2975
+ }
2976
+ let winner = candidates[0];
2977
+ if (candidates.length > 1) {
2978
+ const latest = candidates.every((candidate) => candidate.row.merge === "latest");
2979
+ for (const candidate of candidates.slice(1)) {
2980
+ const a = candidate.row;
2981
+ const b = winner.row;
2982
+ if (latest ? a.fetched_at > b.fetched_at : a.text_chars > b.text_chars || a.text_chars === b.text_chars && a.fetched_at > b.fetched_at) winner = candidate;
2983
+ }
2984
+ }
2985
+ const row = winner.row;
2986
+ const aiInput = row.ai_input === "no" || candidates.some((candidate) => candidate.ai === "no") ? "no" : winner.ai !== "unset" ? winner.ai : candidates.some((candidate) => candidate.ai === "yes") ? "yes" : "unset";
2987
+ let meta = parseJson(this.textOf(n, row.meta_sha), {});
2988
+ for (const candidate of candidates) if (candidate !== winner) meta = {
2989
+ ...parseJson(this.textOf(n, candidate.row.meta_sha), {}),
2990
+ ...meta
2991
+ };
2992
+ const same = this.stmt(`SELECT capture FROM ${n.versions} WHERE doc = ? AND text_sha = ?
2993
+ ORDER BY fetched_at DESC, capture DESC LIMIT ?`).all(doc, row.text_sha, 10);
2994
+ const captures = [.../* @__PURE__ */ new Set([
2995
+ row.capture,
2996
+ ...same.map((entry) => entry.capture),
2997
+ ...winner.others.map((other) => other.capture),
2998
+ ...candidates.filter((candidate) => candidate !== winner).map((candidate) => candidate.row.capture)
2999
+ ])].slice(0, 10);
3000
+ const count = this.stmt(`SELECT count(*) AS n FROM ${n.versions} WHERE doc = ?`).get(doc).n;
3001
+ const url = row.url ?? candidates.map((candidate) => candidate.row.url).find((value) => value !== null) ?? null;
3002
+ const saved = this.stmt(`INSERT INTO ${n.docs} (doc, kind, title, url, meta, text_sha, full_sha, captures, capture_count, capture,
3003
+ blob, fetched_at, recipe, rendered, ai_input, scope, labels, updated_at)
3004
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
3005
+ ON CONFLICT(doc) DO UPDATE SET kind = excluded.kind, title = excluded.title, url = excluded.url,
3006
+ meta = excluded.meta, text_sha = excluded.text_sha, full_sha = excluded.full_sha,
3007
+ captures = excluded.captures, capture_count = excluded.capture_count, capture = excluded.capture,
3008
+ blob = excluded.blob, fetched_at = excluded.fetched_at, recipe = excluded.recipe,
3009
+ rendered = excluded.rendered, ai_input = excluded.ai_input, scope = excluded.scope,
3010
+ labels = excluded.labels, updated_at = excluded.updated_at
3011
+ RETURNING rowid`).get(doc, row.kind, row.title, url, JSON.stringify(meta), row.text_sha, row.full_sha, JSON.stringify(captures), count, row.capture, row.blob, row.fetched_at, row.recipe, row.rendered, aiInput, row.scope, row.labels, now);
3012
+ const text = this.textOf(n, row.text_sha) ?? "";
3013
+ if (before === void 0 || before.text_sha !== row.text_sha || before.title !== row.title) {
3014
+ if (before !== void 0) this.stmt(`DELETE FROM ${n.fts} WHERE rowid = ?`).run(before.rowid);
3015
+ this.stmt(`INSERT INTO ${n.fts} (rowid, title, text) VALUES (?, ?, ?)`).run(saved.rowid, fold(row.title, "search"), fold(text.length > MAX_INDEX_CHARS ? text.slice(0, MAX_INDEX_CHARS) : text, "search"));
3016
+ }
3017
+ return this.docOf(n, doc, text);
3018
+ }
3019
+ docOf(n, doc, knownText) {
3020
+ const row = this.stmt(`SELECT rowid, * FROM ${n.docs} WHERE doc = ?`).get(doc);
3021
+ if (row === void 0) return void 0;
3022
+ return this.toDoc(n, row, knownText);
3023
+ }
3024
+ toDoc(n, row, knownText) {
3025
+ const labels = parseJson(row.labels, void 0);
3026
+ return {
3027
+ doc: row.doc,
3028
+ kind: row.kind,
3029
+ title: row.title,
3030
+ text: knownText ?? this.textOf(n, row.text_sha) ?? "",
3031
+ url: row.url,
3032
+ meta: parseJson(row.meta, {}),
3033
+ captures: parseJson(row.captures, []),
3034
+ capture_count: row.capture_count,
3035
+ capture: row.capture,
3036
+ blob: row.blob,
3037
+ fetched_at: row.fetched_at,
3038
+ recipe: row.recipe,
3039
+ rendered: row.rendered === 1,
3040
+ ai_input: row.ai_input,
3041
+ cut: row.full_sha !== null,
3042
+ ...row.scope !== null ? { scope: row.scope } : {},
3043
+ ...labels !== void 0 ? { labels } : {},
3044
+ updated_at: row.updated_at
3045
+ };
3046
+ }
3047
+ versionTextOf(n, row) {
3048
+ return {
3049
+ ...toVersion(row),
3050
+ text: this.textOf(n, row.text_sha) ?? "",
3051
+ full_text: this.textOf(n, row.full_sha),
3052
+ meta: parseJson(this.textOf(n, row.meta_sha), {})
3053
+ };
3054
+ }
3055
+ get(doc) {
3056
+ return this.live((n) => this.docOf(n, doc));
3057
+ }
3058
+ /**
3059
+ * This method adds one version of a document, and it gives the current document. The rules of
3060
+ * the class choose the current version. An upsert of a known capture replaces that version.
3061
+ */
3062
+ upsert(input, now) {
3063
+ return this.live((n) => this.write(() => {
3064
+ this.putVersion(n, input, now);
3065
+ return this.recompute(n, input.doc, now);
3066
+ }));
3067
+ }
3068
+ /** This method gives the versions of a document, the newest first. */
3069
+ versions(doc, options = {}) {
3070
+ const limit = Math.max(1, Math.floor(options.limit ?? 100));
3071
+ return this.live((n) => this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? ORDER BY fetched_at DESC, capture DESC LIMIT ?`).all(doc, limit).map(toVersion));
3072
+ }
3073
+ /**
3074
+ * This method gives the text of one version of a document. With no capture, it gives the current
3075
+ * version. It also gives the text of a held-back version: the caller applies `isHeldBack` to the
3076
+ * `ai_input` of the version.
3077
+ */
3078
+ versionText(doc, capture) {
3079
+ return this.live((n) => {
3080
+ const id = capture ?? this.docOf(n, doc)?.capture;
3081
+ if (id === void 0) return void 0;
3082
+ const row = this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND capture = ?`).get(doc, id);
3083
+ return row === void 0 ? void 0 : this.versionTextOf(n, row);
3084
+ });
3085
+ }
3086
+ /**
3087
+ * This method gives the texts of the earlier versions of a document, the newest first. Each
3088
+ * text comes one time, with the newest version that has it. It skips the current text. It also
3089
+ * gives held-back texts: the caller applies `isHeldBack` to the `ai_input` of each version.
3090
+ */
3091
+ earlierTexts(doc, options = {}) {
3092
+ const limit = Math.max(1, Math.floor(options.limit ?? 50));
3093
+ return this.live((n) => {
3094
+ const head = this.stmt(`SELECT text_sha FROM ${n.docs} WHERE doc = ?`).get(doc);
3095
+ if (head === void 0) return [];
3096
+ const rows = this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND text_sha != ? ORDER BY fetched_at DESC, capture DESC`).all(doc, head.text_sha);
3097
+ const seen = /* @__PURE__ */ new Set();
3098
+ const out = [];
3099
+ for (const row of rows) {
3100
+ if (seen.has(row.text_sha)) continue;
3101
+ seen.add(row.text_sha);
3102
+ out.push(this.versionTextOf(n, row));
3103
+ if (out.length >= limit) break;
3104
+ }
3105
+ return out;
3106
+ });
3107
+ }
3108
+ /** This method tells if a capture gave a version of a document. */
3109
+ hasVersion(capture) {
3110
+ return this.live((n) => this.stmt(`SELECT 1 AS found FROM ${n.versions} WHERE capture = ? LIMIT 1`).get(capture) !== void 0);
3111
+ }
3112
+ /** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */
3113
+ newestFetch() {
3114
+ return this.live((n) => this.stmt(`SELECT max(fetched_at) AS newest FROM ${n.latest}`).get()?.newest ?? void 0);
3115
+ }
3116
+ /** This method gives the current documents that a capture gave. */
3117
+ docsOfCapture(capture) {
3118
+ return this.live((n) => this.stmt(`SELECT DISTINCT doc FROM ${n.versions} WHERE capture = ?`).all(capture).map((row) => this.docOf(n, row.doc)).filter((doc) => doc !== void 0));
3119
+ }
3120
+ /**
3121
+ * This method searches the documents with BM25, the best first. The syntax of the query is the
3122
+ * syntax of `parseQuery`. It first asks for each part. When that gives too few documents, it
3123
+ * adds the documents that match one part. A document that asked not to be AI input is not in
3124
+ * the results, so a search cannot show its text.
3125
+ */
3126
+ find(q, options) {
3127
+ const query = parseQuery(q);
3128
+ if (query === void 0) return [];
3129
+ const limit = Math.max(1, Math.min(1e3, Math.floor(options.limit) || 10));
3130
+ return this.live((n) => {
3131
+ const kindClause = options.kind === void 0 || options.kind === null ? "" : "AND d.kind = ?";
3132
+ const sql = `SELECT d.rowid AS rowid, d.*, bm25(${n.fts}, 3.0, 1.0) AS rank
3133
+ FROM ${n.fts} JOIN ${n.docs} d ON d.rowid = ${n.fts}.rowid
3134
+ WHERE ${n.fts} MATCH ? AND d.ai_input != 'no' ${kindClause}
3135
+ ORDER BY rank LIMIT ?`;
3136
+ const run = (match) => {
3137
+ const params = [match];
3138
+ if (kindClause.length > 0) params.push(options.kind);
3139
+ params.push(limit);
3140
+ return this.stmt(sql).all(...params);
3141
+ };
3142
+ const rows = run(query.all);
3143
+ if (rows.length < limit && query.parts > 1) {
3144
+ const seen = new Set(rows.map((row) => row.doc));
3145
+ for (const row of run(query.any)) {
3146
+ if (rows.length >= limit) break;
3147
+ if (!seen.has(row.doc)) rows.push(row);
3148
+ }
3149
+ }
3150
+ return rows.map((row) => ({
3151
+ doc: row.doc,
3152
+ kind: row.kind,
3153
+ title: row.title,
3154
+ url: row.url,
3155
+ snippet: snippetOf(this.textOf(n, row.text_sha) ?? "", query.terms),
3156
+ score: Math.round(-row.rank * 1e3) / 1e3,
3157
+ ai_input: row.ai_input,
3158
+ fetched_at: row.fetched_at,
3159
+ capture: row.capture
3160
+ }));
3161
+ });
3162
+ }
3163
+ /**
3164
+ * This method scores a set of documents against a query, with the best first. It leaves out a
3165
+ * document that matches no term, and a document that asked not to be AI input. It ranks the
3166
+ * results of a source again, because the order of the source is only as good as its search engine.
3167
+ */
3168
+ rankWithin(q, docs) {
3169
+ const query = parseQuery(q);
3170
+ if (query === void 0 || docs.length === 0) return [];
3171
+ return this.live((n) => this.stmt(`SELECT d.doc AS doc, bm25(${n.fts}, 3.0, 1.0) AS rank
3172
+ FROM ${n.fts} JOIN ${n.docs} d ON d.rowid = ${n.fts}.rowid
3173
+ WHERE ${n.fts} MATCH ? AND d.ai_input != 'no' AND d.doc IN (SELECT value FROM json_each(?))
3174
+ ORDER BY rank`).all(query.any, JSON.stringify(docs)).map((row) => ({
3175
+ doc: row.doc,
3176
+ score: Math.round(-row.rank * 1e3) / 1e3
3177
+ })));
3178
+ }
3179
+ /** This method gives the cache entry of a URL. The key is canonical, so two spellings of one URL give one entry. */
3180
+ latest(url, variant) {
3181
+ const key = keyOf(url, variant);
3182
+ return this.live((n) => {
3183
+ const row = this.stmt(`SELECT * FROM ${n.latest} WHERE url = ? AND variant = ?`).get(key.url, key.variant);
3184
+ return row === void 0 ? void 0 : toLatest(row);
3185
+ });
3186
+ }
3187
+ putLatest(n, entry) {
3188
+ const key = keyOf(entry.url, entry.variant);
3189
+ this.stmt(`INSERT INTO ${n.latest} (url, variant, capture, blob, etag, last_modified, content_type, fetched_at,
3190
+ status, ai_input, location, truncated, fresh_until, vary_key)
3191
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
3192
+ ON CONFLICT(url, variant) DO UPDATE SET capture = excluded.capture, blob = excluded.blob,
3193
+ etag = excluded.etag, last_modified = excluded.last_modified, content_type = excluded.content_type,
3194
+ fetched_at = excluded.fetched_at, status = excluded.status, ai_input = excluded.ai_input,
3195
+ location = excluded.location, truncated = excluded.truncated, fresh_until = excluded.fresh_until,
3196
+ vary_key = excluded.vary_key
3197
+ WHERE excluded.fetched_at >= ${n.latest}.fetched_at`).run(key.url, key.variant, entry.capture, entry.blob, entry.etag, entry.last_modified, entry.content_type, entry.fetched_at, entry.status, entry.ai_input, entry.location ?? null, entry.truncated === true ? 1 : null, entry.fresh_until ?? null, entry.vary_key ?? null);
3198
+ }
3199
+ /** This method keeps the last answer for a URL. An entry that is older than the stored entry changes nothing. */
3200
+ setLatest(entry) {
3201
+ this.live((n) => this.write(() => this.putLatest(n, entry)));
3202
+ }
3203
+ counts() {
3204
+ return this.live((n) => ({
3205
+ docs: this.stmt(`SELECT count(*) AS n FROM ${n.docs}`).get().n,
3206
+ urls: this.stmt(`SELECT count(*) AS n FROM ${n.latest}`).get().n
3207
+ }));
3208
+ }
3209
+ /** This method empties the current generation. A rebuild uses `beginRebuild`, which keeps the index until its commit. */
3210
+ reset() {
3211
+ this.live((n) => this.write(() => {
3212
+ for (const table of [
3213
+ n.fts,
3214
+ n.docs,
3215
+ n.versions,
3216
+ n.texts,
3217
+ n.latest
3218
+ ]) this.db.exec(`DELETE FROM ${table}`);
3219
+ }));
3220
+ }
3221
+ /**
3222
+ * This method starts a rebuild into a new generation of the tables. The current generation
3223
+ * answers each read and takes each write until the commit. A failed rebuild thus leaves the
3224
+ * index as it was.
3225
+ */
3226
+ beginRebuild(options = {}) {
3227
+ const batchSize = Math.max(1, Math.floor(options.batch ?? 500));
3228
+ const generation = this.write(() => {
3229
+ const next = Number(this.getMeta("next_generation") ?? "2");
3230
+ this.setMeta("next_generation", String(next + 1));
3231
+ this.setMeta(`building:${String(next)}`, (/* @__PURE__ */ new Date()).toISOString());
3232
+ this.db.exec(ddlOf(namesOf(next)));
3233
+ return next;
3234
+ });
3235
+ const n = namesOf(generation);
3236
+ const versions = [];
3237
+ const latest = /* @__PURE__ */ new Map();
3238
+ let open = true;
3239
+ const keyText = (url, variant) => {
3240
+ const key = keyOf(url, variant);
3241
+ return `${key.variant} ${key.url}`;
3242
+ };
3243
+ const flush = () => {
3244
+ if (!open) throw new Error("the rebuild is over");
3245
+ if (versions.length === 0 && latest.size === 0) return;
3246
+ this.write(() => {
3247
+ const touched = /* @__PURE__ */ new Map();
3248
+ for (const { input, now } of versions) {
3249
+ this.putVersion(n, input, now);
3250
+ touched.set(input.doc, now);
3251
+ }
3252
+ for (const entry of latest.values()) this.putLatest(n, entry);
3253
+ for (const [doc, now] of touched) this.recompute(n, doc, now);
3254
+ });
3255
+ versions.length = 0;
3256
+ latest.clear();
3257
+ };
3258
+ /** This function copies the texts of the versions of the current generation that agree with `where`. */
3259
+ const copyTexts = (current, where, params) => {
3260
+ for (const column of [
3261
+ "text_sha",
3262
+ "meta_sha",
3263
+ "full_sha"
3264
+ ]) this.db.prepare(`INSERT OR IGNORE INTO ${n.texts} (sha, body)
3265
+ SELECT t.sha, t.body FROM ${current.texts} t
3266
+ WHERE t.sha IN (SELECT ${column} FROM ${current.versions} WHERE ${where})`).run(...params);
3267
+ };
3268
+ return {
3269
+ upsert: (input, now) => {
3270
+ if (!open) throw new Error("the rebuild is over");
3271
+ versions.push({
3272
+ input,
3273
+ now
3274
+ });
3275
+ if (versions.length >= batchSize) flush();
3276
+ },
3277
+ setLatest: (entry) => {
3278
+ if (!open) throw new Error("the rebuild is over");
3279
+ const key = keyText(entry.url, entry.variant);
3280
+ const before = latest.get(key);
3281
+ if (before === void 0 || entry.fetched_at >= before.fetched_at) latest.set(key, entry);
3282
+ if (latest.size >= batchSize) flush();
3283
+ },
3284
+ latest: (url, variant) => {
3285
+ const waiting = latest.get(keyText(url, variant));
3286
+ if (waiting !== void 0) return waiting;
3287
+ const key = keyOf(url, variant);
3288
+ const row = this.stmt(`SELECT * FROM ${n.latest} WHERE url = ? AND variant = ?`).get(key.url, key.variant);
3289
+ return row === void 0 ? void 0 : toLatest(row);
3290
+ },
3291
+ flush,
3292
+ carry: (recipes) => {
3293
+ flush();
3294
+ const known = JSON.stringify([...recipes]);
3295
+ const current = namesOf(this.generation());
3296
+ const docs = this.db.prepare(`SELECT DISTINCT doc FROM ${current.versions} WHERE recipe NOT IN (SELECT value FROM json_each(?))`).all(known);
3297
+ const where = "recipe NOT IN (SELECT value FROM json_each(?)) AND doc IN (SELECT value FROM json_each(?))";
3298
+ for (let at = 0; at < docs.length; at += batchSize) {
3299
+ const part = JSON.stringify(docs.slice(at, at + batchSize).map((row) => row.doc));
3300
+ this.write(() => {
3301
+ copyTexts(current, where, [known, part]);
3302
+ this.db.prepare(`INSERT OR IGNORE INTO ${n.versions} SELECT * FROM ${current.versions} WHERE ${where}`).run(known, part);
3303
+ const now = (/* @__PURE__ */ new Date()).toISOString();
3304
+ for (const doc of JSON.parse(part)) this.recompute(n, doc, now);
3305
+ });
3306
+ }
3307
+ return docs.length;
3308
+ },
3309
+ commit: (since) => {
3310
+ flush();
3311
+ open = false;
3312
+ return this.write(() => {
3313
+ const current = this.generation();
3314
+ const old = namesOf(current);
3315
+ copyTexts(old, "fetched_at >= ?", [since]);
3316
+ const added = this.db.prepare(`SELECT DISTINCT doc FROM ${old.versions} WHERE fetched_at >= ?`).all(since);
3317
+ this.db.prepare(`INSERT OR IGNORE INTO ${n.versions} SELECT * FROM ${old.versions} WHERE fetched_at >= ?`).run(since);
3318
+ const now = (/* @__PURE__ */ new Date()).toISOString();
3319
+ for (const { doc } of added) this.recompute(n, doc, now);
3320
+ for (const row of this.db.prepare(`SELECT * FROM ${old.latest} WHERE fetched_at >= ?`).all(since)) this.putLatest(n, toLatest(row));
3321
+ this.setMeta("generation", String(generation));
3322
+ this.db.prepare("DELETE FROM index_meta WHERE key = ?").run(`building:${String(generation)}`);
3323
+ this.dropGeneration(current);
3324
+ this.empty = false;
3325
+ return {
3326
+ docs: this.db.prepare(`SELECT count(*) AS n FROM ${n.docs}`).get().n,
3327
+ urls: this.db.prepare(`SELECT count(*) AS n FROM ${n.latest}`).get().n
3328
+ };
3329
+ });
3330
+ },
3331
+ abort: () => {
3332
+ open = false;
3333
+ versions.length = 0;
3334
+ latest.clear();
3335
+ if (this.closed) return;
3336
+ this.write(() => this.dropGeneration(generation));
3337
+ }
3338
+ };
3339
+ }
3340
+ /** This method closes the file. A second close does nothing. */
3341
+ close() {
3342
+ if (this.closed) return;
3343
+ this.closed = true;
3344
+ this.statements.clear();
3345
+ this.db.close();
3346
+ }
3347
+ };
3348
+ //#endregion
3349
+ //#region src/hoststate.ts
3350
+ /** A stop with no end goes into the table as this value, which is past each date that JavaScript can hold. */
3351
+ const NO_END = 9e15;
3352
+ /** A process that shares the store asks again after this wait, at most, during the lease of a different process. */
3353
+ const SHARED_WAIT_MS = 1e3;
3354
+ /** The store removes the rows of idle hosts after this many reservations. */
3355
+ const SWEEP_EVERY = 1e3;
3356
+ /**
3357
+ * This store keeps the state of each host in a SQLite file. Two or more processes on one store
3358
+ * share it, so they share the pace and the stops of each host. Thus a host gets one request at a
3359
+ * time from all of them. Each method is one transaction (`BEGIN IMMEDIATE`), and it waits for the lock
3360
+ * of a different process (`busy_timeout`).
3361
+ *
3362
+ * The lease of a request belongs to the store that gave it. A different process that asks during
3363
+ * the lease gets a wait of one second or less, and asks again. A process that stops during a
3364
+ * request holds its lease until the lease ends (5 minutes by default).
3365
+ *
3366
+ * The state is not a projection of the capture log. A deletion of the file loses only the pace,
3367
+ * the counts and the stops, and the crawler starts them again.
3368
+ */
3369
+ var SqliteHostStateStore = class {
3370
+ path;
3371
+ db;
3372
+ statements = /* @__PURE__ */ new Map();
3373
+ /** This id names the leases of this store. */
3374
+ owner = randomUUID();
3375
+ reservations = 0;
3376
+ closed = false;
3377
+ pollMs;
3378
+ constructor(path, options = {}) {
3379
+ this.path = path;
3380
+ if (path !== ":memory:") mkdirSync(dirname(path), {
3381
+ recursive: true,
3382
+ mode: 448
3383
+ });
3384
+ this.pollMs = Math.max(1, Math.min(SHARED_WAIT_MS, options.pollMs ?? SHARED_WAIT_MS));
3385
+ this.db = new DatabaseSync(path);
3386
+ const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 1e4));
3387
+ this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);
3388
+ if (path !== ":memory:") this.db.exec("PRAGMA journal_mode = WAL;");
3389
+ this.db.exec("PRAGMA synchronous = NORMAL;");
3390
+ this.db.exec(`CREATE TABLE IF NOT EXISTS hosts (
3391
+ host TEXT PRIMARY KEY,
3392
+ next_at REAL NOT NULL DEFAULT 0,
3393
+ leased_until REAL NOT NULL DEFAULT 0,
3394
+ owner TEXT,
3395
+ failures INTEGER NOT NULL DEFAULT 0,
3396
+ waits INTEGER NOT NULL DEFAULT 0,
3397
+ stop_until REAL,
3398
+ stop_reason TEXT
3399
+ )`);
3400
+ }
3401
+ stmt(sql) {
3402
+ let statement = this.statements.get(sql);
3403
+ if (statement === void 0) {
3404
+ statement = this.db.prepare(sql);
3405
+ this.statements.set(sql, statement);
3406
+ }
3407
+ return statement;
3408
+ }
3409
+ /** This method does one step in one transaction. */
3410
+ step(work) {
3411
+ if (this.closed) throw new Error("the host state store is closed");
3412
+ this.db.exec("BEGIN IMMEDIATE");
3413
+ try {
3414
+ const result = work();
3415
+ this.db.exec("COMMIT");
3416
+ return result;
3417
+ } catch (error) {
3418
+ if (this.db.isTransaction) this.db.exec("ROLLBACK");
3419
+ throw error;
3420
+ }
3421
+ }
3422
+ row(host) {
3423
+ this.stmt("INSERT OR IGNORE INTO hosts (host) VALUES (?)").run(host);
3424
+ return this.stmt("SELECT next_at, leased_until, owner, failures, waits, stop_until, stop_reason FROM hosts WHERE host = ?").get(host);
3425
+ }
3426
+ /** This method removes the hosts that have nothing to remember: no rest, no lease, no counts and no stop. */
3427
+ sweep(now) {
3428
+ this.stmt(`DELETE FROM hosts WHERE next_at <= ? AND leased_until <= ? AND failures = 0 AND waits = 0
3429
+ AND (stop_until IS NULL OR stop_until <= ?)`).run(now, now, now);
3430
+ }
3431
+ reserve(host, now, leaseMs) {
3432
+ return this.step(() => {
3433
+ this.reservations += 1;
3434
+ if (this.reservations % SWEEP_EVERY === 0) this.sweep(now);
3435
+ const row = this.row(host);
3436
+ if (row.leased_until > now) return row.owner === this.owner ? row.leased_until - now : Math.min(this.pollMs, row.leased_until - now);
3437
+ if (row.next_at > now) return row.next_at - now;
3438
+ this.stmt("UPDATE hosts SET leased_until = ?, owner = ? WHERE host = ?").run(now + leaseMs, this.owner, host);
3439
+ return 0;
3440
+ });
3441
+ }
3442
+ record(host, result) {
3443
+ return this.step(() => {
3444
+ const row = this.row(host);
3445
+ let { failures, waits } = row;
3446
+ if (result === "success") {
3447
+ failures = 0;
3448
+ waits = 0;
3449
+ } else if (result === "failure") failures += 1;
3450
+ else if (result === "wait") waits += 1;
3451
+ if (failures !== row.failures || waits !== row.waits) this.stmt("UPDATE hosts SET failures = ?, waits = ? WHERE host = ?").run(failures, waits, host);
3452
+ return {
3453
+ failures,
3454
+ waits
3455
+ };
3456
+ });
3457
+ }
3458
+ release(host, nextAt) {
3459
+ this.step(() => {
3460
+ this.row(host);
3461
+ this.stmt(`UPDATE hosts SET next_at = max(next_at, ?),
3462
+ leased_until = CASE WHEN owner = ? THEN 0 ELSE leased_until END,
3463
+ owner = CASE WHEN owner = ? THEN NULL ELSE owner END
3464
+ WHERE host = ?`).run(nextAt, this.owner, this.owner, host);
3465
+ });
3466
+ }
3467
+ defer(host, until) {
3468
+ this.step(() => {
3469
+ this.row(host);
3470
+ this.stmt("UPDATE hosts SET next_at = max(next_at, ?) WHERE host = ?").run(until, host);
3471
+ });
3472
+ }
3473
+ stop(host, until, reason) {
3474
+ const end = Number.isFinite(until) ? until : NO_END;
3475
+ this.step(() => {
3476
+ this.row(host);
3477
+ this.stmt(`UPDATE hosts SET stop_until = ?, stop_reason = ?
3478
+ WHERE host = ? AND (stop_until IS NULL OR stop_until < ?)`).run(end, reason, host, end);
3479
+ });
3480
+ }
3481
+ stopped(host, now) {
3482
+ if (this.closed) throw new Error("the host state store is closed");
3483
+ const row = this.stmt("SELECT stop_until, stop_reason FROM hosts WHERE host = ?").get(host);
3484
+ if (row === void 0 || row.stop_until === null || row.stop_until <= now) return void 0;
3485
+ return {
3486
+ until: row.stop_until >= NO_END ? Number.POSITIVE_INFINITY : row.stop_until,
3487
+ reason: row.stop_reason ?? "stopped"
3488
+ };
3489
+ }
3490
+ reset(host) {
3491
+ this.step(() => {
3492
+ if (host === void 0) this.stmt("UPDATE hosts SET failures = 0, waits = 0, stop_until = NULL, stop_reason = NULL").run();
3493
+ else this.stmt("UPDATE hosts SET failures = 0, waits = 0, stop_until = NULL, stop_reason = NULL WHERE host = ?").run(host);
3494
+ });
3495
+ }
3496
+ /** This value is the number of hosts that the store holds. */
3497
+ get size() {
3498
+ return this.stmt("SELECT count(*) AS n FROM hosts").get().n;
3499
+ }
3500
+ /** This method closes the file. A second close does nothing. */
3501
+ close() {
3502
+ if (this.closed) return;
3503
+ this.closed = true;
3504
+ this.statements.clear();
3505
+ this.db.close();
3506
+ }
3507
+ };
3508
+ //#endregion
3509
+ //#region src/log.ts
3510
+ const MONTH_FILE = /^(\d{2})(?:\.[^.]+)?\.ndjson$/;
3511
+ const YEAR_DIR = /^\d{4}$/;
3512
+ const NEWLINE = 10;
3513
+ /** This function writes the whole buffer. A write to a file can write less than the buffer, and the loop writes the rest. */
3514
+ const writeAll = async (handle, bytes) => {
3515
+ let at = 0;
3516
+ while (at < bytes.length) {
3517
+ const { bytesWritten } = await handle.write(bytes, at, bytes.length - at);
3518
+ at += bytesWritten;
3519
+ }
3520
+ };
3521
+ /**
3522
+ * This function reads a file line by line, as bytes, so a long file does not become one string. It
3523
+ * gives each line and tells if a line break ends it.
3524
+ */
3525
+ async function* linesOf(path) {
3526
+ const decoder = new TextDecoder("utf-8");
3527
+ let rest;
3528
+ for await (const chunk of createReadStream(path)) {
3529
+ const data = rest === void 0 ? chunk : Buffer.concat([rest, chunk]);
3530
+ rest = void 0;
3531
+ let start = 0;
3532
+ for (;;) {
3533
+ const end = data.indexOf(NEWLINE, start);
3534
+ if (end === -1) break;
3535
+ yield {
3536
+ text: decoder.decode(data.subarray(start, end)),
3537
+ complete: true
3538
+ };
3539
+ start = end + 1;
3540
+ }
3541
+ if (start < data.length) rest = Buffer.from(data.subarray(start));
3542
+ }
3543
+ if (rest !== void 0 && rest.length > 0) yield {
3544
+ text: decoder.decode(rest),
3545
+ complete: false
3546
+ };
3547
+ }
3548
+ const monthKey = (year, file) => `${year}-${MONTH_FILE.exec(file)?.[1] ?? "00"}`;
3549
+ /**
3550
+ * The capture log keeps each fetch decision as one line of JSON in
3551
+ * `<dir>/<source>/<yyyy>/<mm>.ndjson`. It is the record of each access that the crawler makes.
3552
+ *
3553
+ * These are the rules of the log:
3554
+ *
3555
+ * - The log appends a batch of records with one write, and it writes the batch to the disk before
3556
+ * the appends finish.
3557
+ * - An append always starts on a new line. After a crash leaves a partial line, the next append
3558
+ * starts after a line break. The partial line stays as a damaged line, and the record is good.
3559
+ * - A scan reads each file as a stream. It skips a damaged line, counts it, and continues.
3560
+ * - The log does not remove or change a line.
3561
+ *
3562
+ * Two processes can append to one log, because each batch is one write to a file in append mode.
3563
+ */
3564
+ var CaptureLog = class CaptureLog {
3565
+ dir;
3566
+ handles = /* @__PURE__ */ new Map();
3567
+ pending = [];
3568
+ draining;
3569
+ sync;
3570
+ constructor(dir, options = {}) {
3571
+ this.dir = dir;
3572
+ this.sync = options.sync ?? true;
3573
+ }
3574
+ pathOf(record) {
3575
+ return join(this.dir, record.source, record.received_at.slice(0, 4), `${record.received_at.slice(5, 7)}.ndjson`);
3576
+ }
3577
+ async handleFor(path) {
3578
+ const known = this.handles.get(path);
3579
+ if (known !== void 0) return known;
3580
+ await mkdir(dirname(path), {
3581
+ recursive: true,
3582
+ mode: 448
3583
+ });
3584
+ const handle = await open(path, "a+", 384);
3585
+ this.handles.set(path, handle);
3586
+ return handle;
3587
+ }
3588
+ /** This method tells if the last byte of a file is not a line break: a partial line that a crash left. */
3589
+ static async needsBreak(handle) {
3590
+ const { size } = await handle.stat();
3591
+ if (size === 0) return false;
3592
+ const last = Buffer.alloc(1);
3593
+ await handle.read(last, 0, 1, size - 1);
3594
+ return last[0] !== NEWLINE;
3595
+ }
3596
+ append(record) {
3597
+ let line;
3598
+ try {
3599
+ line = `${JSON.stringify(record)}\n`;
3600
+ } catch (error) {
3601
+ return Promise.reject(error instanceof Error ? error : new Error(String(error)));
3602
+ }
3603
+ return new Promise((resolve, reject) => {
3604
+ this.pending.push({
3605
+ path: this.pathOf(record),
3606
+ line,
3607
+ resolve,
3608
+ reject
3609
+ });
3610
+ this.kick();
3611
+ });
3612
+ }
3613
+ /** This method starts the writer when it is not in progress. A record that comes as the writer stops starts it again. */
3614
+ kick() {
3615
+ if (this.draining !== void 0) return;
3616
+ this.draining = this.drain().finally(() => {
3617
+ this.draining = void 0;
3618
+ if (this.pending.length > 0) this.kick();
3619
+ });
3620
+ }
3621
+ /** This method writes the waiting records in batches: one write and one sync for each file. */
3622
+ async drain() {
3623
+ while (this.pending.length > 0) {
3624
+ const batch = this.pending.splice(0);
3625
+ const byPath = /* @__PURE__ */ new Map();
3626
+ for (const item of batch) {
3627
+ const list = byPath.get(item.path);
3628
+ if (list === void 0) byPath.set(item.path, [item]);
3629
+ else list.push(item);
3630
+ }
3631
+ for (const [path, items] of byPath) try {
3632
+ const handle = await this.handleFor(path);
3633
+ const lead = await CaptureLog.needsBreak(handle) ? "\n" : "";
3634
+ await writeAll(handle, Buffer.from(`${lead}${items.map((item) => item.line).join("")}`, "utf8"));
3635
+ if (this.sync) await handle.datasync();
3636
+ for (const item of items) item.resolve();
3637
+ } catch (error) {
3638
+ const handle = this.handles.get(path);
3639
+ this.handles.delete(path);
3640
+ await handle?.close().catch(() => void 0);
3641
+ for (const item of items) item.reject(error);
3642
+ }
3643
+ }
3644
+ }
3645
+ /** This method waits until each append in progress is on the disk. */
3646
+ async flush() {
3647
+ while (this.draining !== void 0) await this.draining;
3648
+ }
3649
+ /** This method gives the record files of one source, in the sequence of time. It skips the months before `since`. */
3650
+ async filesOf(source, since) {
3651
+ const sourceDir = join(this.dir, source);
3652
+ if (!existsSync(sourceDir)) return [];
3653
+ const from = since?.slice(0, 7);
3654
+ const years = (await readdir(sourceDir, { withFileTypes: true })).filter((entry) => entry.isDirectory() && YEAR_DIR.test(entry.name)).map((entry) => entry.name).sort();
3655
+ const files = [];
3656
+ for (const year of years) {
3657
+ if (from !== void 0 && year < from.slice(0, 4)) continue;
3658
+ const months = (await readdir(join(sourceDir, year))).filter((name) => MONTH_FILE.test(name)).sort();
3659
+ for (const month of months) {
3660
+ if (from !== void 0 && monthKey(year, month) < from) continue;
3661
+ files.push(join(sourceDir, year, month));
3662
+ }
3663
+ }
3664
+ return files;
3665
+ }
3666
+ /** This method reads the records of one source in the sequence of the files. */
3667
+ async *readSource(source, filter, options) {
3668
+ for (const path of await this.filesOf(source, filter?.since)) {
3669
+ let number = 0;
3670
+ for await (const { text, complete } of linesOf(path)) {
3671
+ number += 1;
3672
+ const trimmed = text.trim();
3673
+ if (trimmed.length === 0) continue;
3674
+ let row;
3675
+ try {
3676
+ row = JSON.parse(trimmed);
3677
+ if (typeof row !== "object" || row === null || typeof row.received_at !== "string") throw new Error("not a record");
3678
+ } catch {
3679
+ if (complete) options?.onDamaged?.({
3680
+ path,
3681
+ line: number
3682
+ });
3683
+ continue;
3684
+ }
3685
+ if (filter?.kind !== void 0 && row.kind !== filter.kind) continue;
3686
+ if (filter?.since !== void 0 && row.received_at < filter.since) continue;
3687
+ yield row;
3688
+ }
3689
+ }
3690
+ }
3691
+ /**
3692
+ * This method reads the records that agree with the filter. It first waits for the appends that
3693
+ * are in progress. It reads each file as a stream, so its memory does not grow with the log.
3694
+ *
3695
+ * The records of one source come in the sequence of the files. The records of two or more sources
3696
+ * merge in the sequence of `received_at`.
3697
+ */
3698
+ async *scan(filter, options) {
3699
+ await this.flush();
3700
+ const sources = filter?.source !== void 0 ? [filter.source] : existsSync(this.dir) ? (await readdir(this.dir, { withFileTypes: true })).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort() : [];
3701
+ if (sources.length === 1) {
3702
+ yield* this.readSource(sources[0], filter, options);
3703
+ return;
3704
+ }
3705
+ const streams = sources.map((source) => this.readSource(source, filter, options)[Symbol.asyncIterator]());
3706
+ const heads = [];
3707
+ for (const stream of streams) heads.push((await stream.next()).value);
3708
+ for (;;) {
3709
+ let pick = -1;
3710
+ for (const [index, head] of heads.entries()) if (head !== void 0 && (pick === -1 || head.received_at < heads[pick].received_at)) pick = index;
3711
+ if (pick === -1) return;
3712
+ yield heads[pick];
3713
+ heads[pick] = (await streams[pick].next()).value;
3714
+ }
3715
+ }
3716
+ /** This method writes the appends in progress, then closes the files. A later append opens them again. */
3717
+ async close() {
3718
+ await this.flush();
3719
+ const handles = [...this.handles.values()];
3720
+ this.handles.clear();
3721
+ for (const handle of handles) await handle.close();
3722
+ }
3723
+ };
3724
+ //#endregion
3725
+ //#region src/pdf.ts
3726
+ const MAX_PAGES = 500;
3727
+ const MAX_CHARS = 2e5;
3728
+ const TIME_BUDGET_MS = 2e4;
3729
+ /**
3730
+ * The text streams of 35 stored menus and papers inflate to 0.6 MB or less. The parser of pdf.js
3731
+ * reads a page with no pause. Thus a page of 8 MB of text operators stops the process for
3732
+ * approximately one second.
3733
+ */
3734
+ const MAX_INFLATED_BYTES = 8388608;
3735
+ /** This function finds a PDF from its first bytes or its content type. Some servers put bytes before `%PDF-`. */
3736
+ const isPdf = (body, contentType) => {
3737
+ if (/\bpdf\b/i.test(contentType ?? "")) return true;
3738
+ const end = Math.min(body.length - 4, 1024);
3739
+ for (let index = 0; index < end; index += 1) if (body[index] === 37 && body[index + 1] === 80 && body[index + 2] === 68 && body[index + 3] === 70 && body[index + 4] === 45) return true;
3740
+ return false;
3741
+ };
3742
+ /** This function gives one line of text for each line of the PDF. It removes the empty lines. */
3743
+ const tidy = (page) => page.split(/\r?\n/).map((line) => line.replace(/\s+/g, " ").trim()).filter((line) => line.length > 0).join("\n");
3744
+ /** A word processor writes the name of its file as the title ("Microsoft Word - menu-final-v3.docx"). */
3745
+ const JUNK_TITLE = /^(?:microsoft (?:word|powerpoint|excel) - |untitled\b|document\d*$|presentation\d*$)|\.(?:docx?|pdf|pptx?|xlsx?|indd|ai|psd|rtf|odt|pages|txt)$/i;
3746
+ const latin1 = (bytes, start, end) => {
3747
+ let out = "";
3748
+ for (let index = Math.max(0, start); index < end && index < bytes.length; index += 1) out += String.fromCharCode(bytes[index]);
3749
+ return out;
3750
+ };
3751
+ const indexOf = (bytes, needle, from) => {
3752
+ const first = needle.charCodeAt(0);
3753
+ for (let index = bytes.indexOf(first, from); index !== -1; index = bytes.indexOf(first, index + 1)) {
3754
+ let match = true;
3755
+ for (let at = 1; at < needle.length; at += 1) if (bytes[index + at] !== needle.charCodeAt(at)) {
3756
+ match = false;
3757
+ break;
3758
+ }
3759
+ if (match) return index;
3760
+ }
3761
+ return -1;
3762
+ };
3763
+ /**
3764
+ * This function adds up the bytes that the compressed streams of a file inflate to, and stops at
3765
+ * the limit. It tries each stream: a stream in another format fails at its first bytes. An image
3766
+ * stream does not count, because the text reader does not decode it. The function gives true when
3767
+ * the file stays inside the limit.
3768
+ */
3769
+ const inflatesWithin = (bytes, limit) => {
3770
+ let total = 0;
3771
+ let from = 0;
3772
+ for (;;) {
3773
+ const keyword = indexOf(bytes, "stream", from);
3774
+ if (keyword === -1) return true;
3775
+ from = keyword + 6;
3776
+ if (latin1(bytes, keyword - 3, keyword) === "end") continue;
3777
+ const next = bytes[keyword + 6];
3778
+ if (next !== 10 && next !== 13) continue;
3779
+ const dictionary = latin1(bytes, keyword - 1024, keyword);
3780
+ const head = dictionary.slice(Math.max(0, dictionary.lastIndexOf("<<")));
3781
+ if (/\/Subtype\s*\/Image\b/.test(head)) continue;
3782
+ let start = keyword + 7;
3783
+ if (next === 13 && bytes[start] === 10) start += 1;
3784
+ const declared = /\/Length\s+(\d+)(?!\s+\d+\s+R)/.exec(head);
3785
+ const stop = indexOf(bytes, "endstream", start);
3786
+ const end = declared !== null && start + Number(declared[1]) <= bytes.length ? start + Number(declared[1]) : stop === -1 ? bytes.length : stop;
3787
+ try {
3788
+ total += inflateSync(bytes.subarray(start, end), {
3789
+ maxOutputLength: Math.max(1, limit - total + 1),
3790
+ finishFlush: constants.Z_SYNC_FLUSH
3791
+ }).length;
3792
+ } catch (error) {
3793
+ if (error instanceof RangeError) return false;
3794
+ }
3795
+ if (total > limit) return false;
3796
+ from = Math.max(from, end);
3797
+ }
3798
+ };
3799
+ /** This function lets the other work of the process run, for example another request. */
3800
+ const pause = () => new Promise((resolve) => setImmediate(resolve));
3801
+ /** This function gives the value of a promise, or undefined when the time ends first. */
3802
+ const within = async (promise, ms) => {
3803
+ let timer;
3804
+ const late = new Promise((resolve) => {
3805
+ timer = setTimeout(() => resolve(void 0), Math.max(0, ms));
3806
+ });
3807
+ try {
3808
+ return await Promise.race([promise, late]);
3809
+ } finally {
3810
+ clearTimeout(timer);
3811
+ }
3812
+ };
3813
+ /** This function keeps the first characters of a text, and does not cut a surrogate pair. */
3814
+ const cutText = (text, length) => {
3815
+ let end = Math.max(0, length);
3816
+ const code = text.charCodeAt(end);
3817
+ if (end > 0 && end < text.length && code >= 56320 && code <= 57343) end -= 1;
3818
+ return text.slice(0, end).trimEnd();
3819
+ };
3820
+ const readPdf = async (bytes, options = {}) => {
3821
+ const maxPages = options.maxPages ?? MAX_PAGES;
3822
+ const maxChars = options.maxChars ?? MAX_CHARS;
3823
+ const budgetMs = options.timeBudgetMs ?? TIME_BUDGET_MS;
3824
+ const maxInflated = options.maxInflatedBytes ?? MAX_INFLATED_BYTES;
3825
+ if (!inflatesWithin(bytes, maxInflated)) return {
3826
+ title: null,
3827
+ text: "",
3828
+ pages: 0,
3829
+ truncated: true,
3830
+ note: `the streams of the file inflate past ${String(Math.round(maxInflated / 1024 / 1024))} MB`
3831
+ };
3832
+ const pdf = await getDocumentProxy(new Uint8Array(bytes));
3833
+ try {
3834
+ const started = Date.now();
3835
+ const texts = [];
3836
+ let chars = 0;
3837
+ let note = null;
3838
+ const last = Math.min(pdf.numPages, maxPages);
3839
+ if (pdf.numPages > maxPages) note = `the reader stops after ${String(maxPages)} pages`;
3840
+ const late = (number) => `the reader stops after ${String(Math.round(budgetMs / 1e3))} seconds, at page ${String(number - 1)}`;
3841
+ for (let number = 1; number <= last; number += 1) {
3842
+ if (number > 1) {
3843
+ await pause();
3844
+ if (Date.now() - started > budgetMs) {
3845
+ note = late(number);
3846
+ break;
3847
+ }
3848
+ }
3849
+ const page = await pdf.getPage(number);
3850
+ const content = await within(page.getTextContent(), budgetMs - (Date.now() - started));
3851
+ if (content === void 0) {
3852
+ note = `the reader stops after ${String(Math.round(budgetMs / 1e3))} seconds, in page ${String(number)}`;
3853
+ break;
3854
+ }
3855
+ const text = tidy(content.items.filter((item) => item.str !== void 0).map((item) => `${item.str ?? ""}${item.hasEOL === true ? "\n" : ""}`).join(""));
3856
+ page.cleanup();
3857
+ if (text.length > 0) {
3858
+ if (text.length > maxChars - chars) {
3859
+ texts.push(cutText(text, maxChars - chars));
3860
+ note = `the reader stops at ${String(maxChars)} characters, at page ${String(number)}`;
3861
+ break;
3862
+ }
3863
+ texts.push(text);
3864
+ chars += text.length + 2;
3865
+ }
3866
+ if (chars >= maxChars && number < pdf.numPages) {
3867
+ note = `the reader stops at ${String(maxChars)} characters, at page ${String(number)}`;
3868
+ break;
3869
+ }
3870
+ }
3871
+ const named = ((await getMeta(pdf).catch(() => void 0))?.info)?.["Title"];
3872
+ const title = typeof named === "string" ? named.replace(/\s+/g, " ").trim() : "";
3873
+ return {
3874
+ title: title.length > 0 && !JUNK_TITLE.test(title) ? title : null,
3875
+ text: texts.join("\n\n"),
3876
+ pages: pdf.numPages,
3877
+ truncated: note !== null,
3878
+ note
3879
+ };
3880
+ } finally {
3881
+ await pdf.loadingTask.destroy();
3882
+ }
3883
+ };
3884
+ //#endregion
3885
+ //#region src/runtime.ts
3886
+ /** This function gives the document id of a web page. Each page has one id. */
3887
+ const pageDocId = (url) => `url:${canonicalUrl(url)}`;
3888
+ /** The JSON-LD that a document keeps, in characters of JSON. */
3889
+ const MAX_JSONLD_CHARS = 3e4;
3890
+ /** These properties make a JSON-LD record large, and they say little about the place: they go first when a record is too large. */
3891
+ const BULKY = [
3892
+ "review",
3893
+ "reviews",
3894
+ "comment",
3895
+ "hasMenu",
3896
+ "hasMenuSection",
3897
+ "hasMenuItem",
3898
+ "menu",
3899
+ "photo",
3900
+ "image",
3901
+ "subjectOf",
3902
+ "makesOffer",
3903
+ "hasOfferCatalog"
3904
+ ];
3905
+ /** These properties say where and when a place is open. A record with one of them is kept first. */
3906
+ const PLACE_KEYS = [
3907
+ "openingHours",
3908
+ "openingHoursSpecification",
3909
+ "address",
3910
+ "geo",
3911
+ "telephone"
3912
+ ];
3913
+ /**
3914
+ * This function keeps the JSON-LD records that fit the limit. A record with hours, an address or a
3915
+ * position goes first. A large record loses its reviews and its menu before it is left out. A
3916
+ * record that does not fit does not stop the records after it.
3917
+ */
3918
+ const keepJsonLd = (records) => {
3919
+ const place = (record) => PLACE_KEYS.some((key) => record[key] !== void 0);
3920
+ const ordered = [...records.filter(place), ...records.filter((record) => !place(record))];
3921
+ const kept = [];
3922
+ let dropped = 0;
3923
+ let room = MAX_JSONLD_CHARS;
3924
+ for (const record of ordered) {
3925
+ let size = JSON.stringify(record).length;
3926
+ let candidate = record;
3927
+ if (size > room) {
3928
+ candidate = Object.fromEntries(Object.entries(record).filter(([key]) => !BULKY.includes(key)));
3929
+ size = JSON.stringify(candidate).length;
3930
+ }
3931
+ if (size > room) {
3932
+ dropped += 1;
3933
+ continue;
3934
+ }
3935
+ kept.push(candidate);
3936
+ room -= size;
3937
+ }
3938
+ return {
3939
+ kept,
3940
+ dropped
3941
+ };
3942
+ };
3943
+ /**
3944
+ * This extractor is the built-in recipe for a web page, plain or rendered. It keeps the visible
3945
+ * text, the JSON-LD (whole objects, up to a limit) and the links. It decodes the body in the
3946
+ * encoding that the page declares, with `decodeBody`. It does not throw: a page that it cannot read
3947
+ * gives a document that names the error. When the reader cut the text, the document keeps the whole
3948
+ * text in `full_text`, so `verify` finds a quote past the cut.
3949
+ */
3950
+ const pageExtractor = (body, capture) => {
3951
+ if (isPdf(body, capture.contentType)) return pdfDocs(body, capture.url);
3952
+ if (capture.contentType !== null && !/html|xml/i.test(capture.contentType)) return [];
3953
+ const doc = pageDocId(capture.url);
3954
+ let page;
3955
+ try {
3956
+ page = extractPage(decodeBody(body, capture.contentType), capture.url);
3957
+ } catch (error) {
3958
+ return [{
3959
+ doc,
3960
+ kind: "page",
3961
+ title: capture.url,
3962
+ text: capture.url,
3963
+ url: capture.url,
3964
+ meta: { extract_error: error instanceof Error ? error.message.slice(0, 300) : String(error) }
3965
+ }];
3966
+ }
3967
+ const jsonld = keepJsonLd(page.jsonld);
3968
+ return [{
3969
+ doc,
3970
+ kind: "page",
3971
+ title: page.title,
3972
+ text: page.text.length > 0 ? page.text : page.title,
3973
+ url: capture.url,
3974
+ meta: {
3975
+ description: page.description,
3976
+ lang: page.lang,
3977
+ jsonld_types: [...new Set(page.jsonld.flatMap(typesOf))],
3978
+ jsonld: jsonld.kept,
3979
+ ...jsonld.dropped > 0 ? { jsonld_dropped: jsonld.dropped } : {},
3980
+ links: page.links.slice(0, 100),
3981
+ visible_chars: page.visibleChars,
3982
+ embedded: page.embedded,
3983
+ needs_render: page.needsRender,
3984
+ render_reason: page.renderReason,
3985
+ canonical: page.canonical,
3986
+ redirect: page.redirect,
3987
+ blocked: page.blocked,
3988
+ ...page.notes.length > 0 ? { notes: page.notes } : {}
3989
+ },
3990
+ ...page.fullText !== void 0 ? { full_text: page.fullText } : {}
3991
+ }];
3992
+ };
3993
+ const MAX_PDF_TEXT_CHARS = 2e5;
3994
+ /**
3995
+ * This function changes a PDF into one document of its text. A PDF with no text, a scan for
3996
+ * example, gives a document of its title only, and the document says that it needs OCR. A damaged
3997
+ * file gives no document.
3998
+ */
3999
+ const pdfDocs = async (body, url) => {
4000
+ let read;
4001
+ try {
4002
+ read = await readPdf(body, { maxChars: MAX_PDF_TEXT_CHARS });
4003
+ } catch {
4004
+ return [];
4005
+ }
4006
+ const file = new URL(url).pathname.split("/").pop() ?? "";
4007
+ let named = file;
4008
+ try {
4009
+ named = decodeURIComponent(file);
4010
+ } catch {}
4011
+ const title = read.title ?? (named || url);
4012
+ const notes = {
4013
+ ...read.truncated ? { truncated: true } : {},
4014
+ ...read.note !== null ? { note: read.note } : {}
4015
+ };
4016
+ if (read.text.length === 0) return [{
4017
+ doc: pageDocId(url),
4018
+ kind: "pdf",
4019
+ title,
4020
+ text: title,
4021
+ url,
4022
+ meta: {
4023
+ format: "pdf",
4024
+ pages: read.pages,
4025
+ visible_chars: 0,
4026
+ ocr_needed: read.note === null,
4027
+ ...notes
4028
+ }
4029
+ }];
4030
+ return [{
4031
+ doc: pageDocId(url),
4032
+ kind: "pdf",
4033
+ title,
4034
+ text: read.text.slice(0, MAX_PDF_TEXT_CHARS),
4035
+ url,
4036
+ meta: {
4037
+ format: "pdf",
4038
+ pages: read.pages,
4039
+ visible_chars: read.text.length,
4040
+ ...notes
4041
+ }
4042
+ }];
4043
+ };
4044
+ var Extractors = class {
4045
+ byRecipe = /* @__PURE__ */ new Map([["page", pageExtractor]]);
4046
+ register(recipe, extractor) {
4047
+ if (this.byRecipe.has(recipe)) throw new Error(`recipe ${recipe} already has an extractor`);
4048
+ this.byRecipe.set(recipe, extractor);
4049
+ }
4050
+ get(recipe) {
4051
+ return this.byRecipe.get(recipe);
4052
+ }
4053
+ /** This method gives the names of the recipes that have an extractor. */
4054
+ recipes() {
4055
+ return [...this.byRecipe.keys()];
4056
+ }
4057
+ };
4058
+ /** The structural fields of the metadata of a page: they hold no text of the page. A held-back page shows only these. */
4059
+ const STRUCTURAL_META = [
4060
+ "jsonld_types",
4061
+ "jsonld_dropped",
4062
+ "visible_chars",
4063
+ "lang",
4064
+ "embedded",
4065
+ "needs_render",
4066
+ "render_reason",
4067
+ "canonical",
4068
+ "redirect",
4069
+ "blocked",
4070
+ "format",
4071
+ "pages",
4072
+ "ocr_needed",
4073
+ "truncated"
4074
+ ];
4075
+ /** The note of a document that is held back. */
4076
+ const HELD_BACK = "This page asked not to be used as AI input, so its text is held back. Open the URL to read it, or read again with userProvided when a person gave this URL directly (the crawler logs that).";
4077
+ /**
4078
+ * This function tells if a document is held back: the page asked not to be AI input, and no person
4079
+ * gave its URL directly. The only way to read a held-back document is `userProvided`.
4080
+ */
4081
+ const isHeldBack = (doc, userProvided = false) => doc.ai_input === "no" && !userProvided;
4082
+ /** This function gives the metadata that a held-back document shows: only the structural fields, which hold no text of the page. */
4083
+ const heldBackMeta = (meta) => Object.fromEntries(Object.entries(meta).filter(([key]) => STRUCTURAL_META.includes(key)));
4084
+ const messageOf = (error) => error instanceof Error ? error.message.split(/\r?\n/)[0] ?? "" : String(error);
4085
+ /** This function tells which bot check or firewall sent its page in place of the content, or gives undefined. */
4086
+ const blockedBy = (doc) => {
4087
+ const blocked = doc.meta["blocked"];
4088
+ return typeof blocked === "string" && blocked.length > 0 ? blocked : void 0;
4089
+ };
4090
+ /** This function reads one capture with its extractor, and adds each document to the index as a version of the capture. */
4091
+ const indexCapture = async (runtime, fetched, recipe, rendered) => {
4092
+ const capture = fetched.capture;
4093
+ if (fetched.body === void 0 || capture === void 0) return { docs: [] };
4094
+ if (fetched.fromCache) {
4095
+ const known = runtime.index.docsOfCapture(capture);
4096
+ if (known.length > 0) return { docs: known };
4097
+ }
4098
+ const extractor = runtime.extractors.get(recipe);
4099
+ if (extractor === void 0) return { docs: [] };
4100
+ let extracted;
4101
+ try {
4102
+ extracted = await extractor(fetched.body, {
4103
+ url: fetched.url,
4104
+ contentType: rendered ? "text/html; charset=utf-8" : fetched.contentType ?? null
4105
+ });
4106
+ } catch (error) {
4107
+ return {
4108
+ docs: [],
4109
+ error: `extract failed (${recipe}): ${messageOf(error)}`
4110
+ };
4111
+ }
4112
+ const blocked = extracted.map(blockedBy).find((name) => name !== void 0);
4113
+ if (blocked !== void 0) {
4114
+ runtime.crawler.reportBlocked(fetched.url, blocked);
4115
+ return {
4116
+ docs: [],
4117
+ blocked,
4118
+ error: `the page is a bot check (${blocked}), not the content, so the crawler keeps nothing of it and stops the host`
4119
+ };
4120
+ }
4121
+ const now = runtime.now();
4122
+ return { docs: extracted.map((doc) => runtime.index.upsert({
4123
+ ...doc,
4124
+ captures: [capture],
4125
+ capture,
4126
+ fetched_at: fetched.fetchedAt ?? now,
4127
+ blob: fetched.blob ?? null,
4128
+ recipe,
4129
+ rendered,
4130
+ ai_input: fetched.signals.ai_input,
4131
+ ...fetched.scope !== void 0 ? { scope: fetched.scope } : {},
4132
+ ...fetched.labels !== void 0 ? { labels: fetched.labels } : {}
4133
+ }, now)) };
4134
+ };
4135
+ /**
4136
+ * This function fetches one URL and adds its documents to the index. Each recipe uses it. A fetch
4137
+ * from the cache gives the documents that the index has for that capture. Thus a repeated
4138
+ * question sends no request and reads nothing again. An extractor that throws gives no document and an
4139
+ * `error`, and the fetch still succeeds.
4140
+ */
4141
+ const fetchAndIndex = async (runtime, request) => {
4142
+ const fetched = await runtime.crawler.fetch(request);
4143
+ if (fetched.outcome !== "ok" && fetched.outcome !== "not-modified") return {
4144
+ fetched,
4145
+ docs: []
4146
+ };
4147
+ return {
4148
+ fetched,
4149
+ ...await indexCapture(runtime, fetched, request.recipe, false)
4150
+ };
4151
+ };
4152
+ /** This function renders a page in the headless browser and adds what the page shows to the index. */
4153
+ const renderAndIndex = async (runtime, request) => {
4154
+ const fetched = await runtime.crawler.render(request);
4155
+ if (fetched.outcome !== "ok") return {
4156
+ fetched,
4157
+ docs: []
4158
+ };
4159
+ return {
4160
+ fetched,
4161
+ ...await indexCapture(runtime, fetched, request.recipe, true)
4162
+ };
4163
+ };
4164
+ const HTML_TYPE = /html|xml/i;
4165
+ /**
4166
+ * This function reads one web page. It first sends a plain request. It opens the browser only when
4167
+ * the page shows almost nothing without JavaScript, or when the caller asks for it. It renders
4168
+ * only an HTML page: not a PDF, a different file or a bot check. A page that sends the reader on
4169
+ * with a `<meta http-equiv="refresh">` gets one more plain fetch, of its target.
4170
+ */
4171
+ const fetchPage = async (runtime, url, options) => {
4172
+ const plainRequest = (target) => ({
4173
+ url: target,
4174
+ mode: "page",
4175
+ recipe: "page",
4176
+ accept: "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5",
4177
+ maxAgeMs: options.maxAgeMs,
4178
+ onBehalfOf: options.onBehalfOf,
4179
+ job: options.job,
4180
+ signal: options.signal,
4181
+ scope: options.scope,
4182
+ labels: options.labels
4183
+ });
4184
+ let plain = await fetchAndIndex(runtime, plainRequest(url));
4185
+ const notes = [];
4186
+ if (plain.error !== void 0) notes.push(plain.error);
4187
+ const target = plain.docs[0]?.meta["redirect"];
4188
+ if (typeof target === "string" && target.length > 0 && plain.blocked === void 0) {
4189
+ const followed = await fetchAndIndex(runtime, plainRequest(target));
4190
+ notes.push(`the page sends the reader on to ${followed.fetched.url}, so the crawler fetched that page`);
4191
+ if (followed.error !== void 0) notes.push(followed.error);
4192
+ plain = followed;
4193
+ }
4194
+ let docs = plain.docs;
4195
+ let rendered = false;
4196
+ let fromCache = plain.fetched.fromCache;
4197
+ const reachable = plain.fetched.outcome === "ok" || plain.fetched.outcome === "not-modified";
4198
+ const first = docs[0];
4199
+ const html = first?.kind === "page" && (plain.fetched.contentType === void 0 || HTML_TYPE.test(plain.fetched.contentType));
4200
+ const wanted = options.render === "always" || options.render === "auto" && first?.meta["needs_render"] === true;
4201
+ if (reachable && wanted) {
4202
+ if (plain.blocked !== void 0) notes.push("the crawler does not render a bot check");
4203
+ else if (first === void 0 || !html) notes.push("the crawler renders only an HTML page, not a PDF or a different file");
4204
+ else {
4205
+ const current = runtime.index.get(first.doc);
4206
+ const stale = current !== void 0 && !current.rendered && current.capture_count > 1;
4207
+ const shown = await renderAndIndex(runtime, {
4208
+ url: plain.fetched.url,
4209
+ recipe: "page",
4210
+ maxAgeMs: stale ? 0 : options.maxAgeMs,
4211
+ onBehalfOf: options.onBehalfOf,
4212
+ job: options.job,
4213
+ signal: options.signal,
4214
+ scope: options.scope,
4215
+ labels: options.labels
4216
+ });
4217
+ if (shown.docs.length > 0) {
4218
+ docs = shown.docs;
4219
+ rendered = true;
4220
+ fromCache = fromCache && shown.fetched.fromCache;
4221
+ } else notes.push(shown.error ?? shown.fetched.note ?? "the render gave no document");
4222
+ }
4223
+ }
4224
+ const doc = docs[0] === void 0 ? void 0 : runtime.index.get(docs[0].doc) ?? docs[0];
4225
+ const held = doc !== void 0 && isHeldBack(doc);
4226
+ const types = doc?.meta["jsonld_types"];
4227
+ const links = doc?.meta["links"];
4228
+ if (held) notes.push(HELD_BACK);
4229
+ const note = [plain.fetched.note, ...notes].filter((part) => part !== void 0 && part.length > 0);
4230
+ return {
4231
+ url,
4232
+ doc: doc?.doc ?? null,
4233
+ outcome: plain.fetched.outcome,
4234
+ status: plain.fetched.status,
4235
+ title: doc === void 0 || held ? null : doc.title,
4236
+ chars: doc?.text.length ?? 0,
4237
+ rendered: doc?.rendered ?? rendered,
4238
+ structured: Array.isArray(types) ? types : [],
4239
+ links: Array.isArray(links) ? links.length : 0,
4240
+ captures: doc === void 0 ? [] : [...doc.captures],
4241
+ capture: doc?.capture ?? null,
4242
+ capture_count: doc?.capture_count ?? 0,
4243
+ fetched_at: doc?.fetched_at ?? plain.fetched.fetchedAt ?? null,
4244
+ from_cache: fromCache,
4245
+ held_back: held,
4246
+ note: note.length > 0 ? note.join("; ") : null
4247
+ };
4248
+ };
4249
+ const knownDoc = (runtime, id) => {
4250
+ const doc = runtime.index.get(id);
4251
+ if (doc === void 0) throw new Error(`no stored document ${id}: find one with find(), or fetch its page`);
4252
+ return doc;
4253
+ };
4254
+ const wholeNumber = (name, value, fallback) => {
4255
+ if (value === void 0) return fallback;
4256
+ if (!Number.isFinite(value) || value < 0) throw new RangeError(`${name} must be a number of 0 or more, not ${String(value)}`);
4257
+ return Math.floor(value);
4258
+ };
4259
+ /** This function moves a position off the second half of a surrogate pair, so a slice does not cut a character. */
4260
+ const onBoundary = (text, position) => {
4261
+ const code = text.charCodeAt(position);
4262
+ return position > 0 && position < text.length && code >= 56320 && code <= 57343 ? position - 1 : position;
4263
+ };
4264
+ /**
4265
+ * This function writes an `override` record to the capture log. A person gave the URL of a
4266
+ * held-back document directly, and the hold stops for one action (`read` or `verify`). The record
4267
+ * takes the scope and the labels of `marks`, or else those of the document, as its captures do.
4268
+ */
4269
+ const recordOverride = async (runtime, doc, action, marks = {}) => {
4270
+ const at = runtime.now();
4271
+ const scope = marks.scope ?? doc.scope;
4272
+ const labels = marks.labels ?? doc.labels;
4273
+ await runtime.log.append(makeRecord({
4274
+ source: "fetch",
4275
+ kind: "override",
4276
+ payload: {
4277
+ doc: doc.doc,
4278
+ url: doc.url,
4279
+ action,
4280
+ reason: "a person provided this URL",
4281
+ at
4282
+ },
4283
+ received_at: at,
4284
+ external_id: doc.url ?? doc.doc,
4285
+ ...scope !== void 0 ? { scope } : {},
4286
+ ...labels !== void 0 ? { labels } : {}
4287
+ }));
4288
+ };
4289
+ /**
4290
+ * This function reads a range of characters of one stored document. It applies the rule for a
4291
+ * held-back document: no title, no text, and only the structural metadata, unless `userProvided`.
4292
+ * The positions do not cut a character in two.
4293
+ */
4294
+ const readDocument = async (runtime, id, options = {}) => {
4295
+ const from = wholeNumber("from", options.from, 0);
4296
+ const length = wholeNumber("length", options.length, 4e3);
4297
+ const doc = knownDoc(runtime, id);
4298
+ const version = options.capture === void 0 ? void 0 : runtime.index.versionText(id, options.capture);
4299
+ if (options.capture !== void 0 && version === void 0) throw new Error(`the document ${id} has no version from the capture ${options.capture}`);
4300
+ const aiInput = doc.ai_input === "no" || version?.ai_input === "no" ? "no" : version?.ai_input ?? doc.ai_input;
4301
+ const userProvided = options.userProvided ?? false;
4302
+ const held = isHeldBack({ ai_input: aiInput }, userProvided);
4303
+ if (aiInput === "no" && userProvided) await recordOverride(runtime, doc, "read");
4304
+ const text = version?.text ?? doc.text;
4305
+ const start = onBoundary(text, Math.min(from, text.length));
4306
+ const end = held ? start : onBoundary(text, Math.min(text.length, start + length));
4307
+ const meta = version?.meta ?? doc.meta;
4308
+ return {
4309
+ doc: doc.doc,
4310
+ kind: version?.kind ?? doc.kind,
4311
+ title: held ? "" : version?.title ?? doc.title,
4312
+ url: version?.url ?? doc.url,
4313
+ meta: held ? heldBackMeta(meta) : meta,
4314
+ from: start,
4315
+ to: end,
4316
+ total: text.length,
4317
+ text: held ? "" : text.slice(start, end),
4318
+ captures: doc.captures,
4319
+ capture: version?.capture ?? doc.capture,
4320
+ fetched_at: version?.fetched_at ?? doc.fetched_at,
4321
+ ai_input: aiInput,
4322
+ held_back: held,
4323
+ note: held ? HELD_BACK : null
4324
+ };
4325
+ };
4326
+ /**
4327
+ * This function examines the versions of a document for a quote, and writes nothing. It also tells
4328
+ * if `userProvided` opened a held-back version, so that `verifyQuote` logs the override.
4329
+ */
4330
+ const examineQuote = (runtime, id, quote, options) => {
4331
+ const doc = knownDoc(runtime, id);
4332
+ const pinned = options.capture === void 0 ? void 0 : runtime.index.versionText(id, options.capture);
4333
+ if (options.capture !== void 0 && pinned === void 0) throw new Error(`the document ${id} has no version from the capture ${options.capture}`);
4334
+ const userProvided = options.userProvided ?? false;
4335
+ const base = {
4336
+ doc: doc.doc,
4337
+ url: doc.url,
4338
+ captures: doc.captures
4339
+ };
4340
+ const aiInput = doc.ai_input === "no" || pinned?.ai_input === "no" ? "no" : doc.ai_input;
4341
+ if (isHeldBack({ ai_input: aiInput }, userProvided)) return {
4342
+ result: {
4343
+ ...base,
4344
+ found: false,
4345
+ start: null,
4346
+ end: null,
4347
+ excerpt: null,
4348
+ capture: null,
4349
+ blob: null,
4350
+ fetched_at: null,
4351
+ stale: false,
4352
+ in_full_text: false,
4353
+ complete: false,
4354
+ count: 0,
4355
+ held_back: true,
4356
+ note: HELD_BACK
4357
+ },
4358
+ override: false
4359
+ };
4360
+ const current = pinned ?? runtime.index.versionText(id);
4361
+ const candidates = pinned !== void 0 ? [pinned] : [...current === void 0 ? [] : [current], ...runtime.index.earlierTexts(id)];
4362
+ const held = candidates.filter((version) => version.ai_input === "no").length;
4363
+ const override = userProvided && (aiInput === "no" || held > 0);
4364
+ for (const [index, version] of candidates.entries()) {
4365
+ if (isHeldBack(version, userProvided)) continue;
4366
+ let hit = locate(version.text, quote);
4367
+ let inFull = false;
4368
+ if (!hit.found && version.full_text !== void 0) {
4369
+ hit = locate(version.full_text, quote);
4370
+ inFull = hit.found;
4371
+ }
4372
+ if (!hit.found) continue;
4373
+ const stale = pinned === void 0 && index > 0;
4374
+ return {
4375
+ result: {
4376
+ ...base,
4377
+ found: true,
4378
+ start: hit.start,
4379
+ end: hit.end,
4380
+ excerpt: hit.excerpt,
4381
+ capture: version.capture,
4382
+ blob: version.blob,
4383
+ fetched_at: version.fetched_at,
4384
+ stale,
4385
+ in_full_text: inFull,
4386
+ complete: true,
4387
+ count: hit.count,
4388
+ held_back: false,
4389
+ note: stale ? "only an earlier capture of the document holds the quote" : null
4390
+ },
4391
+ override
4392
+ };
4393
+ }
4394
+ const incomplete = current !== void 0 && current.meta["truncated"] === true && current.full_text === void 0;
4395
+ const notes = [...incomplete ? ["the reader stopped before the end of the file, so the quote can be in the part that it did not read"] : [], ...held > 0 && !userProvided ? [`the check did not examine the versions that asked not to be AI input (${String(held)})`] : []];
4396
+ return {
4397
+ result: {
4398
+ ...base,
4399
+ found: false,
4400
+ start: null,
4401
+ end: null,
4402
+ excerpt: null,
4403
+ capture: current?.capture ?? null,
4404
+ blob: current?.blob ?? null,
4405
+ fetched_at: current?.fetched_at ?? null,
4406
+ stale: false,
4407
+ in_full_text: false,
4408
+ complete: !incomplete,
4409
+ count: 0,
4410
+ held_back: false,
4411
+ note: notes.length > 0 ? notes.join("; ") : null
4412
+ },
4413
+ override
4414
+ };
4415
+ };
4416
+ /**
4417
+ * This function finds a quote in a stored document, with `locate`. It tells which capture holds the
4418
+ * quote, with its blob, its fetch time and the positions:
4419
+ *
4420
+ * - With `capture`, it examines the version of that capture only.
4421
+ * - With no capture, it examines the current text first. When the quote is not there, it examines
4422
+ * the earlier texts, the newest first. A match there has `stale: true`.
4423
+ * - When the reader cut a text, it also examines the whole text.
4424
+ *
4425
+ * A held-back document gives `found: false` and no positions, so a quote cannot test its text,
4426
+ * unless `userProvided`. The same rule applies to each version. A version that asked not to be AI
4427
+ * input stays held back after a later capture drops the signal. The crawler logs an override when
4428
+ * `userProvided` opens a held-back version.
4429
+ */
4430
+ const verifyQuote = async (runtime, id, quote, options = {}) => {
4431
+ const { result, override } = examineQuote(runtime, id, quote, options);
4432
+ if (override) await recordOverride(runtime, knownDoc(runtime, id), "verify");
4433
+ return result;
4434
+ };
4435
+ /**
4436
+ * This function searches the stored documents. The syntax of the query is the syntax of
4437
+ * `parseQuery`. A held-back document is not in the results.
4438
+ */
4439
+ const findDocuments = async (runtime, q, options = {}) => {
4440
+ await Promise.resolve();
4441
+ return runtime.index.find(q, {
4442
+ limit: options.limit ?? 10,
4443
+ kind: options.kind ?? null
4444
+ });
4445
+ };
4446
+ const RENDERED_MARK = "#rendered";
4447
+ const MAX_ERRORS = 20;
4448
+ /** This function gives the URL of a capture and its variant. A URL of version 0.1 can end in `#rendered`. */
4449
+ const captureTarget = (payload) => {
4450
+ if (payload.url.endsWith(RENDERED_MARK)) return {
4451
+ url: payload.url.slice(0, -9),
4452
+ variant: "rendered"
4453
+ };
4454
+ return {
4455
+ url: payload.url,
4456
+ variant: payload.rendered === true ? "rendered" : "plain"
4457
+ };
4458
+ };
4459
+ const newTally = () => ({
4460
+ captures: 0,
4461
+ failed: 0,
4462
+ missing: 0,
4463
+ unknown: /* @__PURE__ */ new Set(),
4464
+ errors: []
4465
+ });
4466
+ /** This function replays the cache entry of one capture, as the crawler made it at the time of the fetch. */
4467
+ const replayLatest = (build, envelope, payload) => {
4468
+ const { url, variant } = captureTarget(payload);
4469
+ const fetchedAt = payload.fetched_at ?? envelope.received_at;
4470
+ if (payload.outcome === "redirect") {
4471
+ const location = payload.location ?? (typeof payload.note === "string" && payload.note.startsWith("location ") ? payload.note.slice(9) : null);
4472
+ const permanent = payload.status === 301 || payload.status === 308;
4473
+ if (location !== null && (permanent || typeof payload.fresh_until === "string")) build.setLatest({
4474
+ url,
4475
+ variant,
4476
+ capture: envelope.id,
4477
+ blob: null,
4478
+ etag: null,
4479
+ last_modified: null,
4480
+ content_type: null,
4481
+ fetched_at: fetchedAt,
4482
+ status: payload.status ?? 301,
4483
+ ai_input: "unset",
4484
+ location,
4485
+ fresh_until: payload.fresh_until ?? null,
4486
+ vary_key: payload.vary_key ?? null
4487
+ });
4488
+ return;
4489
+ }
4490
+ if (payload.cut === true || payload.unasked === true) return;
4491
+ const previous = build.latest(url, variant);
4492
+ const revisit = envelope.kind === "revisit" || payload.outcome === "not-modified";
4493
+ const truncated = payload.truncated === true;
4494
+ const etag = truncated ? null : payload.etag ?? (revisit ? previous?.etag ?? null : null);
4495
+ const lastModified = truncated ? null : payload.last_modified ?? (revisit ? previous?.last_modified ?? null : null);
4496
+ const kept = revisit ? previous?.content_type ?? void 0 : void 0;
4497
+ const contentType = payload.content_type ?? kept ?? (variant === "rendered" ? "text/html; charset=utf-8" : null);
4498
+ const status = revisit ? previous === void 0 || previous.status < 200 || previous.status >= 300 ? 200 : previous.status : payload.status ?? 200;
4499
+ build.setLatest({
4500
+ url,
4501
+ variant,
4502
+ capture: envelope.id,
4503
+ blob: payload.blob ?? null,
4504
+ etag,
4505
+ last_modified: lastModified,
4506
+ content_type: contentType,
4507
+ fetched_at: fetchedAt,
4508
+ status,
4509
+ ai_input: payload.signals?.ai_input ?? "unset",
4510
+ ...truncated ? { truncated: true } : {},
4511
+ fresh_until: payload.fresh_until ?? null,
4512
+ vary_key: payload.vary_key ?? (revisit ? previous?.vary_key ?? null : null)
4513
+ });
4514
+ };
4515
+ /** This function replays one record of the log: its cache entry, and the documents that its extractor gives. */
4516
+ const replayRecord = async (runtime, target, envelope, tally) => {
4517
+ if (envelope.kind !== "capture" && envelope.kind !== "revisit") return;
4518
+ const payload = envelope.payload;
4519
+ if (payload === null || typeof payload !== "object" || typeof payload.url !== "string") return;
4520
+ if (payload.outcome === "redirect") {
4521
+ replayLatest(target, envelope, payload);
4522
+ return;
4523
+ }
4524
+ if (typeof payload.blob !== "string") return;
4525
+ if (payload.outcome !== "ok" && payload.outcome !== "not-modified") return;
4526
+ tally.captures += 1;
4527
+ replayLatest(target, envelope, payload);
4528
+ const extractor = runtime.extractors.get(payload.recipe);
4529
+ if (extractor === void 0) {
4530
+ if (payload.recipe !== "robots.txt") tally.unknown.add(payload.recipe);
4531
+ return;
4532
+ }
4533
+ const body = await runtime.blobs.get(payload.blob);
4534
+ if (body === void 0) {
4535
+ tally.missing += 1;
4536
+ return;
4537
+ }
4538
+ const { url, variant } = captureTarget(payload);
4539
+ const latest = target.latest(url, variant);
4540
+ let extracted;
4541
+ try {
4542
+ extracted = await extractor(body, {
4543
+ url,
4544
+ contentType: variant === "rendered" ? "text/html; charset=utf-8" : payload.content_type ?? latest?.content_type ?? null
4545
+ });
4546
+ } catch (error) {
4547
+ tally.failed += 1;
4548
+ if (tally.errors.length < MAX_ERRORS) tally.errors.push({
4549
+ capture: envelope.id,
4550
+ error: messageOf(error)
4551
+ });
4552
+ return;
4553
+ }
4554
+ if (extracted.some((doc) => blockedBy(doc) !== void 0)) return;
4555
+ for (const doc of extracted) target.upsert({
4556
+ ...doc,
4557
+ captures: [envelope.id],
4558
+ capture: envelope.id,
4559
+ fetched_at: payload.fetched_at ?? envelope.received_at,
4560
+ blob: payload.blob,
4561
+ recipe: payload.recipe,
4562
+ rendered: variant === "rendered",
4563
+ ai_input: payload.signals?.ai_input ?? "unset",
4564
+ ...envelope.scope !== void 0 ? { scope: envelope.scope } : {},
4565
+ ...envelope.labels !== void 0 ? { labels: envelope.labels } : {}
4566
+ }, envelope.received_at);
4567
+ };
4568
+ /**
4569
+ * This function builds the reading index again from the capture log and the blobs. The index is a
4570
+ * projection, and this function is the proof that its deletion loses nothing. These are its rules:
4571
+ *
4572
+ * - It builds a new generation of the tables and commits it in batches. The old index answers
4573
+ * until the end, and a failed rebuild leaves it as it was.
4574
+ * - It reads the log as a stream, so its memory does not grow with the log.
4575
+ * - It counts and skips a damaged line, a missing blob and an extractor that throws.
4576
+ * - The documents of a recipe that has no extractor in this runtime stay, from the old index.
4577
+ * - A capture that a different writer adds during the rebuild goes into the new index.
4578
+ */
4579
+ const reindex = async (runtime, options = {}) => {
4580
+ const since = runtime.now();
4581
+ const build = runtime.index.beginRebuild(options.batch === void 0 ? {} : { batch: options.batch });
4582
+ const tally = newTally();
4583
+ let damaged = 0;
4584
+ try {
4585
+ const scan = runtime.captures.scan({ source: "fetch" }, { onDamaged: () => {
4586
+ damaged += 1;
4587
+ } });
4588
+ for await (const envelope of scan) await replayRecord(runtime, build, envelope, tally);
4589
+ const carried = build.carry(new Set(runtime.extractors.recipes()));
4590
+ const counts = build.commit(since);
4591
+ return {
4592
+ captures: tally.captures,
4593
+ ...counts,
4594
+ failed: tally.failed,
4595
+ missing: tally.missing,
4596
+ damaged,
4597
+ carried,
4598
+ unknown_recipes: [...tally.unknown].sort(),
4599
+ errors: tally.errors
4600
+ };
4601
+ } catch (error) {
4602
+ build.abort();
4603
+ throw error;
4604
+ }
4605
+ };
4606
+ /** The tail starts this long before the newest cache entry. Two writers and their clocks can differ. */
4607
+ const TAIL_SLACK_MS = 6e5;
4608
+ /** This function tells if the index missed a record of the log: no cache entry as new as it, or no version of its documents. */
4609
+ const missed = (runtime, envelope) => {
4610
+ if (envelope.kind !== "capture" && envelope.kind !== "revisit") return false;
4611
+ const payload = envelope.payload;
4612
+ if (payload === null || typeof payload !== "object" || typeof payload.url !== "string") return false;
4613
+ const fetchedAt = payload.fetched_at ?? envelope.received_at;
4614
+ const { url, variant } = captureTarget(payload);
4615
+ const entry = runtime.index.latest(url, variant);
4616
+ if (payload.outcome === "redirect") return entry === void 0 || entry.fetched_at < fetchedAt;
4617
+ if (typeof payload.blob !== "string" || payload.outcome !== "ok" && payload.outcome !== "not-modified") return false;
4618
+ if (entry === void 0 || entry.fetched_at < fetchedAt) return true;
4619
+ return runtime.extractors.get(payload.recipe) !== void 0 && !runtime.index.hasVersion(envelope.id);
4620
+ };
4621
+ /**
4622
+ * This function brings the reading index up to the capture log, with no request. `openCrawler`
4623
+ * uses it at open. An application that uses `createRuntime` uses it the same way:
4624
+ *
4625
+ * - A stale index (a new schema, or a deleted file) is built again with `reindex`.
4626
+ * - An index that is behind the log gets the captures that it missed. A process can stop between
4627
+ * the log and the index, and SQLite can lose its last transactions after a power cut. The
4628
+ * function reads the log from a short time before the newest cache entry.
4629
+ *
4630
+ * It gives the counts, or undefined when the index missed nothing.
4631
+ */
4632
+ const catchUpIndex = async (runtime) => {
4633
+ if (runtime.index.stale) return {
4634
+ mode: "rebuild",
4635
+ ...await reindex(runtime)
4636
+ };
4637
+ const newest = runtime.index.newestFetch();
4638
+ const since = newest === void 0 ? void 0 : (/* @__PURE__ */ new Date(Date.parse(newest) - TAIL_SLACK_MS)).toISOString();
4639
+ const tally = newTally();
4640
+ let damaged = 0;
4641
+ const scan = runtime.captures.scan({
4642
+ source: "fetch",
4643
+ ...since === void 0 ? {} : { since }
4644
+ }, { onDamaged: () => {
4645
+ damaged += 1;
4646
+ } });
4647
+ for await (const envelope of scan) if (missed(runtime, envelope)) await replayRecord(runtime, runtime.index, envelope, tally);
4648
+ if (tally.captures === 0) return void 0;
4649
+ return {
4650
+ mode: "tail",
4651
+ captures: tally.captures,
4652
+ ...runtime.index.counts(),
4653
+ failed: tally.failed,
4654
+ missing: tally.missing,
4655
+ damaged,
4656
+ carried: 0,
4657
+ unknown_recipes: [...tally.unknown].sort(),
4658
+ errors: tally.errors
4659
+ };
4660
+ };
4661
+ //#endregion
4662
+ //#region src/open.ts
4663
+ var open_exports = /* @__PURE__ */ __exportAll({
4664
+ BOT_PAGE: () => BOT_PAGE,
4665
+ DEFAULT_TOKEN: () => DEFAULT_TOKEN,
4666
+ createRuntime: () => createRuntime,
4667
+ defaultUserAgent: () => defaultUserAgent,
4668
+ durationMs: () => durationMs,
4669
+ isMailbox: () => isMailbox,
4670
+ openCrawler: () => openCrawler
4671
+ });
4672
+ /**
4673
+ * This function gives the default `User-Agent` header. The header names the crawler, its version
4674
+ * and the page about it, so a site owner can find out what made each request.
4675
+ */
4676
+ const defaultUserAgent = (contact) => `${DEFAULT_TOKEN}/${VERSION} (+${BOT_PAGE}${contact === void 0 ? "" : `; ${contact}`})`;
4677
+ /** A mailbox is `name@host`, or a name and `<name@host>`. A URL is not a mailbox, also when it holds an `@`. */
4678
+ const MAILBOX = /^(?:[^\s<>@:/]+@[^\s<>@/]+|[^<>]*<[^\s<>@:/]+@[^\s<>@/]+>)$/;
4679
+ /**
4680
+ * This function tells if a contact is a mailbox. The `From` header holds only a mailbox (RFC 9110,
4681
+ * section 10.1.2). Thus the crawler sends a mailbox in `From` and in the `User-Agent`, and a URL in
4682
+ * the `User-Agent` only.
4683
+ */
4684
+ const isMailbox = (contact) => MAILBOX.test(contact.trim());
4685
+ const UNITS = {
4686
+ ms: 1,
4687
+ s: 1e3,
4688
+ m: 6e4,
4689
+ h: 36e5,
4690
+ d: 864e5,
4691
+ w: 6048e5
4692
+ };
4693
+ /**
4694
+ * This function changes a duration into milliseconds. The units are `ms`, `s`, `m` (minutes), `h`,
4695
+ * `d` and `w`, in any case. The unit `M` is an error, because a month is not a unit. A bare `0`
4696
+ * is zero. The function throws an error for a text that it cannot read.
4697
+ */
4698
+ const durationMs = (duration) => {
4699
+ if (typeof duration === "number") {
4700
+ if (!Number.isFinite(duration) || duration < 0) throw new Error(`not a duration: ${String(duration)}`);
4701
+ return duration;
4702
+ }
4703
+ if (/^\s*0+(\.0+)?\s*$/.test(duration)) return 0;
4704
+ const match = /^\s*(\d+(?:\.\d+)?)\s*(ms|s|m|h|d|w)\s*$/i.exec(duration);
4705
+ const unit = match?.[2];
4706
+ const factor = unit === void 0 || unit === "M" ? void 0 : UNITS[unit.toLowerCase()];
4707
+ if (match === null || factor === void 0) throw new Error(`not a duration: "${duration}" (write a number and ms, s, m, h, d or w; m is minutes)`);
4708
+ return Number(match[1]) * factor;
4709
+ };
4710
+ /** The parts of a store that can be a symbolic link to a different place only by mistake or by attack. */
4711
+ const STORE_PARTS = [
4712
+ "captures",
4713
+ "blobs",
4714
+ "index.sqlite",
4715
+ "hosts.sqlite"
4716
+ ];
4717
+ /**
4718
+ * This function makes the folder of a store for the owner only, where the system has modes. It
4719
+ * refuses a store folder of a different user, and a part of the store that is a symbolic link.
4720
+ * The store folder itself can be a link, for example to a larger disk.
4721
+ */
4722
+ const guardStore = (dir) => {
4723
+ mkdirSync(dir, {
4724
+ recursive: true,
4725
+ mode: 448
4726
+ });
4727
+ const real = lstatSync(dir).isSymbolicLink() ? realpathSync(dir) : dir;
4728
+ const info = statSync(real);
4729
+ if (!info.isDirectory()) throw new Error(`the store ${dir} is not a folder`);
4730
+ if (process.platform !== "win32" && typeof process.getuid === "function" && info.uid !== process.getuid()) throw new Error(`the store ${dir} belongs to a different user: use a folder of your own`);
4731
+ for (const part of STORE_PARTS) {
4732
+ let link = false;
4733
+ try {
4734
+ link = lstatSync(join(dir, part)).isSymbolicLink();
4735
+ } catch {}
4736
+ if (link) throw new Error(`${part} in the store ${dir} is a symbolic link: the crawler does not follow it`);
4737
+ }
4738
+ };
4739
+ /**
4740
+ * This function makes a runtime. It is the one place that puts the log, the blob store, the index,
4741
+ * the host state, the crawler and the extractors together. The function `openCrawler` uses it.
4742
+ * An application that needs the parts uses it too. The runtime and the crawler share one clock.
4743
+ */
4744
+ const createRuntime = (options) => {
4745
+ const dir = options.dir;
4746
+ if (dir === void 0 && (options.log === void 0 || options.blobs === void 0 || options.index === void 0)) throw new Error("a runtime needs dir, or a log, a blob store and an index");
4747
+ const contact = options.contact;
4748
+ if (contact !== void 0 && !/^[\t -~]*$/.test(contact)) throw new Error(`contact "${contact}" has a character that an HTTP header cannot hold: use printable ASCII`);
4749
+ if (dir !== void 0) guardStore(dir);
4750
+ const owned = [];
4751
+ const log = options.log ?? new CaptureLog(join(dir, "captures"));
4752
+ const blobs = options.blobs ?? new FileBlobStore(join(dir, "blobs"));
4753
+ let index = options.index;
4754
+ if (index === void 0) {
4755
+ const made = new DocIndex(join(dir, "index.sqlite"));
4756
+ owned.push(() => made.close());
4757
+ index = made;
4758
+ }
4759
+ let hostState = options.hostState;
4760
+ if (hostState === void 0) {
4761
+ if (dir === void 0) hostState = new MemoryHostStateStore();
4762
+ else {
4763
+ const made = new SqliteHostStateStore(join(dir, "hosts.sqlite"));
4764
+ owned.push(() => made.close());
4765
+ hostState = made;
4766
+ }
4767
+ }
4768
+ const clock = options.clock ?? realClock;
4769
+ const renderer = options.renderer === false ? void 0 : options.renderer ?? checkedRenderer(playwrightRenderer());
4770
+ const crawler = new Crawler({
4771
+ token: options.token ?? "PoliteCrawler",
4772
+ userAgent: options.userAgent ?? defaultUserAgent(contact),
4773
+ from: contact !== void 0 && isMailbox(contact) ? contact.trim() : void 0,
4774
+ log,
4775
+ blobs,
4776
+ index,
4777
+ renderer,
4778
+ maxActive: options.maxActive,
4779
+ maxRenders: options.maxRenders,
4780
+ dispatcher: options.dispatcher,
4781
+ egress: options.egress,
4782
+ politeness: options.politeness,
4783
+ hostState,
4784
+ clock,
4785
+ random: options.random,
4786
+ scope: options.scope,
4787
+ scopeRank: options.scopeRank,
4788
+ obeyDefaultToken: options.obeyDefaultToken,
4789
+ onEvent: options.onEvent
4790
+ });
4791
+ const extractors = new Extractors();
4792
+ for (const [recipe, extractor] of Object.entries(options.extractors ?? {})) extractors.register(recipe, extractor);
4793
+ let closing;
4794
+ return {
4795
+ crawler,
4796
+ index,
4797
+ blobs,
4798
+ captures: log,
4799
+ log,
4800
+ extractors,
4801
+ hostState,
4802
+ now: () => new Date(clock.now()).toISOString(),
4803
+ close() {
4804
+ closing ??= (async () => {
4805
+ await crawler.close();
4806
+ for (const close of owned) await close();
4807
+ })();
4808
+ return closing;
4809
+ }
4810
+ };
4811
+ };
4812
+ /**
4813
+ * This function opens a crawler on a folder. The folder keeps the capture log (`captures/`), the
4814
+ * response bodies (`blobs/`), the search index (`index.sqlite`) and the state of each host
4815
+ * (`hosts.sqlite`). The index is a projection: `reindex` makes it again from the log and the bodies.
4816
+ * At open, `catchUpIndex` rebuilds an index of an earlier version, and gives an index that is
4817
+ * behind the log the captures that it missed.
4818
+ *
4819
+ * Each method gives a promise, and none throws before its promise. After `close`, each method
4820
+ * rejects.
4821
+ */
4822
+ const openCrawler = (options) => {
4823
+ const runtime = createRuntime(options);
4824
+ let closed = false;
4825
+ let rebuilt;
4826
+ const ready = existsSync(join(options.dir, "captures")) ? catchUpIndex(runtime).then((done) => {
4827
+ if (done?.mode === "rebuild") {
4828
+ const { mode, ...counts } = done;
4829
+ rebuilt = mode === "rebuild" ? counts : void 0;
4830
+ }
4831
+ }, () => void 0) : Promise.resolve();
4832
+ const live = async () => {
4833
+ if (closed) throw new Error("the crawler is closed");
4834
+ await ready;
4835
+ if (closed) throw new Error("the crawler is closed");
4836
+ };
4837
+ return {
4838
+ runtime,
4839
+ async fetch(url, fetchOptions = {}) {
4840
+ await live();
4841
+ rebuilt = void 0;
4842
+ const maxAge = fetchOptions.maxAge ?? "24h";
4843
+ return fetchPage(runtime, url, {
4844
+ render: fetchOptions.render ?? "auto",
4845
+ maxAgeMs: maxAge === "auto" ? "auto" : durationMs(maxAge),
4846
+ onBehalfOf: [...options.onBehalfOf ?? [], ...fetchOptions.onBehalfOf ?? []],
4847
+ signal: fetchOptions.signal,
4848
+ scope: fetchOptions.scope,
4849
+ labels: fetchOptions.labels
4850
+ });
4851
+ },
4852
+ async find(q, findOptions = {}) {
4853
+ await live();
4854
+ return findDocuments(runtime, q, findOptions);
4855
+ },
4856
+ async read(id, readOptions = {}) {
4857
+ await live();
4858
+ return readDocument(runtime, id, readOptions);
4859
+ },
4860
+ async verify(id, quote, verifyOptions = {}) {
4861
+ await live();
4862
+ return verifyQuote(runtime, id, quote, verifyOptions);
4863
+ },
4864
+ async versions(id, versionOptions = {}) {
4865
+ await live();
4866
+ const held = runtime.index.get(id)?.ai_input === "no";
4867
+ return runtime.index.versions(id, versionOptions).map((version) => held || version.ai_input === "no" ? {
4868
+ ...version,
4869
+ title: ""
4870
+ } : version);
4871
+ },
4872
+ async reindex() {
4873
+ await live();
4874
+ const done = rebuilt;
4875
+ rebuilt = void 0;
4876
+ return done ?? reindex(runtime);
4877
+ },
4878
+ async close() {
4879
+ closed = true;
4880
+ await ready;
4881
+ await runtime.close();
4882
+ }
4883
+ };
4884
+ };
4885
+ //#endregion
4886
+ export { DEFAULT_SECRET_PARAMS as A, readPdf as C, locate as D, DocIndex as E, FileBlobStore as F, makeRecord as M, EgressRefused as N, CAPTURE_SCHEMA as O, egressLookup as P, isPdf as S, SqliteHostStateStore as T, readDocument as _, openCrawler as a, renderAndIndex as b, HELD_BACK as c, fetchPage as d, findDocuments as f, pageExtractor as g, pageDocId as h, isMailbox as i, renderedKey as j, Crawler as k, catchUpIndex as l, isHeldBack as m, defaultUserAgent as n, open_exports as o, heldBackMeta as p, durationMs as r, Extractors as s, createRuntime as t, fetchAndIndex as u, recordOverride as v, CaptureLog as w, verifyQuote as x, reindex as y };
4887
+
4888
+ //# sourceMappingURL=open-CJR4b44v.mjs.map