polite-crawler 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +293 -2
- package/build/browser-gJ_h4AhB.mjs +511 -0
- package/build/browser-gJ_h4AhB.mjs.map +1 -0
- package/build/classify-CrfyXcwo.d.mts +51 -0
- package/build/classify.d.mts +2 -0
- package/build/classify.mjs +116 -0
- package/build/classify.mjs.map +1 -0
- package/build/cli.mjs +348 -0
- package/build/cli.mjs.map +1 -0
- package/build/hours.d.mts +113 -0
- package/build/hours.mjs +905 -0
- package/build/hours.mjs.map +1 -0
- package/build/html-nWYFVsRU.d.mts +98 -0
- package/build/html-oNpGAHwi.mjs +1038 -0
- package/build/html-oNpGAHwi.mjs.map +1 -0
- package/build/html.d.mts +2 -0
- package/build/html.mjs +1 -0
- package/build/index.d.mts +131 -0
- package/build/index.mjs +8 -0
- package/build/mcp.d.mts +40 -0
- package/build/mcp.mjs +480 -0
- package/build/mcp.mjs.map +1 -0
- package/build/open-CJR4b44v.mjs +4888 -0
- package/build/open-CJR4b44v.mjs.map +1 -0
- package/build/open-CZqwY7GH.d.mts +1461 -0
- package/build/politeness-C5bZ1OKM.d.mts +274 -0
- package/build/politeness.d.mts +2 -0
- package/build/politeness.mjs +372 -0
- package/build/politeness.mjs.map +1 -0
- package/build/robots-R-qaEBOb.d.mts +144 -0
- package/build/robots.d.mts +2 -0
- package/build/robots.mjs +369 -0
- package/build/robots.mjs.map +1 -0
- package/build/signals-BHpy2xb2.d.mts +46 -0
- package/build/signals.d.mts +2 -0
- package/build/signals.mjs +113 -0
- package/build/signals.mjs.map +1 -0
- package/build/version-rVx8_y1_.mjs +11 -0
- package/build/version-rVx8_y1_.mjs.map +1 -0
- package/package.json +131 -3
- package/record.schema.json +198 -0
|
@@ -0,0 +1,1038 @@
|
|
|
1
|
+
import { Readability } from "@mozilla/readability";
|
|
2
|
+
import { parseHTML } from "linkedom";
|
|
3
|
+
//#region src/urls.ts
|
|
4
|
+
/**
|
|
5
|
+
* This module gives the canonical form of a URL. The document id of a page and the cache key of the
|
|
6
|
+
* crawler use it. Thus two spellings of one page are one document and one cached answer.
|
|
7
|
+
*/
|
|
8
|
+
const TRACKING = /^(utm_[a-z_]+|fbclid|gclid|gclsrc|dclid|gbraid|wbraid|msclkid|yclid|ttclid|twclid|li_fat_id|srsltid|mc_[a-z]+|_ga|_gl|_hsenc|_hsmi|__hstc|__hssc|__hsfp|mkt_tok|ref_src|igshid)$/i;
|
|
9
|
+
/** This function decodes the escapes of unreserved characters (`%7E` is `~`), and writes the other escapes in capitals. */
|
|
10
|
+
const normalizeEscapes = (value) => value.replace(/%[0-9a-f]{2}/gi, (escape) => {
|
|
11
|
+
const char = String.fromCharCode(Number.parseInt(escape.slice(1), 16));
|
|
12
|
+
return /^[A-Za-z0-9\-._~]$/.test(char) ? char : escape.toUpperCase();
|
|
13
|
+
});
|
|
14
|
+
/**
|
|
15
|
+
* This function gives one URL for each page: a lowercase host, and no fragment, tracking
|
|
16
|
+
* parameters, empty query or default port. It decodes the escapes of unreserved characters.
|
|
17
|
+
*/
|
|
18
|
+
const canonicalUrl = (value) => {
|
|
19
|
+
const url = new URL(value);
|
|
20
|
+
url.hash = "";
|
|
21
|
+
url.hostname = url.hostname.toLowerCase();
|
|
22
|
+
if (url.protocol === "https:" && url.port === "443" || url.protocol === "http:" && url.port === "80") url.port = "";
|
|
23
|
+
for (const key of Array.from(url.searchParams.keys())) if (TRACKING.test(key)) url.searchParams.delete(key);
|
|
24
|
+
if (url.pathname.includes("%")) url.pathname = normalizeEscapes(url.pathname);
|
|
25
|
+
url.search = url.search.includes("%") ? normalizeEscapes(url.search) : url.search;
|
|
26
|
+
return url.href;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* This function gives the cache key of a URL: its canonical form. A text that is not a URL stays as
|
|
30
|
+
* it is. The crawler and `reindex` use the same key, so a rebuilt index answers the same requests.
|
|
31
|
+
*/
|
|
32
|
+
const cacheKeyOf = (value) => {
|
|
33
|
+
try {
|
|
34
|
+
return canonicalUrl(value);
|
|
35
|
+
} catch {
|
|
36
|
+
return value;
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
//#endregion
|
|
40
|
+
//#region src/text.ts
|
|
41
|
+
/**
|
|
42
|
+
* The body decoder changes the bytes of a response into text. It finds the encoding in this order,
|
|
43
|
+
* as the WHATWG encoding rules and the browsers do:
|
|
44
|
+
*
|
|
45
|
+
* 1. A byte order mark.
|
|
46
|
+
* 2. The `charset` of the `Content-Type` header.
|
|
47
|
+
* 3. A `<meta charset>` or `http-equiv` declaration in the first 2,048 bytes of an HTML page, or the
|
|
48
|
+
* encoding of an XML declaration.
|
|
49
|
+
* 4. UTF-8 when the bytes are UTF-8, and windows-1252 when they are not.
|
|
50
|
+
*
|
|
51
|
+
* It uses only `TextDecoder`, so it operates in a browser too. A byte that the encoding does not
|
|
52
|
+
* know becomes U+FFFD. The decoder does not throw.
|
|
53
|
+
*/
|
|
54
|
+
const PRESCAN_BYTES = 2048;
|
|
55
|
+
/** This function gives the WHATWG name of an encoding label, or undefined when the runtime does not know the label. */
|
|
56
|
+
const encodingOf = (label) => {
|
|
57
|
+
try {
|
|
58
|
+
return new TextDecoder(label.trim().toLowerCase()).encoding;
|
|
59
|
+
} catch {
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
const bomOf = (bytes) => {
|
|
64
|
+
if (bytes[0] === 239 && bytes[1] === 187 && bytes[2] === 191) return "utf-8";
|
|
65
|
+
if (bytes[0] === 254 && bytes[1] === 255) return "utf-16be";
|
|
66
|
+
if (bytes[0] === 255 && bytes[1] === 254) return "utf-16le";
|
|
67
|
+
};
|
|
68
|
+
/** This function gives the `charset` parameter of a media type, or undefined. */
|
|
69
|
+
const charsetOf = (contentType) => {
|
|
70
|
+
if (contentType === null) return void 0;
|
|
71
|
+
const match = /;\s*charset\s*=\s*(?:"([^"]*)"|([^\s;"]+))/i.exec(contentType.slice(0, 512));
|
|
72
|
+
return match?.[1] ?? match?.[2] ?? void 0;
|
|
73
|
+
};
|
|
74
|
+
/** This function reads the encoding that the start of an HTML or XML document declares. */
|
|
75
|
+
const declaredCharset = (bytes) => {
|
|
76
|
+
let head = "";
|
|
77
|
+
const end = Math.min(bytes.length, PRESCAN_BYTES);
|
|
78
|
+
for (let index = 0; index < end; index += 1) head += String.fromCharCode(bytes[index]);
|
|
79
|
+
const xml = /^\s*<\?xml[^>]*?\bencoding\s*=\s*["']([^"']+)["']/i.exec(head);
|
|
80
|
+
if (xml !== null) return xml[1];
|
|
81
|
+
const markup = head.replace(/<!--[\s\S]*?-->/g, "");
|
|
82
|
+
return /<meta\b[^>]*?\bcharset\s*=\s*["']?\s*([A-Za-z0-9_:.+-]+)/i.exec(markup)?.[1];
|
|
83
|
+
};
|
|
84
|
+
/**
|
|
85
|
+
* This function counts the UTF-8 sequences of more than one byte, and the bytes that no UTF-8
|
|
86
|
+
* sequence explains. An incomplete sequence at the end is no error: a body cut at its size limit
|
|
87
|
+
* ends so.
|
|
88
|
+
*/
|
|
89
|
+
const utf8Evidence = (bytes) => {
|
|
90
|
+
let multibyte = 0;
|
|
91
|
+
let invalid = 0;
|
|
92
|
+
let index = 0;
|
|
93
|
+
while (index < bytes.length) {
|
|
94
|
+
const lead = bytes[index];
|
|
95
|
+
if (lead < 128) {
|
|
96
|
+
index += 1;
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
const need = lead >= 194 && lead <= 223 ? 1 : lead >= 224 && lead <= 239 ? 2 : lead >= 240 && lead <= 244 ? 3 : 0;
|
|
100
|
+
if (need === 0) {
|
|
101
|
+
invalid += 1;
|
|
102
|
+
index += 1;
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
const low = lead === 224 ? 160 : lead === 240 ? 144 : 128;
|
|
106
|
+
const high = lead === 237 ? 159 : lead === 244 ? 143 : 191;
|
|
107
|
+
let ok = true;
|
|
108
|
+
let at = 1;
|
|
109
|
+
while (at <= need && index + at < bytes.length) {
|
|
110
|
+
const next = bytes[index + at];
|
|
111
|
+
if (at === 1 ? next < low || next > high : next < 128 || next > 191) {
|
|
112
|
+
ok = false;
|
|
113
|
+
break;
|
|
114
|
+
}
|
|
115
|
+
at += 1;
|
|
116
|
+
}
|
|
117
|
+
if (ok && at <= need) break;
|
|
118
|
+
if (ok) {
|
|
119
|
+
multibyte += 1;
|
|
120
|
+
index += need + 1;
|
|
121
|
+
} else {
|
|
122
|
+
invalid += 1;
|
|
123
|
+
index += 1;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return {
|
|
127
|
+
multibyte,
|
|
128
|
+
invalid
|
|
129
|
+
};
|
|
130
|
+
};
|
|
131
|
+
/**
|
|
132
|
+
* This function gives the WHATWG name of the encoding of a body, for example `utf-8` or
|
|
133
|
+
* `windows-1252`. The function `decodeBody` uses it.
|
|
134
|
+
*/
|
|
135
|
+
const bodyEncoding = (bytes, contentType) => {
|
|
136
|
+
const bom = bomOf(bytes);
|
|
137
|
+
if (bom !== void 0) return bom;
|
|
138
|
+
const header = charsetOf(contentType);
|
|
139
|
+
const fromHeader = header === void 0 ? void 0 : encodingOf(header);
|
|
140
|
+
if (fromHeader !== void 0) return fromHeader;
|
|
141
|
+
if (contentType === null || /html|xml/i.test(contentType)) {
|
|
142
|
+
const label = declaredCharset(bytes);
|
|
143
|
+
let declared = label === void 0 ? void 0 : encodingOf(label);
|
|
144
|
+
if (declared?.startsWith("utf-16") === true) declared = "utf-8";
|
|
145
|
+
if (declared === "x-user-defined") declared = "windows-1252";
|
|
146
|
+
if (declared === "utf-8") return declared;
|
|
147
|
+
if (declared !== void 0) {
|
|
148
|
+
const evidence = utf8Evidence(bytes);
|
|
149
|
+
return evidence.invalid === 0 && evidence.multibyte > 0 ? "utf-8" : declared;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if (/json/i.test(contentType ?? "")) return "utf-8";
|
|
153
|
+
const evidence = utf8Evidence(bytes);
|
|
154
|
+
return evidence.invalid > 0 && evidence.invalid >= evidence.multibyte ? "windows-1252" : "utf-8";
|
|
155
|
+
};
|
|
156
|
+
/**
|
|
157
|
+
* These are the characters of the bytes 0x80 to 0x9F in windows-1252, from the WHATWG index.
|
|
158
|
+
* Node.js 22 decodes windows-1252 as ISO-8859-1, which gives a control character for each of
|
|
159
|
+
* these bytes. A later runtime gives the correct characters, and then the map changes nothing.
|
|
160
|
+
*/
|
|
161
|
+
const WINDOWS_1252_HIGH = [
|
|
162
|
+
8364,
|
|
163
|
+
129,
|
|
164
|
+
8218,
|
|
165
|
+
402,
|
|
166
|
+
8222,
|
|
167
|
+
8230,
|
|
168
|
+
8224,
|
|
169
|
+
8225,
|
|
170
|
+
710,
|
|
171
|
+
8240,
|
|
172
|
+
352,
|
|
173
|
+
8249,
|
|
174
|
+
338,
|
|
175
|
+
141,
|
|
176
|
+
381,
|
|
177
|
+
143,
|
|
178
|
+
144,
|
|
179
|
+
8216,
|
|
180
|
+
8217,
|
|
181
|
+
8220,
|
|
182
|
+
8221,
|
|
183
|
+
8226,
|
|
184
|
+
8211,
|
|
185
|
+
8212,
|
|
186
|
+
732,
|
|
187
|
+
8482,
|
|
188
|
+
353,
|
|
189
|
+
8250,
|
|
190
|
+
339,
|
|
191
|
+
157,
|
|
192
|
+
382,
|
|
193
|
+
376
|
|
194
|
+
];
|
|
195
|
+
const C1_CONTROLS = new RegExp(`[${String.fromCharCode(128)}-${String.fromCharCode(159)}]`, "g");
|
|
196
|
+
const repairWindows1252 = (text) => text.replace(C1_CONTROLS, (char) => String.fromCharCode(WINDOWS_1252_HIGH[char.charCodeAt(0) - 128]));
|
|
197
|
+
/**
|
|
198
|
+
* This function decodes the body of a response. It reads the encoding from the byte order mark,
|
|
199
|
+
* the `Content-Type` header and the document, with `bodyEncoding`. It does not throw.
|
|
200
|
+
*/
|
|
201
|
+
const decodeBody = (bytes, contentType) => {
|
|
202
|
+
const encoding = bodyEncoding(bytes, contentType);
|
|
203
|
+
try {
|
|
204
|
+
const text = new TextDecoder(encoding, { fatal: false }).decode(bytes);
|
|
205
|
+
return encoding === "windows-1252" ? repairWindows1252(text) : text;
|
|
206
|
+
} catch {
|
|
207
|
+
return new TextDecoder("utf-8", { fatal: false }).decode(bytes);
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
//#endregion
|
|
211
|
+
//#region src/html.ts
|
|
212
|
+
const SKIP = /* @__PURE__ */ new Set([
|
|
213
|
+
"script",
|
|
214
|
+
"style",
|
|
215
|
+
"noscript",
|
|
216
|
+
"template",
|
|
217
|
+
"svg",
|
|
218
|
+
"math",
|
|
219
|
+
"title",
|
|
220
|
+
"meta",
|
|
221
|
+
"link",
|
|
222
|
+
"base",
|
|
223
|
+
"iframe",
|
|
224
|
+
"frame",
|
|
225
|
+
"object",
|
|
226
|
+
"embed",
|
|
227
|
+
"canvas",
|
|
228
|
+
"audio",
|
|
229
|
+
"video",
|
|
230
|
+
"datalist",
|
|
231
|
+
"param",
|
|
232
|
+
"source",
|
|
233
|
+
"track"
|
|
234
|
+
]);
|
|
235
|
+
/** These elements show as blocks: each starts a line. The list follows the default style sheet of HTML. */
|
|
236
|
+
const BLOCK = /* @__PURE__ */ new Set([
|
|
237
|
+
"address",
|
|
238
|
+
"article",
|
|
239
|
+
"aside",
|
|
240
|
+
"blockquote",
|
|
241
|
+
"body",
|
|
242
|
+
"br",
|
|
243
|
+
"caption",
|
|
244
|
+
"center",
|
|
245
|
+
"dd",
|
|
246
|
+
"details",
|
|
247
|
+
"dialog",
|
|
248
|
+
"dir",
|
|
249
|
+
"div",
|
|
250
|
+
"dl",
|
|
251
|
+
"dt",
|
|
252
|
+
"fieldset",
|
|
253
|
+
"figcaption",
|
|
254
|
+
"figure",
|
|
255
|
+
"footer",
|
|
256
|
+
"form",
|
|
257
|
+
"frameset",
|
|
258
|
+
"h1",
|
|
259
|
+
"h2",
|
|
260
|
+
"h3",
|
|
261
|
+
"h4",
|
|
262
|
+
"h5",
|
|
263
|
+
"h6",
|
|
264
|
+
"head",
|
|
265
|
+
"header",
|
|
266
|
+
"hgroup",
|
|
267
|
+
"hr",
|
|
268
|
+
"html",
|
|
269
|
+
"legend",
|
|
270
|
+
"li",
|
|
271
|
+
"listing",
|
|
272
|
+
"main",
|
|
273
|
+
"marquee",
|
|
274
|
+
"menu",
|
|
275
|
+
"nav",
|
|
276
|
+
"noframes",
|
|
277
|
+
"ol",
|
|
278
|
+
"optgroup",
|
|
279
|
+
"option",
|
|
280
|
+
"p",
|
|
281
|
+
"plaintext",
|
|
282
|
+
"pre",
|
|
283
|
+
"search",
|
|
284
|
+
"section",
|
|
285
|
+
"summary",
|
|
286
|
+
"table",
|
|
287
|
+
"tbody",
|
|
288
|
+
"tfoot",
|
|
289
|
+
"thead",
|
|
290
|
+
"tr",
|
|
291
|
+
"ul",
|
|
292
|
+
"xmp"
|
|
293
|
+
]);
|
|
294
|
+
/**
|
|
295
|
+
* These elements run inline with the text around them. Any other element, a custom element too,
|
|
296
|
+
* ends a word, so its text stays apart from the next word.
|
|
297
|
+
*/
|
|
298
|
+
const INLINE = /* @__PURE__ */ new Set([
|
|
299
|
+
"a",
|
|
300
|
+
"abbr",
|
|
301
|
+
"acronym",
|
|
302
|
+
"b",
|
|
303
|
+
"bdi",
|
|
304
|
+
"bdo",
|
|
305
|
+
"big",
|
|
306
|
+
"blink",
|
|
307
|
+
"cite",
|
|
308
|
+
"code",
|
|
309
|
+
"data",
|
|
310
|
+
"del",
|
|
311
|
+
"dfn",
|
|
312
|
+
"em",
|
|
313
|
+
"font",
|
|
314
|
+
"i",
|
|
315
|
+
"img",
|
|
316
|
+
"ins",
|
|
317
|
+
"kbd",
|
|
318
|
+
"label",
|
|
319
|
+
"mark",
|
|
320
|
+
"nobr",
|
|
321
|
+
"picture",
|
|
322
|
+
"q",
|
|
323
|
+
"rb",
|
|
324
|
+
"rp",
|
|
325
|
+
"rt",
|
|
326
|
+
"rtc",
|
|
327
|
+
"ruby",
|
|
328
|
+
"s",
|
|
329
|
+
"samp",
|
|
330
|
+
"slot",
|
|
331
|
+
"small",
|
|
332
|
+
"span",
|
|
333
|
+
"strike",
|
|
334
|
+
"strong",
|
|
335
|
+
"sub",
|
|
336
|
+
"sup",
|
|
337
|
+
"time",
|
|
338
|
+
"tt",
|
|
339
|
+
"u",
|
|
340
|
+
"var",
|
|
341
|
+
"wbr"
|
|
342
|
+
]);
|
|
343
|
+
/** A table row becomes one line ("Sunday 10 am–5 pm"), with a space between its cells. */
|
|
344
|
+
const CELL = /* @__PURE__ */ new Set(["td", "th"]);
|
|
345
|
+
const WORD_END = /* @__PURE__ */ new Set(["a", "button"]);
|
|
346
|
+
/** A page shorter than this keeps all its text. On a longer page, the text of Readability comes first. */
|
|
347
|
+
const SHORT_PAGE_CHARS = 6e3;
|
|
348
|
+
const MAX_TEXT_CHARS = 5e4;
|
|
349
|
+
/** On a long page, this part of the text is kept for the lines after the article: a footer keeps its hours. */
|
|
350
|
+
const REST_RESERVE_CHARS = 12e3;
|
|
351
|
+
const MAX_LINKS = 300;
|
|
352
|
+
/** Readability takes seconds on a page with more elements or links than this, and more time again on deep markup. */
|
|
353
|
+
const READABILITY_MAX_ELEMENTS = 1e4;
|
|
354
|
+
const READABILITY_MAX_LINKS = 1500;
|
|
355
|
+
const READABILITY_MAX_DEPTH = 150;
|
|
356
|
+
/**
|
|
357
|
+
* The parser of linkedom takes time in proportion to the square of the depth of the markup: 200,000
|
|
358
|
+
* levels take 30 seconds. The reader reads the tags below this depth as text.
|
|
359
|
+
*/
|
|
360
|
+
const MAX_DEPTH = 1e3;
|
|
361
|
+
const parse = (html) => parseHTML(html).document;
|
|
362
|
+
/** These elements have no end tag. */
|
|
363
|
+
const VOID = /* @__PURE__ */ new Set([
|
|
364
|
+
"area",
|
|
365
|
+
"base",
|
|
366
|
+
"basefont",
|
|
367
|
+
"bgsound",
|
|
368
|
+
"br",
|
|
369
|
+
"col",
|
|
370
|
+
"embed",
|
|
371
|
+
"frame",
|
|
372
|
+
"hr",
|
|
373
|
+
"img",
|
|
374
|
+
"input",
|
|
375
|
+
"keygen",
|
|
376
|
+
"link",
|
|
377
|
+
"meta",
|
|
378
|
+
"param",
|
|
379
|
+
"source",
|
|
380
|
+
"track",
|
|
381
|
+
"wbr"
|
|
382
|
+
]);
|
|
383
|
+
/** The content of these elements is text up to their end tag, not markup. */
|
|
384
|
+
const RAW_TEXT = /* @__PURE__ */ new Set([
|
|
385
|
+
"iframe",
|
|
386
|
+
"noembed",
|
|
387
|
+
"noframes",
|
|
388
|
+
"noscript",
|
|
389
|
+
"plaintext",
|
|
390
|
+
"script",
|
|
391
|
+
"style",
|
|
392
|
+
"textarea",
|
|
393
|
+
"title",
|
|
394
|
+
"xmp"
|
|
395
|
+
]);
|
|
396
|
+
/** A new element of one of these kinds closes the open one ("<li>one<li>two"). */
|
|
397
|
+
const IMPLIED_END = /* @__PURE__ */ new Set([
|
|
398
|
+
"dd",
|
|
399
|
+
"dt",
|
|
400
|
+
"li",
|
|
401
|
+
"optgroup",
|
|
402
|
+
"option",
|
|
403
|
+
"p",
|
|
404
|
+
"rb",
|
|
405
|
+
"rp",
|
|
406
|
+
"rt",
|
|
407
|
+
"rtc",
|
|
408
|
+
"tbody",
|
|
409
|
+
"td",
|
|
410
|
+
"tfoot",
|
|
411
|
+
"th",
|
|
412
|
+
"thead",
|
|
413
|
+
"tr"
|
|
414
|
+
]);
|
|
415
|
+
const isLetter = (code) => code >= 97 && code <= 122 || code >= 65 && code <= 90;
|
|
416
|
+
const isNameChar = (code) => isLetter(code) || code >= 48 && code <= 57 || code === 45 || code === 58 || code === 95 || code === 46;
|
|
417
|
+
/** This function gives the index after the `>` of a tag. A `>` inside a quoted attribute value does not end it. */
|
|
418
|
+
const endOfTag = (html, from) => {
|
|
419
|
+
let quote = 0;
|
|
420
|
+
let afterEquals = false;
|
|
421
|
+
for (let at = from; at < html.length; at += 1) {
|
|
422
|
+
const code = html.charCodeAt(at);
|
|
423
|
+
if (quote !== 0) {
|
|
424
|
+
if (code === quote) quote = 0;
|
|
425
|
+
} else if ((code === 34 || code === 39) && afterEquals) quote = code;
|
|
426
|
+
else if (code === 62) return at + 1;
|
|
427
|
+
if (code === 61) afterEquals = true;
|
|
428
|
+
else if (code !== 32 && code !== 9 && code !== 10 && code !== 13) afterEquals = false;
|
|
429
|
+
}
|
|
430
|
+
return html.length;
|
|
431
|
+
};
|
|
432
|
+
/**
|
|
433
|
+
* This function reads the tags below `MAX_DEPTH` levels as text: a block tag becomes a line break
|
|
434
|
+
* and another tag becomes nothing. It counts the open elements roughly, in one pass. When it
|
|
435
|
+
* removes nothing, it gives the same string.
|
|
436
|
+
*/
|
|
437
|
+
const flattenDeepMarkup = (html) => {
|
|
438
|
+
const stack = [];
|
|
439
|
+
const open = /* @__PURE__ */ new Map();
|
|
440
|
+
const extra = /* @__PURE__ */ new Map();
|
|
441
|
+
const out = [];
|
|
442
|
+
let copied = 0;
|
|
443
|
+
let dropped = 0;
|
|
444
|
+
const popTo = (name) => {
|
|
445
|
+
if ((open.get(name) ?? 0) === 0) return;
|
|
446
|
+
for (;;) {
|
|
447
|
+
const top = stack.pop();
|
|
448
|
+
open.set(top, (open.get(top) ?? 1) - 1);
|
|
449
|
+
if (top === name) return;
|
|
450
|
+
}
|
|
451
|
+
};
|
|
452
|
+
const replace = (start, end, name) => {
|
|
453
|
+
out.push(html.slice(copied, start), BLOCK.has(name) ? "\n" : INLINE.has(name) ? "" : " ");
|
|
454
|
+
copied = end;
|
|
455
|
+
};
|
|
456
|
+
let index = 0;
|
|
457
|
+
while (index < html.length) {
|
|
458
|
+
const lt = html.indexOf("<", index);
|
|
459
|
+
if (lt === -1) break;
|
|
460
|
+
if (html.startsWith("<!--", lt)) {
|
|
461
|
+
const end = html.indexOf("-->", lt + 4);
|
|
462
|
+
index = end === -1 ? html.length : end + 3;
|
|
463
|
+
continue;
|
|
464
|
+
}
|
|
465
|
+
const next = html.charCodeAt(lt + 1);
|
|
466
|
+
if (next === 33 || next === 63) {
|
|
467
|
+
const end = html.indexOf(">", lt);
|
|
468
|
+
index = end === -1 ? html.length : end + 1;
|
|
469
|
+
continue;
|
|
470
|
+
}
|
|
471
|
+
const closing = next === 47;
|
|
472
|
+
const nameStart = lt + (closing ? 2 : 1);
|
|
473
|
+
if (!isLetter(html.charCodeAt(nameStart))) {
|
|
474
|
+
index = lt + 1;
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
let nameEnd = nameStart + 1;
|
|
478
|
+
while (nameEnd < html.length && isNameChar(html.charCodeAt(nameEnd))) nameEnd += 1;
|
|
479
|
+
const name = html.slice(nameStart, nameEnd).toLowerCase();
|
|
480
|
+
const tagEnd = endOfTag(html, nameEnd);
|
|
481
|
+
index = tagEnd;
|
|
482
|
+
if (closing) {
|
|
483
|
+
const removed = extra.get(name) ?? 0;
|
|
484
|
+
if (removed > 0) {
|
|
485
|
+
extra.set(name, removed - 1);
|
|
486
|
+
replace(lt, tagEnd, name);
|
|
487
|
+
} else popTo(name);
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
if (RAW_TEXT.has(name)) {
|
|
491
|
+
const end = new RegExp(`</${name}[\\s/>]`, "gi");
|
|
492
|
+
end.lastIndex = tagEnd;
|
|
493
|
+
index = end.exec(html)?.index ?? html.length;
|
|
494
|
+
continue;
|
|
495
|
+
}
|
|
496
|
+
if (VOID.has(name) || html.charCodeAt(tagEnd - 2) === 47) continue;
|
|
497
|
+
if (IMPLIED_END.has(name)) popTo(name);
|
|
498
|
+
if (stack.length >= MAX_DEPTH) {
|
|
499
|
+
extra.set(name, (extra.get(name) ?? 0) + 1);
|
|
500
|
+
dropped += 1;
|
|
501
|
+
replace(lt, tagEnd, name);
|
|
502
|
+
continue;
|
|
503
|
+
}
|
|
504
|
+
stack.push(name);
|
|
505
|
+
open.set(name, (open.get(name) ?? 0) + 1);
|
|
506
|
+
}
|
|
507
|
+
if (dropped === 0) return {
|
|
508
|
+
html,
|
|
509
|
+
dropped
|
|
510
|
+
};
|
|
511
|
+
out.push(html.slice(copied));
|
|
512
|
+
return {
|
|
513
|
+
html: out.join(""),
|
|
514
|
+
dropped
|
|
515
|
+
};
|
|
516
|
+
};
|
|
517
|
+
const attribute = (node, name) => typeof node.getAttribute === "function" ? node.getAttribute(name) : null;
|
|
518
|
+
/** A `<template shadowrootmode>` holds a declarative shadow root: the browser shows its content. */
|
|
519
|
+
const isShadowRoot = (node) => attribute(node, "shadowrootmode") !== null || attribute(node, "shadowroot") !== null;
|
|
520
|
+
/** The reader reads only these properties of an inline style. */
|
|
521
|
+
const STYLE_PROPERTIES = /* @__PURE__ */ new Set(["display", "font-size"]);
|
|
522
|
+
/**
|
|
523
|
+
* This function reads `display` and `font-size` from the `style` attribute of an element. The last
|
|
524
|
+
* declaration wins, and an `!important` declaration wins over a declaration that is not important.
|
|
525
|
+
* It reads no style sheet. Its time grows in proportion to the length of the attribute.
|
|
526
|
+
*/
|
|
527
|
+
const inlineStyle = (node) => {
|
|
528
|
+
const found = /* @__PURE__ */ new Map();
|
|
529
|
+
const style = attribute(node, "style");
|
|
530
|
+
if (style === null || style.length === 0) return found;
|
|
531
|
+
const important = /* @__PURE__ */ new Set();
|
|
532
|
+
for (const declaration of style.split(";")) {
|
|
533
|
+
const colon = declaration.indexOf(":");
|
|
534
|
+
if (colon === -1) continue;
|
|
535
|
+
const property = declaration.slice(0, colon).trim().toLowerCase();
|
|
536
|
+
if (!STYLE_PROPERTIES.has(property)) continue;
|
|
537
|
+
let value = declaration.slice(colon + 1).trim().toLowerCase();
|
|
538
|
+
const bang = value.lastIndexOf("!");
|
|
539
|
+
const strong = bang !== -1 && value.slice(bang + 1).trim() === "important";
|
|
540
|
+
if (strong) value = value.slice(0, bang).trim();
|
|
541
|
+
if (important.has(property) && !strong) continue;
|
|
542
|
+
if (strong) important.add(property);
|
|
543
|
+
found.set(property, value);
|
|
544
|
+
}
|
|
545
|
+
return found;
|
|
546
|
+
};
|
|
547
|
+
/**
|
|
548
|
+
* This function tells if a reader of the page does not get an element and its content. A browser
|
|
549
|
+
* does not show an element with `display: none` or with the attribute `hidden` (unless its inline
|
|
550
|
+
* style sets a different display). A screen reader skips an element with `aria-hidden="true"`, and
|
|
551
|
+
* a model reads a page as a screen reader does.
|
|
552
|
+
*/
|
|
553
|
+
const notRendered = (node, style) => {
|
|
554
|
+
const display = style.get("display");
|
|
555
|
+
if (display === "none") return true;
|
|
556
|
+
if (display === void 0 && attribute(node, "hidden") !== null) return true;
|
|
557
|
+
return attribute(node, "aria-hidden")?.trim().toLowerCase() === "true";
|
|
558
|
+
};
|
|
559
|
+
/** A font size of zero: `0`, `0px`, `0em` or `0%`. */
|
|
560
|
+
const ZERO_SIZE = /^\+?(?:0+(?:\.0*)?|\.0+)(?:[a-z]{1,4}|%)?$/;
|
|
561
|
+
/** These sizes are a part of the size of the parent, so a part of zero stays zero. */
|
|
562
|
+
const RELATIVE_SIZE = /^(?:\+?[\d.]+(?:em|ex|ch|cap|ic|lh|%)|larger|smaller|inherit|unset|revert|revert-layer)$/;
|
|
563
|
+
/** This function tells if the text of an element has a zero size. A child that sets a size of its own shows its text again. */
|
|
564
|
+
const zeroSize = (fontSize, parentZero) => {
|
|
565
|
+
if (fontSize === void 0) return parentZero;
|
|
566
|
+
if (ZERO_SIZE.test(fontSize)) return true;
|
|
567
|
+
return parentZero && RELATIVE_SIZE.test(fontSize);
|
|
568
|
+
};
|
|
569
|
+
/**
|
|
570
|
+
* This function walks a tree with a stack of its own. Thus the depth of the markup cannot overflow
|
|
571
|
+
* the stack. It reads only the text that a reader gets (refer to `notRendered` and `zeroSize`), and it
|
|
572
|
+
* gives the nodes that it did not read.
|
|
573
|
+
*/
|
|
574
|
+
const walkText = (root) => {
|
|
575
|
+
const parts = [];
|
|
576
|
+
const unseen = [];
|
|
577
|
+
let seam = false;
|
|
578
|
+
let elements = 0;
|
|
579
|
+
let links = 0;
|
|
580
|
+
let depth = 0;
|
|
581
|
+
const stack = [];
|
|
582
|
+
const enter = (node) => {
|
|
583
|
+
if (node.nodeType === 3) {
|
|
584
|
+
if (stack[stack.length - 1]?.zero === true) {
|
|
585
|
+
unseen.push(node);
|
|
586
|
+
return;
|
|
587
|
+
}
|
|
588
|
+
const text = node.textContent ?? "";
|
|
589
|
+
if (seam && /^[^\s.,;:!?)\]–—-]/.test(text)) parts.push(" ");
|
|
590
|
+
if (text.length > 0) seam = false;
|
|
591
|
+
parts.push(text);
|
|
592
|
+
return;
|
|
593
|
+
}
|
|
594
|
+
if (node.nodeType !== 1 && node.nodeType !== 9 && node.nodeType !== 11) return;
|
|
595
|
+
const name = node.nodeName.toLowerCase();
|
|
596
|
+
let container = node;
|
|
597
|
+
if (name === "template") {
|
|
598
|
+
if (!isShadowRoot(node)) return;
|
|
599
|
+
container = node.content ?? node;
|
|
600
|
+
} else if (SKIP.has(name)) return;
|
|
601
|
+
let zero = stack[stack.length - 1]?.zero ?? false;
|
|
602
|
+
if (node.nodeType === 1) {
|
|
603
|
+
const style = inlineStyle(node);
|
|
604
|
+
if (notRendered(node, style)) {
|
|
605
|
+
unseen.push(node);
|
|
606
|
+
return;
|
|
607
|
+
}
|
|
608
|
+
zero = zeroSize(style.get("font-size"), zero);
|
|
609
|
+
elements += 1;
|
|
610
|
+
if (name === "a") links += 1;
|
|
611
|
+
if (BLOCK.has(name)) parts.push("\n");
|
|
612
|
+
else if (!INLINE.has(name) && !CELL.has(name)) seam = true;
|
|
613
|
+
}
|
|
614
|
+
stack.push(container.nodeType === 1 ? {
|
|
615
|
+
name,
|
|
616
|
+
next: container.firstChild,
|
|
617
|
+
list: void 0,
|
|
618
|
+
index: 0,
|
|
619
|
+
zero
|
|
620
|
+
} : {
|
|
621
|
+
name,
|
|
622
|
+
next: void 0,
|
|
623
|
+
list: container.childNodes,
|
|
624
|
+
index: 0,
|
|
625
|
+
zero
|
|
626
|
+
});
|
|
627
|
+
if (stack.length > depth) depth = stack.length;
|
|
628
|
+
};
|
|
629
|
+
const leave = (name) => {
|
|
630
|
+
const block = BLOCK.has(name);
|
|
631
|
+
if (block) parts.push("\n");
|
|
632
|
+
else if (CELL.has(name)) parts.push(" ");
|
|
633
|
+
if (block) seam = false;
|
|
634
|
+
else if (WORD_END.has(name) || !INLINE.has(name)) seam = true;
|
|
635
|
+
};
|
|
636
|
+
enter(root);
|
|
637
|
+
while (stack.length > 0) {
|
|
638
|
+
const frame = stack[stack.length - 1];
|
|
639
|
+
let child;
|
|
640
|
+
if (frame.list !== void 0) {
|
|
641
|
+
child = frame.index < frame.list.length ? frame.list[frame.index] : void 0;
|
|
642
|
+
frame.index += 1;
|
|
643
|
+
} else {
|
|
644
|
+
child = frame.next;
|
|
645
|
+
if (child !== null && child !== void 0) frame.next = child.nextSibling;
|
|
646
|
+
}
|
|
647
|
+
if (child === null || child === void 0) {
|
|
648
|
+
stack.pop();
|
|
649
|
+
leave(frame.name);
|
|
650
|
+
} else enter(child);
|
|
651
|
+
}
|
|
652
|
+
return {
|
|
653
|
+
text: parts.join("").split("\n").map((line) => line.replace(/[-]/g, "").replace(/\s+/g, " ").trim()).filter((line) => line.length > 0).join("\n"),
|
|
654
|
+
elements,
|
|
655
|
+
links,
|
|
656
|
+
depth,
|
|
657
|
+
unseen
|
|
658
|
+
};
|
|
659
|
+
};
|
|
660
|
+
/** This function gives the text that a reader sees, with one line for each block. */
|
|
661
|
+
const visibleText = (root) => walkText(root).text;
|
|
662
|
+
const WRAPPERS = [
|
|
663
|
+
[/^<!--/, /-->$/],
|
|
664
|
+
[/^\/\/\s{0,8}<!\[CDATA\[/, /\/\/\s{0,8}\]\]>$/],
|
|
665
|
+
[/^\/\*\s{0,8}<!\[CDATA\[\s{0,8}\*\//, /\/\*\s{0,8}\]\]>\s{0,8}\*\/$/],
|
|
666
|
+
[/^<!\[CDATA\[/, /\]\]>$/]
|
|
667
|
+
];
|
|
668
|
+
const unwrap = (source) => {
|
|
669
|
+
let text = source.replace(/^/, "").trim();
|
|
670
|
+
for (const [open, close] of WRAPPERS) {
|
|
671
|
+
text = text.replace(open, "").trim();
|
|
672
|
+
text = text.replace(close, "").trim();
|
|
673
|
+
}
|
|
674
|
+
return text;
|
|
675
|
+
};
|
|
676
|
+
/**
|
|
677
|
+
* This function cuts a script into its top-level JSON values. It also repairs two common faults: a
|
|
678
|
+
* comma before a closing bracket, and a raw control character inside a string.
|
|
679
|
+
*/
|
|
680
|
+
const jsonValues = (source) => {
|
|
681
|
+
const values = [];
|
|
682
|
+
let out = [];
|
|
683
|
+
let depth = 0;
|
|
684
|
+
let inString = false;
|
|
685
|
+
let escaped = false;
|
|
686
|
+
let pendingComma = false;
|
|
687
|
+
for (let index = 0; index < source.length; index += 1) {
|
|
688
|
+
const char = source[index];
|
|
689
|
+
const code = source.charCodeAt(index);
|
|
690
|
+
if (inString) {
|
|
691
|
+
if (escaped) escaped = false;
|
|
692
|
+
else if (char === "\\") escaped = true;
|
|
693
|
+
else if (char === "\"") inString = false;
|
|
694
|
+
if (depth === 0) continue;
|
|
695
|
+
out.push(code >= 32 ? char : char === "\n" ? "\\n" : char === "\r" ? "\\r" : char === " " ? "\\t" : `\\u${code.toString(16).padStart(4, "0")}`);
|
|
696
|
+
continue;
|
|
697
|
+
}
|
|
698
|
+
if (depth === 0) {
|
|
699
|
+
if (char === "{" || char === "[") {
|
|
700
|
+
depth = 1;
|
|
701
|
+
out = [char];
|
|
702
|
+
pendingComma = false;
|
|
703
|
+
} else if (char === "\"") inString = true;
|
|
704
|
+
continue;
|
|
705
|
+
}
|
|
706
|
+
if (code === 32 || code === 9 || code === 10 || code === 13) {
|
|
707
|
+
out.push(char);
|
|
708
|
+
continue;
|
|
709
|
+
}
|
|
710
|
+
if (char === ",") {
|
|
711
|
+
if (pendingComma) out.push(",");
|
|
712
|
+
pendingComma = true;
|
|
713
|
+
continue;
|
|
714
|
+
}
|
|
715
|
+
if (pendingComma) {
|
|
716
|
+
if (char !== "}" && char !== "]") out.push(",");
|
|
717
|
+
pendingComma = false;
|
|
718
|
+
}
|
|
719
|
+
out.push(char);
|
|
720
|
+
if (char === "\"") inString = true;
|
|
721
|
+
else if (char === "{" || char === "[") depth += 1;
|
|
722
|
+
else if (char === "}" || char === "]") {
|
|
723
|
+
depth -= 1;
|
|
724
|
+
if (depth === 0) {
|
|
725
|
+
values.push(out.join(""));
|
|
726
|
+
out = [];
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
if (depth > 0) values.push(out.join(""));
|
|
731
|
+
return values;
|
|
732
|
+
};
|
|
733
|
+
const parseJsonLd = (sources) => {
|
|
734
|
+
const records = [];
|
|
735
|
+
let unreadable = 0;
|
|
736
|
+
const push = (first) => {
|
|
737
|
+
const stack = [first];
|
|
738
|
+
const found = [];
|
|
739
|
+
while (stack.length > 0) {
|
|
740
|
+
const value = stack.pop();
|
|
741
|
+
if (Array.isArray(value)) {
|
|
742
|
+
for (let index = value.length - 1; index >= 0; index -= 1) stack.push(value[index]);
|
|
743
|
+
continue;
|
|
744
|
+
}
|
|
745
|
+
if (value === null || typeof value !== "object") continue;
|
|
746
|
+
const record = value;
|
|
747
|
+
const graph = record["@graph"];
|
|
748
|
+
if (graph !== void 0 && graph !== null && typeof graph === "object") {
|
|
749
|
+
const { "@graph": _graph, ...rest } = record;
|
|
750
|
+
stack.push(graph);
|
|
751
|
+
if (Object.keys(rest).some((key) => key !== "@context")) found.push(rest);
|
|
752
|
+
continue;
|
|
753
|
+
}
|
|
754
|
+
found.push(record);
|
|
755
|
+
}
|
|
756
|
+
records.push(...found);
|
|
757
|
+
};
|
|
758
|
+
for (const source of sources) {
|
|
759
|
+
const text = unwrap(source);
|
|
760
|
+
if (text.length === 0) continue;
|
|
761
|
+
try {
|
|
762
|
+
push(JSON.parse(text));
|
|
763
|
+
continue;
|
|
764
|
+
} catch {}
|
|
765
|
+
let failed = false;
|
|
766
|
+
let any = false;
|
|
767
|
+
for (const value of jsonValues(text)) try {
|
|
768
|
+
push(JSON.parse(value));
|
|
769
|
+
any = true;
|
|
770
|
+
} catch {
|
|
771
|
+
failed = true;
|
|
772
|
+
}
|
|
773
|
+
if (failed || !any) unreadable += 1;
|
|
774
|
+
}
|
|
775
|
+
return {
|
|
776
|
+
records,
|
|
777
|
+
unreadable
|
|
778
|
+
};
|
|
779
|
+
};
|
|
780
|
+
/**
|
|
781
|
+
* This function reads the JSON-LD blocks and makes arrays and `@graph` flat. It repairs a comma
|
|
782
|
+
* before a closing bracket, a raw line break in a string and two values in one block. It ignores
|
|
783
|
+
* a block that it cannot repair.
|
|
784
|
+
*/
|
|
785
|
+
const readJsonLd = (sources) => parseJsonLd(sources).records;
|
|
786
|
+
/** This function gives the types that a JSON-LD object declares, without the schema.org prefix. */
|
|
787
|
+
const typesOf = (record) => {
|
|
788
|
+
const raw = record["@type"];
|
|
789
|
+
return (Array.isArray(raw) ? raw : raw === void 0 ? [] : [raw]).filter((value) => typeof value === "string").map((value) => value.replace(/^.*[/#]/, "").replace(/^[A-Za-z][\w.+-]*:/, ""));
|
|
790
|
+
};
|
|
791
|
+
const isWeb = (url) => url.protocol === "http:" || url.protocol === "https:";
|
|
792
|
+
const siteOf = (host) => host.toLowerCase().replace(/^www\./, "");
|
|
793
|
+
/** The marks of the pages that a bot check or a firewall sends in place of the content. */
|
|
794
|
+
const CHALLENGES = [
|
|
795
|
+
["Incapsula", /_Incapsula_Resource|Incapsula incident ID/i],
|
|
796
|
+
["Cloudflare", /\/cdn-cgi\/challenge-platform\/|cf-chl-|cf_chl_opt|cf-browser-verification/i],
|
|
797
|
+
["Sucuri", /sucuri_cloudproxy|Sucuri WebSite Firewall/i],
|
|
798
|
+
["DataDome", /captcha-delivery\.com/i],
|
|
799
|
+
["PerimeterX", /_pxCaptcha|px-captcha|captcha\.px-cdn\.net/i],
|
|
800
|
+
["AWS WAF", /AwsWafIntegration|token\.awswaf\.com|\/challenge\.js.{0,80}awswaf/i],
|
|
801
|
+
["DDoS-Guard", /ddos-guard\.net\/|__ddg1_/i]
|
|
802
|
+
];
|
|
803
|
+
const CHALLENGE_TITLE = /^(?:just a moment\.*|attention required!? \| cloudflare|access to this page has been denied\.?|please verify you are a human|are you a robot\??|ddos-guard|checking your browser.{0,40})$/i;
|
|
804
|
+
const ASKS_FOR_SCRIPT = /(?:enable|turn on|activate|switch on|allow)\s+(?:your\s+)?javascript|javascript\s+(?:is\s+)?(?:required|disabled|turned off|not enabled|must be enabled|needs to be enabled)|(?:requires?|needs?)\s+javascript|javascript\s+aktivieren|activ(?:er|ez)\s+(?:le\s+)?javascript|habilit(?:ar|e)\s+(?:el\s+)?javascript/i;
|
|
805
|
+
/** The noscript of a comment widget (Disqus) asks for JavaScript for the comments, not for the page. */
|
|
806
|
+
const COMMENTS = /comments|disqus/i;
|
|
807
|
+
/** These elements hold the whole page of an application that its scripts draw. */
|
|
808
|
+
const APP_ROOTS = "#root, #app, #__next, #__nuxt, #___gatsby, app-root, my-app, [data-reactroot]";
|
|
809
|
+
const EMBEDDED = [
|
|
810
|
+
["next", (_html, doc) => doc.querySelector("script[id=\"__NEXT_DATA__\"]") !== null],
|
|
811
|
+
["next-app", (html) => html.includes("self.__next_f.push")],
|
|
812
|
+
["nuxt", (html) => html.includes("__NUXT_DATA__") || html.includes("window.__NUXT__")],
|
|
813
|
+
["apollo", (html) => html.includes("__APOLLO_STATE__")]
|
|
814
|
+
];
|
|
815
|
+
/** This function reads `content="0; url=/home"`. A refresh with no URL reloads the page, and is no redirect. */
|
|
816
|
+
const refreshTarget = (content) => {
|
|
817
|
+
const text = content.slice(0, 2e3).trim();
|
|
818
|
+
const delay = /^\d+(?:\.\d*)?/.exec(text)?.[0] ?? "";
|
|
819
|
+
const url = text.slice(delay.length).replace(/^[\s;,]+/, "").replace(/^url\s*=\s*/i, "").replace(/^(["'])(.*)\1$/s, "$2").trim();
|
|
820
|
+
return url.length === 0 ? void 0 : {
|
|
821
|
+
delay: Number(delay || 0),
|
|
822
|
+
url
|
|
823
|
+
};
|
|
824
|
+
};
|
|
825
|
+
/** This function cuts text to a length at the end of a line where it can. It does not cut a surrogate pair. */
|
|
826
|
+
const cutHead = (text, length) => {
|
|
827
|
+
if (text.length <= length) return text;
|
|
828
|
+
if (length <= 0) return "";
|
|
829
|
+
const line = text.lastIndexOf("\n", length);
|
|
830
|
+
let end = line > length - 2e3 && line > 0 ? line : length;
|
|
831
|
+
const code = text.charCodeAt(end - 1);
|
|
832
|
+
if (code >= 55296 && code <= 56319) end -= 1;
|
|
833
|
+
return text.slice(0, end);
|
|
834
|
+
};
|
|
835
|
+
/** This function keeps the end of the text, from the start of a line where it can. */
|
|
836
|
+
const cutTail = (text, length) => {
|
|
837
|
+
if (text.length <= length) return text;
|
|
838
|
+
if (length <= 0) return "";
|
|
839
|
+
let start = text.length - length;
|
|
840
|
+
const line = text.indexOf("\n", start);
|
|
841
|
+
if (line !== -1 && line < start + 2e3) start = line + 1;
|
|
842
|
+
const code = text.charCodeAt(start);
|
|
843
|
+
if (code >= 56320 && code <= 57343) start += 1;
|
|
844
|
+
return text.slice(start);
|
|
845
|
+
};
|
|
846
|
+
/** This function keeps the start and the end of a text that is too long: a footer keeps its hours. */
|
|
847
|
+
const capText = (text) => text.length <= MAX_TEXT_CHARS ? text : `${cutHead(text, 37998)}\n\n${cutTail(text, REST_RESERVE_CHARS)}`;
|
|
848
|
+
/** A line that can label the lines below it: short, with no number, and not a sentence ("Opening hours"). */
|
|
849
|
+
const isHeading = (line) => line.length <= 40 && !/[.!?]$/.test(line) && !/[0-9]/.test(line);
|
|
850
|
+
/**
|
|
851
|
+
* This function finds the lines of the article in the lines of the page, one for one and in order.
|
|
852
|
+
* Thus a line that the article and a sidebar share stays in the sidebar ("10 am – 5 pm").
|
|
853
|
+
*/
|
|
854
|
+
const articleLines = (all, main) => {
|
|
855
|
+
const taken = new Uint8Array(all.length);
|
|
856
|
+
const positions = /* @__PURE__ */ new Map();
|
|
857
|
+
all.forEach((line, index) => {
|
|
858
|
+
const list = positions.get(line);
|
|
859
|
+
if (list === void 0) positions.set(line, [index]);
|
|
860
|
+
else list.push(index);
|
|
861
|
+
});
|
|
862
|
+
const cursors = /* @__PURE__ */ new Map();
|
|
863
|
+
let at = 0;
|
|
864
|
+
let matched = 0;
|
|
865
|
+
for (const line of main) {
|
|
866
|
+
const list = positions.get(line);
|
|
867
|
+
if (list === void 0) continue;
|
|
868
|
+
let cursor = cursors.get(line) ?? 0;
|
|
869
|
+
while (cursor < list.length && list[cursor] < at) cursor += 1;
|
|
870
|
+
cursors.set(line, cursor);
|
|
871
|
+
const position = list[cursor];
|
|
872
|
+
if (position === void 0) continue;
|
|
873
|
+
if (matched > 0 && position - at > 400 && list.length > 1) continue;
|
|
874
|
+
taken[position] = 1;
|
|
875
|
+
cursors.set(line, cursor + 1);
|
|
876
|
+
at = position + 1;
|
|
877
|
+
matched += 1;
|
|
878
|
+
}
|
|
879
|
+
return taken;
|
|
880
|
+
};
|
|
881
|
+
/** This function gives the lines of the page that are not in the article. Each run keeps the heading above it. */
|
|
882
|
+
const restOf = (all, taken) => {
|
|
883
|
+
const rest = [];
|
|
884
|
+
for (let index = 0; index < all.length; index += 1) {
|
|
885
|
+
if (taken[index] === 1) continue;
|
|
886
|
+
const above = all[index - 1];
|
|
887
|
+
if (index > 0 && taken[index - 1] === 1 && above !== void 0 && isHeading(above)) rest.push(above);
|
|
888
|
+
rest.push(all[index]);
|
|
889
|
+
}
|
|
890
|
+
return rest;
|
|
891
|
+
};
|
|
892
|
+
const extractPage = (html, pageUrl) => {
|
|
893
|
+
const notes = [];
|
|
894
|
+
const flat = flattenDeepMarkup(html);
|
|
895
|
+
if (flat.dropped > 0) notes.push(`the markup nests deeper than ${String(MAX_DEPTH)} elements, so the reader read the deeper tags as text`);
|
|
896
|
+
const doc = parse(flat.html);
|
|
897
|
+
const meta = (selector) => doc.querySelector(selector)?.getAttribute("content")?.trim() || null;
|
|
898
|
+
const scripts = [...doc.querySelectorAll("script")];
|
|
899
|
+
const parsed = parseJsonLd(scripts.filter((node) => /^\s*application\/ld\+json\s*(?:;|$)/i.test(node.getAttribute("type") ?? "")).map((node) => node.textContent ?? ""));
|
|
900
|
+
const jsonld = parsed.records;
|
|
901
|
+
if (parsed.unreadable > 0) notes.push(`${String(parsed.unreadable)} JSON-LD ${parsed.unreadable === 1 ? "block" : "blocks"} could not be read`);
|
|
902
|
+
const page = (() => {
|
|
903
|
+
try {
|
|
904
|
+
return new URL(pageUrl);
|
|
905
|
+
} catch {
|
|
906
|
+
return;
|
|
907
|
+
}
|
|
908
|
+
})();
|
|
909
|
+
const baseHref = doc.querySelector("base[href]")?.getAttribute("href");
|
|
910
|
+
const base = (() => {
|
|
911
|
+
try {
|
|
912
|
+
const resolved = new URL(baseHref ?? pageUrl, pageUrl);
|
|
913
|
+
return isWeb(resolved) ? resolved.href : pageUrl;
|
|
914
|
+
} catch {
|
|
915
|
+
return pageUrl;
|
|
916
|
+
}
|
|
917
|
+
})();
|
|
918
|
+
const resolve = (raw) => {
|
|
919
|
+
try {
|
|
920
|
+
const url = new URL(raw.trim(), base);
|
|
921
|
+
if (!isWeb(url)) return void 0;
|
|
922
|
+
url.username = "";
|
|
923
|
+
url.password = "";
|
|
924
|
+
return url;
|
|
925
|
+
} catch {
|
|
926
|
+
return;
|
|
927
|
+
}
|
|
928
|
+
};
|
|
929
|
+
const self = page === void 0 ? void 0 : canonicalUrl(page.href);
|
|
930
|
+
const refresh = [...doc.querySelectorAll("meta[http-equiv]")].filter((node) => /^\s*refresh\s*$/i.test(node.getAttribute("http-equiv") ?? "")).map((node) => refreshTarget(node.getAttribute("content") ?? "")).find((target) => target !== void 0);
|
|
931
|
+
const refreshUrl = refresh === void 0 ? void 0 : resolve(refresh.url);
|
|
932
|
+
const redirect = refresh !== void 0 && refresh.delay <= 5 && refreshUrl !== void 0 && canonicalUrl(refreshUrl.href) !== self ? canonicalUrl(refreshUrl.href) : null;
|
|
933
|
+
const canonical = (() => {
|
|
934
|
+
const node = [...doc.querySelectorAll("link[rel][href]")].find((link) => (link.getAttribute("rel") ?? "").toLowerCase().split(/\s+/).includes("canonical"));
|
|
935
|
+
const url = node === void 0 ? void 0 : resolve(node.getAttribute("href") ?? "");
|
|
936
|
+
if (url === void 0 || page === void 0 || siteOf(url.hostname) !== siteOf(page.hostname)) return null;
|
|
937
|
+
return canonicalUrl(url.href);
|
|
938
|
+
})();
|
|
939
|
+
const clean = (value) => (value ?? "").replace(/\s+/g, " ").trim();
|
|
940
|
+
const titles = [...doc.querySelectorAll("title")];
|
|
941
|
+
const inSvg = (node) => {
|
|
942
|
+
for (let parent = node.parentNode; parent; parent = parent.parentNode) if (parent.nodeName.toLowerCase() === "svg") return true;
|
|
943
|
+
return false;
|
|
944
|
+
};
|
|
945
|
+
const title = meta("meta[property=\"og:title\"]") || clean(titles.find((node) => !inSvg(node))?.textContent) || clean(doc.querySelector("h1")?.textContent) || pageUrl;
|
|
946
|
+
const description = meta("meta[name=\"description\"]") ?? meta("meta[property=\"og:description\"]");
|
|
947
|
+
const lang = doc.documentElement?.getAttribute("lang")?.trim() || null;
|
|
948
|
+
const noscript = [...doc.querySelectorAll("noscript")].map((node) => node.textContent ?? "").filter((text) => !COMMENTS.test(text)).join(" ");
|
|
949
|
+
const embedded = EMBEDDED.filter(([, found]) => found(html, doc)).map(([name]) => name);
|
|
950
|
+
const appShell = [...doc.querySelectorAll(APP_ROOTS)].some((node) => (node.textContent ?? "").trim().length === 0);
|
|
951
|
+
const walked = walkText(doc);
|
|
952
|
+
for (const node of walked.unseen) node.remove?.();
|
|
953
|
+
const seen = new Set(self === void 0 ? [] : [self]);
|
|
954
|
+
const links = [];
|
|
955
|
+
if (redirect !== null) {
|
|
956
|
+
seen.add(redirect);
|
|
957
|
+
links.push({
|
|
958
|
+
href: redirect,
|
|
959
|
+
text: ""
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
for (const anchor of doc.querySelectorAll("a[href]")) {
|
|
963
|
+
if (links.length >= MAX_LINKS) break;
|
|
964
|
+
const url = resolve(anchor.getAttribute("href") ?? "");
|
|
965
|
+
if (url === void 0) continue;
|
|
966
|
+
const href = canonicalUrl(url.href);
|
|
967
|
+
const text = (anchor.textContent ?? "").replace(/\s+/g, " ").trim() || anchor.getAttribute("aria-label")?.trim() || anchor.getAttribute("title")?.trim() || anchor.querySelector("img[alt]")?.getAttribute("alt")?.trim() || "";
|
|
968
|
+
if (href === redirect && links[0]?.href === redirect && links[0].text === "") {
|
|
969
|
+
links[0] = {
|
|
970
|
+
href,
|
|
971
|
+
text: text.slice(0, 120)
|
|
972
|
+
};
|
|
973
|
+
continue;
|
|
974
|
+
}
|
|
975
|
+
if (seen.has(href)) continue;
|
|
976
|
+
seen.add(href);
|
|
977
|
+
links.push({
|
|
978
|
+
href,
|
|
979
|
+
text: text.slice(0, 120)
|
|
980
|
+
});
|
|
981
|
+
}
|
|
982
|
+
const all = walked.text;
|
|
983
|
+
let text = capText(all);
|
|
984
|
+
let cut = text.length < all.length;
|
|
985
|
+
if (all.length > SHORT_PAGE_CHARS) {
|
|
986
|
+
const main = (() => {
|
|
987
|
+
if (walked.elements > READABILITY_MAX_ELEMENTS || walked.links > READABILITY_MAX_LINKS || walked.depth > READABILITY_MAX_DEPTH) {
|
|
988
|
+
notes.push("the page is too large for Readability, so its text is in page order");
|
|
989
|
+
return "";
|
|
990
|
+
}
|
|
991
|
+
try {
|
|
992
|
+
const article = new Readability(doc, {
|
|
993
|
+
charThreshold: 500,
|
|
994
|
+
disableJSONLD: true,
|
|
995
|
+
serializer: (node) => node
|
|
996
|
+
}).parse();
|
|
997
|
+
return article?.content === null || article?.content === void 0 ? "" : visibleText(article.content);
|
|
998
|
+
} catch {
|
|
999
|
+
notes.push("Readability could not read the page, so its text is in page order");
|
|
1000
|
+
return "";
|
|
1001
|
+
}
|
|
1002
|
+
})();
|
|
1003
|
+
if (main.length > 500) {
|
|
1004
|
+
const lines = all.split("\n");
|
|
1005
|
+
const rest = restOf(lines, articleLines(lines, main.split("\n"))).join("\n");
|
|
1006
|
+
const head = cutHead(main, MAX_TEXT_CHARS - Math.min(rest.length, REST_RESERVE_CHARS) - 2);
|
|
1007
|
+
const tail = cutHead(rest, MAX_TEXT_CHARS - head.length - 2);
|
|
1008
|
+
text = rest.length > 0 ? `${head}\n\n${tail}` : head;
|
|
1009
|
+
cut = head.length < main.length || tail.length < rest.length;
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
if (cut) notes.push(`the text is cut to ${String(MAX_TEXT_CHARS)} characters`);
|
|
1013
|
+
const visibleChars = all.length;
|
|
1014
|
+
const pageTitle = clean(titles[0]?.textContent);
|
|
1015
|
+
const blocked = visibleChars < 2e3 ? CHALLENGE_TITLE.test(pageTitle) ? CHALLENGES.find(([, pattern]) => pattern.test(html))?.[0] ?? "a bot check" : CHALLENGES.find(([, pattern]) => pattern.test(html))?.[0] ?? null : null;
|
|
1016
|
+
const reason = blocked !== null || redirect !== null ? null : visibleChars < 80 ? `only ${String(visibleChars)} characters of visible text` : visibleChars < 1200 && ASKS_FOR_SCRIPT.test(noscript) ? "the page asks for JavaScript" : visibleChars < 1200 && appShell ? "an empty application root" : null;
|
|
1017
|
+
return {
|
|
1018
|
+
title: title.replace(/\s+/g, " ").slice(0, 300),
|
|
1019
|
+
description,
|
|
1020
|
+
text,
|
|
1021
|
+
...cut ? { fullText: all } : {},
|
|
1022
|
+
jsonld,
|
|
1023
|
+
links,
|
|
1024
|
+
lang,
|
|
1025
|
+
visibleChars,
|
|
1026
|
+
embedded,
|
|
1027
|
+
needsRender: reason !== null,
|
|
1028
|
+
renderReason: reason,
|
|
1029
|
+
canonical,
|
|
1030
|
+
redirect,
|
|
1031
|
+
blocked,
|
|
1032
|
+
notes
|
|
1033
|
+
};
|
|
1034
|
+
};
|
|
1035
|
+
//#endregion
|
|
1036
|
+
export { bodyEncoding as a, canonicalUrl as c, visibleText as i, readJsonLd as n, decodeBody as o, typesOf as r, cacheKeyOf as s, extractPage as t };
|
|
1037
|
+
|
|
1038
|
+
//# sourceMappingURL=html-oNpGAHwi.mjs.map
|