polite-crawler 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +557 -10
- package/build/browser-CKRNbIDT.mjs +62 -0
- package/build/browser-CKRNbIDT.mjs.map +1 -0
- package/build/cli.mjs +795 -121
- package/build/cli.mjs.map +1 -1
- package/build/diff-Dk2eSvi-.mjs +648 -0
- package/build/diff-Dk2eSvi-.mjs.map +1 -0
- package/build/duration-iIuMMofi.mjs +38 -0
- package/build/duration-iIuMMofi.mjs.map +1 -0
- package/build/extract-DcwNrTK7.mjs +1793 -0
- package/build/extract-DcwNrTK7.mjs.map +1 -0
- package/build/{html-nWYFVsRU.d.mts → html-BACljlkf.d.mts} +8 -9
- package/build/{html-oNpGAHwi.mjs → html-DkQUAE_d.mjs} +18 -39
- package/build/html-DkQUAE_d.mjs.map +1 -0
- package/build/html.d.mts +2 -1
- package/build/html.mjs +3 -1
- package/build/index.d.mts +94 -4
- package/build/index.mjs +13 -5
- package/build/log-Dx-wXjJl.mjs +342 -0
- package/build/log-Dx-wXjJl.mjs.map +1 -0
- package/build/mcp.d.mts +7 -4
- package/build/mcp.mjs +556 -32
- package/build/mcp.mjs.map +1 -1
- package/build/open-Ca1lcWR3.d.mts +2875 -0
- package/build/open-DnrWgkC0.mjs +3161 -0
- package/build/open-DnrWgkC0.mjs.map +1 -0
- package/build/{browser-gJ_h4AhB.mjs → render-C1CUraWF.mjs} +31 -74
- package/build/render-C1CUraWF.mjs.map +1 -0
- package/build/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/build/{open-DYZ7KL2A.mjs → runtime-k8mo1_7U.mjs} +986 -1657
- package/build/runtime-k8mo1_7U.mjs.map +1 -0
- package/build/urls-BHFeSyEK.mjs +62 -0
- package/build/urls-BHFeSyEK.mjs.map +1 -0
- package/build/urls-BTDSgDLC.d.mts +16 -0
- package/build/wacz-sfOLT8e9.mjs +275 -0
- package/build/wacz-sfOLT8e9.mjs.map +1 -0
- package/build/warc-VIPscWxY.mjs +728 -0
- package/build/warc-VIPscWxY.mjs.map +1 -0
- package/build/webbotauth-CjJdTAfE.mjs +568 -0
- package/build/webbotauth-CjJdTAfE.mjs.map +1 -0
- package/package.json +1 -1
- package/record.schema.json +15 -1
- package/build/browser-gJ_h4AhB.mjs.map +0 -1
- package/build/html-oNpGAHwi.mjs.map +0 -1
- package/build/open-DYZ7KL2A.mjs.map +0 -1
- package/build/open-eHcriTds.d.mts +0 -1499
- package/build/version-no12qgos.mjs +0 -11
- package/build/version-no12qgos.mjs.map +0 -1
|
@@ -0,0 +1,1793 @@
|
|
|
1
|
+
import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
|
|
2
|
+
import { T as foldText, _ as versionAtTime, b as archivedNote, c as isHeldBack, l as isoTime, n as HELD_BACK, p as recordOverride, w as fold, x as archivedOf } from "./runtime-k8mo1_7U.mjs";
|
|
3
|
+
import { n as canonicalUrl } from "./urls-BHFeSyEK.mjs";
|
|
4
|
+
import { r as typesOf } from "./html-DkQUAE_d.mjs";
|
|
5
|
+
//#region src/extract.ts
|
|
6
|
+
var extract_exports = /* @__PURE__ */ __exportAll({
|
|
7
|
+
FIELD_KINDS: () => FIELD_KINDS,
|
|
8
|
+
checkFields: () => checkFields,
|
|
9
|
+
extractDocument: () => extractDocument
|
|
10
|
+
});
|
|
11
|
+
/** These kinds read the text of the page. The kind `json-ld:<path>` reads the JSON-LD. */
|
|
12
|
+
const FIELD_KINDS = [
|
|
13
|
+
"text",
|
|
14
|
+
"number",
|
|
15
|
+
"money",
|
|
16
|
+
"date",
|
|
17
|
+
"url",
|
|
18
|
+
"email",
|
|
19
|
+
"phone"
|
|
20
|
+
];
|
|
21
|
+
const prepare = (name, text, work) => {
|
|
22
|
+
work.steps += text.length;
|
|
23
|
+
return {
|
|
24
|
+
name,
|
|
25
|
+
text,
|
|
26
|
+
folded: foldText(text, "quote"),
|
|
27
|
+
cache: /* @__PURE__ */ new Map()
|
|
28
|
+
};
|
|
29
|
+
};
|
|
30
|
+
const cached = (doc, key, make) => {
|
|
31
|
+
if (doc.cache.has(key)) return doc.cache.get(key);
|
|
32
|
+
const value = make();
|
|
33
|
+
doc.cache.set(key, value);
|
|
34
|
+
return value;
|
|
35
|
+
};
|
|
36
|
+
/** This function gives the original span of the folded units from `i` to `j`. */
|
|
37
|
+
const spanOf = (doc, i, j) => ({
|
|
38
|
+
start: doc.folded.start[i],
|
|
39
|
+
end: doc.folded.end[j - 1],
|
|
40
|
+
in: doc.name
|
|
41
|
+
});
|
|
42
|
+
const EXCERPT = 60;
|
|
43
|
+
/** This function moves a position off the second half of a surrogate pair. */
|
|
44
|
+
const onBoundary = (text, position, forward) => {
|
|
45
|
+
const code = text.charCodeAt(position);
|
|
46
|
+
if (position > 0 && position < text.length && code >= 56320 && code <= 57343) return forward ? position + 1 : position - 1;
|
|
47
|
+
return position;
|
|
48
|
+
};
|
|
49
|
+
const excerptOf = (text, start, end) => {
|
|
50
|
+
const from = onBoundary(text, Math.max(0, start - EXCERPT), false);
|
|
51
|
+
const to = onBoundary(text, Math.min(text.length, end + EXCERPT), true);
|
|
52
|
+
return `${from > 0 ? "…" : ""}${text.slice(from, to)}${to < text.length ? "…" : ""}`;
|
|
53
|
+
};
|
|
54
|
+
const isDigitCode = (code) => code >= 48 && code <= 57;
|
|
55
|
+
const isDigitAt = (text, at) => at >= 0 && at < text.length && isDigitCode(text.charCodeAt(at));
|
|
56
|
+
const WORD = /^[\p{L}\p{N}]$/u;
|
|
57
|
+
/** This function tells if the unit at a position is a letter or a digit. */
|
|
58
|
+
const isWordAt = (text, at) => {
|
|
59
|
+
if (at < 0 || at >= text.length) return false;
|
|
60
|
+
const code = text.charCodeAt(at);
|
|
61
|
+
if (code < 128) return isDigitCode(code) || code >= 97 && code <= 122 || code >= 65 && code <= 90;
|
|
62
|
+
return WORD.test(text[at]);
|
|
63
|
+
};
|
|
64
|
+
const isLetterAt = (text, at) => isWordAt(text, at) && !isDigitAt(text, at);
|
|
65
|
+
const MAX_CANDIDATES = 5;
|
|
66
|
+
const LABEL_ORDER = [
|
|
67
|
+
"unicode",
|
|
68
|
+
"whitespace",
|
|
69
|
+
"case",
|
|
70
|
+
"thousands-separator",
|
|
71
|
+
"decimal-comma",
|
|
72
|
+
"number-form",
|
|
73
|
+
"currency-symbol",
|
|
74
|
+
"currency-position",
|
|
75
|
+
"amount-only",
|
|
76
|
+
"date-iso",
|
|
77
|
+
"date-ymd",
|
|
78
|
+
"date-dmy",
|
|
79
|
+
"date-mdy",
|
|
80
|
+
"date-numeric",
|
|
81
|
+
"date-written",
|
|
82
|
+
"date-form",
|
|
83
|
+
"date-partial",
|
|
84
|
+
"url-resolved",
|
|
85
|
+
"url-scheme",
|
|
86
|
+
"url-form",
|
|
87
|
+
"email-case",
|
|
88
|
+
"phone-digits",
|
|
89
|
+
"phone-suffix"
|
|
90
|
+
];
|
|
91
|
+
const labelText = (labels) => {
|
|
92
|
+
const set = new Set(labels);
|
|
93
|
+
return LABEL_ORDER.filter((label) => set.has(label)).join("+");
|
|
94
|
+
};
|
|
95
|
+
const empty = (status, kind, note = null) => ({
|
|
96
|
+
status,
|
|
97
|
+
kind,
|
|
98
|
+
span: null,
|
|
99
|
+
path: null,
|
|
100
|
+
raw: null,
|
|
101
|
+
excerpt: null,
|
|
102
|
+
normalization: null,
|
|
103
|
+
reading: null,
|
|
104
|
+
count: 0,
|
|
105
|
+
candidates: [],
|
|
106
|
+
note
|
|
107
|
+
});
|
|
108
|
+
/**
|
|
109
|
+
* This function decides the status of a field from its places. An exact match wins. A normalized
|
|
110
|
+
* match needs one meaning: two meanings, or a place that the text does not make clear, give
|
|
111
|
+
* `ambiguous`. The cost is linear in the number of places.
|
|
112
|
+
*/
|
|
113
|
+
const decide = (kind, claimText, claims, cands, comparator, work) => {
|
|
114
|
+
let exactCount = 0;
|
|
115
|
+
let exactFirst;
|
|
116
|
+
for (const cand of cands) {
|
|
117
|
+
work.steps += 1;
|
|
118
|
+
if (cand.exact === claimText) {
|
|
119
|
+
exactCount += 1;
|
|
120
|
+
exactFirst ??= {
|
|
121
|
+
where: cand.where,
|
|
122
|
+
cand,
|
|
123
|
+
raw: cand.exact
|
|
124
|
+
};
|
|
125
|
+
} else if (cand.alt !== void 0 && cand.alt.exact === claimText) {
|
|
126
|
+
exactCount += 1;
|
|
127
|
+
exactFirst ??= {
|
|
128
|
+
where: cand.alt.where,
|
|
129
|
+
cand,
|
|
130
|
+
raw: cand.alt.exact
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
if (exactFirst !== void 0) {
|
|
135
|
+
const reading = exactFirst.cand.readings[0];
|
|
136
|
+
return {
|
|
137
|
+
...empty("verified", kind),
|
|
138
|
+
span: exactFirst.where.span,
|
|
139
|
+
path: exactFirst.where.path,
|
|
140
|
+
raw: exactFirst.where === exactFirst.cand.where ? exactFirst.cand.raw : exactFirst.raw,
|
|
141
|
+
excerpt: exactFirst.cand.excerpt(),
|
|
142
|
+
reading: exactFirst.cand.readings.length === 1 && reading !== void 0 ? comparator.display(reading) : null,
|
|
143
|
+
count: exactCount
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
const matches = [];
|
|
147
|
+
for (const cand of cands) {
|
|
148
|
+
let best;
|
|
149
|
+
for (const reading of cand.readings) for (const claim of claims) {
|
|
150
|
+
work.steps += 1;
|
|
151
|
+
const compared = comparator.compare(claim, reading);
|
|
152
|
+
if (compared !== void 0 && (best === void 0 || compared.clean && !best.clean)) best = compared;
|
|
153
|
+
}
|
|
154
|
+
if (best === void 0) continue;
|
|
155
|
+
matches.push({
|
|
156
|
+
cand,
|
|
157
|
+
compared: best,
|
|
158
|
+
clean: best.clean && cand.readings.length === 1 && claims.length === 1
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
if (matches.length === 0) return empty("not-found", kind);
|
|
162
|
+
const same = comparator.same ?? ((a, b) => a.key === b.key);
|
|
163
|
+
const candidateOf = (match) => ({
|
|
164
|
+
span: match.cand.where.span,
|
|
165
|
+
path: match.cand.where.path,
|
|
166
|
+
raw: match.cand.raw,
|
|
167
|
+
excerpt: match.cand.excerpt(),
|
|
168
|
+
readings: match.cand.readings.map(comparator.display)
|
|
169
|
+
});
|
|
170
|
+
const clean = matches.filter((match) => match.clean);
|
|
171
|
+
if (clean.length > 0) {
|
|
172
|
+
const meanings = [];
|
|
173
|
+
for (const match of clean) {
|
|
174
|
+
work.steps += 1;
|
|
175
|
+
if (meanings.some((known) => same(known.compared, match.compared))) continue;
|
|
176
|
+
if (meanings.length < MAX_CANDIDATES) meanings.push(match);
|
|
177
|
+
else break;
|
|
178
|
+
}
|
|
179
|
+
const first = clean[0];
|
|
180
|
+
if (meanings.length === 1) return {
|
|
181
|
+
...empty("normalized", kind),
|
|
182
|
+
span: first.cand.where.span,
|
|
183
|
+
path: first.cand.where.path,
|
|
184
|
+
raw: first.cand.raw,
|
|
185
|
+
excerpt: first.cand.excerpt(),
|
|
186
|
+
normalization: labelText(first.compared.labels),
|
|
187
|
+
reading: first.compared.reading,
|
|
188
|
+
count: clean.length
|
|
189
|
+
};
|
|
190
|
+
return {
|
|
191
|
+
...empty("ambiguous", kind, "the value matches places with different meanings, so the check does not choose one"),
|
|
192
|
+
count: clean.length,
|
|
193
|
+
candidates: meanings.map(candidateOf)
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
const note = claims.length > 1 ? "the value itself has more than one meaning: give it in a form with one meaning" : "the stored text matches, but it does not give one meaning, so the check does not choose one";
|
|
197
|
+
return {
|
|
198
|
+
...empty("ambiguous", kind, note),
|
|
199
|
+
count: matches.length,
|
|
200
|
+
candidates: matches.slice(0, MAX_CANDIDATES).map(candidateOf)
|
|
201
|
+
};
|
|
202
|
+
};
|
|
203
|
+
/** This function gives each start of a needle in a text, in linear time (Knuth, Morris and Pratt). */
|
|
204
|
+
const occurrences = (hay, needle, work) => {
|
|
205
|
+
const m = needle.length;
|
|
206
|
+
const fail = new Int32Array(m);
|
|
207
|
+
for (let q = 0, i = 1; i < m; i += 1) {
|
|
208
|
+
while (q > 0 && needle.charCodeAt(i) !== needle.charCodeAt(q)) {
|
|
209
|
+
q = fail[q - 1];
|
|
210
|
+
work.steps += 1;
|
|
211
|
+
}
|
|
212
|
+
if (needle.charCodeAt(i) === needle.charCodeAt(q)) q += 1;
|
|
213
|
+
fail[i] = q;
|
|
214
|
+
work.steps += 1;
|
|
215
|
+
}
|
|
216
|
+
const hits = [];
|
|
217
|
+
for (let q = 0, i = 0; i < hay.length; i += 1) {
|
|
218
|
+
while (q > 0 && hay.charCodeAt(i) !== needle.charCodeAt(q)) {
|
|
219
|
+
q = fail[q - 1];
|
|
220
|
+
work.steps += 1;
|
|
221
|
+
}
|
|
222
|
+
if (hay.charCodeAt(i) === needle.charCodeAt(q)) q += 1;
|
|
223
|
+
work.steps += 1;
|
|
224
|
+
if (q === m) {
|
|
225
|
+
hits.push(i - m + 1);
|
|
226
|
+
q = fail[q - 1];
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return hits;
|
|
230
|
+
};
|
|
231
|
+
const IGNORABLE = /\p{Default_Ignorable_Code_Point}/gu;
|
|
232
|
+
const QUOTES = /[‘’‚‛′]/g;
|
|
233
|
+
const DOUBLE_QUOTES = /[“”„‟″]/g;
|
|
234
|
+
const DASHES = /[‐‑‒–—―−﹘﹣]/g;
|
|
235
|
+
/** This function applies the Unicode part of the fold: NFKC, no ignorable characters, and ASCII quotes and dashes. */
|
|
236
|
+
const unicodeForm = (text) => text.normalize("NFKC").replace(IGNORABLE, "").replace(QUOTES, "'").replace(DOUBLE_QUOTES, "\"").replace(DASHES, "-");
|
|
237
|
+
const spaceForm = (text) => text.replace(/\s+/gu, " ").trim();
|
|
238
|
+
const LAYER_SETS = [
|
|
239
|
+
["unicode"],
|
|
240
|
+
["whitespace"],
|
|
241
|
+
["case"],
|
|
242
|
+
["unicode", "whitespace"],
|
|
243
|
+
["unicode", "case"],
|
|
244
|
+
["whitespace", "case"]
|
|
245
|
+
];
|
|
246
|
+
/** This function names the parts of the fold that make the stored text equal to the value. */
|
|
247
|
+
const textLabels = (stored, value) => {
|
|
248
|
+
const apply = (text, layers) => {
|
|
249
|
+
let out = text;
|
|
250
|
+
if (layers.includes("unicode")) out = unicodeForm(out);
|
|
251
|
+
if (layers.includes("whitespace")) out = spaceForm(out);
|
|
252
|
+
if (layers.includes("case")) out = out.toLowerCase();
|
|
253
|
+
return out;
|
|
254
|
+
};
|
|
255
|
+
for (const layers of LAYER_SETS) if (apply(stored, layers) === apply(value, layers)) return [...layers];
|
|
256
|
+
return [
|
|
257
|
+
"unicode",
|
|
258
|
+
"whitespace",
|
|
259
|
+
"case"
|
|
260
|
+
];
|
|
261
|
+
};
|
|
262
|
+
/**
|
|
263
|
+
* This function finds a text value in a text, with the fold of `verify`. A match must start and
|
|
264
|
+
* end at the edge of a word. The positions are in the original text.
|
|
265
|
+
*/
|
|
266
|
+
const checkText = (doc, claim, work) => {
|
|
267
|
+
const needle = fold(claim, "quote");
|
|
268
|
+
if (needle.length === 0) return empty("invalid", "text", "the value has no text");
|
|
269
|
+
const hay = doc.folded.text;
|
|
270
|
+
const hits = occurrences(hay, needle, work);
|
|
271
|
+
const wordStart = isWordAt(needle, 0);
|
|
272
|
+
const wordEnd = isWordAt(needle, needle.length - 1);
|
|
273
|
+
let count = 0;
|
|
274
|
+
let next = 0;
|
|
275
|
+
let firstExact;
|
|
276
|
+
let first;
|
|
277
|
+
for (const at of hits) {
|
|
278
|
+
work.steps += 1;
|
|
279
|
+
if (at < next) continue;
|
|
280
|
+
const end = at + needle.length;
|
|
281
|
+
if (wordStart && isWordAt(hay, at - 1)) continue;
|
|
282
|
+
if (wordEnd && isWordAt(hay, end)) continue;
|
|
283
|
+
next = end;
|
|
284
|
+
count += 1;
|
|
285
|
+
const span = spanOf(doc, at, end);
|
|
286
|
+
first ??= span;
|
|
287
|
+
if (firstExact === void 0 && doc.text.slice(span.start, span.end) === claim) firstExact = span;
|
|
288
|
+
}
|
|
289
|
+
if (first === void 0) return empty("not-found", "text");
|
|
290
|
+
const span = firstExact ?? first;
|
|
291
|
+
const stored = doc.text.slice(span.start, span.end);
|
|
292
|
+
return {
|
|
293
|
+
...empty(firstExact !== void 0 ? "verified" : "normalized", "text"),
|
|
294
|
+
span,
|
|
295
|
+
raw: stored,
|
|
296
|
+
excerpt: excerptOf(doc.text, span.start, span.end),
|
|
297
|
+
normalization: firstExact !== void 0 ? null : labelText(textLabels(stored, claim)),
|
|
298
|
+
count
|
|
299
|
+
};
|
|
300
|
+
};
|
|
301
|
+
const THOUSANDS = /* @__PURE__ */ new Set([
|
|
302
|
+
",",
|
|
303
|
+
".",
|
|
304
|
+
"'",
|
|
305
|
+
" "
|
|
306
|
+
]);
|
|
307
|
+
const DECIMALS = /* @__PURE__ */ new Set([".", ","]);
|
|
308
|
+
/** The no-break and thin spaces that separate the thousands in some languages. */
|
|
309
|
+
const GROUP_SPACES = /* @__PURE__ */ new Set([
|
|
310
|
+
160,
|
|
311
|
+
8239,
|
|
312
|
+
8201,
|
|
313
|
+
8199
|
|
314
|
+
]);
|
|
315
|
+
const makeReading = (sign, int, frac, group, point) => {
|
|
316
|
+
const intPart = int.replace(/^0+(?=\d)/, "");
|
|
317
|
+
const fracPart = frac.replace(/0+$/, "");
|
|
318
|
+
const zero = /^0+$/.test(intPart) && fracPart === "";
|
|
319
|
+
return {
|
|
320
|
+
value: `${sign === "-" && !zero ? "-" : ""}${intPart}${fracPart === "" ? "" : `.${fracPart}`}`,
|
|
321
|
+
plain: `${sign}${int}${frac === "" ? "" : `.${frac}`}`,
|
|
322
|
+
group,
|
|
323
|
+
point
|
|
324
|
+
};
|
|
325
|
+
};
|
|
326
|
+
/**
|
|
327
|
+
* This function gives each meaning of a number as written. A group of exactly three digits after
|
|
328
|
+
* one separator gives two meanings (`1,234` is 1234 or 1.234), and the function gives the two.
|
|
329
|
+
*/
|
|
330
|
+
const numberReadings = (written) => {
|
|
331
|
+
let sign = "";
|
|
332
|
+
let rest = written;
|
|
333
|
+
if (rest.startsWith("-") || rest.startsWith("+")) {
|
|
334
|
+
sign = rest[0];
|
|
335
|
+
rest = rest.slice(1);
|
|
336
|
+
}
|
|
337
|
+
const groups = [];
|
|
338
|
+
const seps = [];
|
|
339
|
+
let from = 0;
|
|
340
|
+
for (let at = 0; at < rest.length; at += 1) {
|
|
341
|
+
if (isDigitCode(rest.charCodeAt(at))) continue;
|
|
342
|
+
groups.push(rest.slice(from, at));
|
|
343
|
+
seps.push(rest[at]);
|
|
344
|
+
from = at + 1;
|
|
345
|
+
}
|
|
346
|
+
groups.push(rest.slice(from));
|
|
347
|
+
if (groups.some((group) => group.length === 0)) return [];
|
|
348
|
+
const found = /* @__PURE__ */ new Map();
|
|
349
|
+
const add = (int, frac, group, point) => {
|
|
350
|
+
const reading = makeReading(sign, int, frac, group, point);
|
|
351
|
+
if (!found.has(reading.value)) found.set(reading.value, reading);
|
|
352
|
+
};
|
|
353
|
+
if (seps.length === 0) {
|
|
354
|
+
add(groups[0], "", null, null);
|
|
355
|
+
return [...found.values()];
|
|
356
|
+
}
|
|
357
|
+
const t = seps[0];
|
|
358
|
+
const grouped = (count) => {
|
|
359
|
+
const head = groups[0];
|
|
360
|
+
if (head.length > 3 || head.startsWith("0")) return false;
|
|
361
|
+
for (let k = 1; k <= count; k += 1) if (groups[k].length !== 3 || seps[k - 1] !== t) return false;
|
|
362
|
+
return true;
|
|
363
|
+
};
|
|
364
|
+
if (THOUSANDS.has(t) && grouped(seps.length)) add(groups.join(""), "", t, null);
|
|
365
|
+
const d = seps[seps.length - 1];
|
|
366
|
+
if (seps.length >= 2 && DECIMALS.has(d) && d !== t && THOUSANDS.has(t) && grouped(seps.length - 1)) add(groups.slice(0, -1).join(""), groups[groups.length - 1], t, d);
|
|
367
|
+
if (seps.length === 1 && DECIMALS.has(d)) add(groups[0], groups[1], null, d);
|
|
368
|
+
return [...found.values()];
|
|
369
|
+
};
|
|
370
|
+
/** This function writes a JavaScript `number` in decimal digits, with no exponent. */
|
|
371
|
+
const decimalOf = (value) => {
|
|
372
|
+
if (!Number.isFinite(value)) return void 0;
|
|
373
|
+
const text = String(value);
|
|
374
|
+
const match = /^(-?)(\d+)(?:\.(\d+))?e([+-]\d+)$/i.exec(text);
|
|
375
|
+
if (match === null) return text;
|
|
376
|
+
const digits = `${match[2]}${match[3] ?? ""}`;
|
|
377
|
+
const point = match[2].length + Number(match[4]);
|
|
378
|
+
const body = point <= 0 ? `0.${"0".repeat(-point)}${digits}` : point >= digits.length ? `${digits}${"0".repeat(point - digits.length)}` : `${digits.slice(0, point)}.${digits.slice(point)}`;
|
|
379
|
+
return `${match[1]}${body}`;
|
|
380
|
+
};
|
|
381
|
+
/** This function reads a proposed number. A JavaScript `number` has one meaning: its point is a decimal point. */
|
|
382
|
+
const claimNumber = (value) => {
|
|
383
|
+
if (typeof value === "number") {
|
|
384
|
+
const text = decimalOf(value);
|
|
385
|
+
if (text === void 0) return [];
|
|
386
|
+
const sign = text.startsWith("-") ? "-" : "";
|
|
387
|
+
const [int = "", frac = ""] = text.replace(/^-/, "").split(".");
|
|
388
|
+
return [makeReading(sign, int, frac, null, frac === "" ? null : ".")];
|
|
389
|
+
}
|
|
390
|
+
const text = fold(value, "quote").trim();
|
|
391
|
+
if (!/^[+-]?\d[\d.,' ]*$/.test(text) || !/\d$/.test(text)) return [];
|
|
392
|
+
return numberReadings(text);
|
|
393
|
+
};
|
|
394
|
+
/**
|
|
395
|
+
* This function finds the numbers of a folded text. A number is digits with separators. A letter
|
|
396
|
+
* before it makes a name (`A320`), so it is not a number. Each unit is read a constant number of times.
|
|
397
|
+
*/
|
|
398
|
+
const numberTokens = (doc, work) => cached(doc, "number", () => {
|
|
399
|
+
const f = doc.folded.text;
|
|
400
|
+
const n = f.length;
|
|
401
|
+
const out = [];
|
|
402
|
+
const isSep = (at) => {
|
|
403
|
+
const char = f[at];
|
|
404
|
+
if (char === "," || char === "." || char === "'") return true;
|
|
405
|
+
if (char !== " ") return false;
|
|
406
|
+
const from = doc.folded.start[at];
|
|
407
|
+
return doc.folded.end[at] - from === 1 && GROUP_SPACES.has(doc.text.charCodeAt(from)) && (at + 1 >= n || doc.folded.start[at + 1] === from + 1);
|
|
408
|
+
};
|
|
409
|
+
let i = 0;
|
|
410
|
+
while (i < n) {
|
|
411
|
+
work.steps += 1;
|
|
412
|
+
if (!isDigitAt(f, i)) {
|
|
413
|
+
i += 1;
|
|
414
|
+
continue;
|
|
415
|
+
}
|
|
416
|
+
const start = i;
|
|
417
|
+
let j = i;
|
|
418
|
+
while (j < n && isDigitAt(f, j)) {
|
|
419
|
+
j += 1;
|
|
420
|
+
work.steps += 1;
|
|
421
|
+
}
|
|
422
|
+
while (j + 1 < n && isSep(j) && isDigitAt(f, j + 1)) {
|
|
423
|
+
j += 1;
|
|
424
|
+
while (j < n && isDigitAt(f, j)) {
|
|
425
|
+
j += 1;
|
|
426
|
+
work.steps += 1;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
i = j;
|
|
430
|
+
const before = start > 0 ? f[start - 1] : "";
|
|
431
|
+
if (isLetterAt(f, start - 1) || before === "." || before === "," && isDigitAt(f, start - 2)) continue;
|
|
432
|
+
const from = (before === "-" || before === "+") && !isWordAt(f, start - 2) ? start - 1 : start;
|
|
433
|
+
const readings = numberReadings(f.slice(from, j));
|
|
434
|
+
if (readings.length > 0) {
|
|
435
|
+
out.push({
|
|
436
|
+
i: from,
|
|
437
|
+
j,
|
|
438
|
+
readings
|
|
439
|
+
});
|
|
440
|
+
continue;
|
|
441
|
+
}
|
|
442
|
+
let a = start;
|
|
443
|
+
while (a < j) {
|
|
444
|
+
let b = a;
|
|
445
|
+
while (b < j && isDigitAt(f, b)) b += 1;
|
|
446
|
+
out.push({
|
|
447
|
+
i: a,
|
|
448
|
+
j: b,
|
|
449
|
+
readings: [makeReading("", f.slice(a, b), "", null, null)]
|
|
450
|
+
});
|
|
451
|
+
a = b + 1;
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
return out;
|
|
455
|
+
});
|
|
456
|
+
const numberLabels = (claim, reading) => {
|
|
457
|
+
const labels = /* @__PURE__ */ new Set();
|
|
458
|
+
if (claim.group !== reading.group) labels.add("thousands-separator");
|
|
459
|
+
if (claim.point !== reading.point && (claim.point === "," || reading.point === ",")) labels.add("decimal-comma");
|
|
460
|
+
if (claim.plain !== reading.plain) labels.add("number-form");
|
|
461
|
+
return [...labels];
|
|
462
|
+
};
|
|
463
|
+
const NUMBER = {
|
|
464
|
+
compare: (claim, reading) => {
|
|
465
|
+
if (claim.value !== reading.value) return void 0;
|
|
466
|
+
const labels = numberLabels(claim, reading);
|
|
467
|
+
return {
|
|
468
|
+
clean: true,
|
|
469
|
+
key: reading.value,
|
|
470
|
+
reading: reading.value,
|
|
471
|
+
labels: labels.length === 0 ? ["unicode"] : labels
|
|
472
|
+
};
|
|
473
|
+
},
|
|
474
|
+
display: (reading) => reading.value
|
|
475
|
+
};
|
|
476
|
+
const numberCands = (doc, work) => numberTokens(doc, work).map((token) => {
|
|
477
|
+
const span = spanOf(doc, token.i, token.j);
|
|
478
|
+
const exact = doc.text.slice(span.start, span.end);
|
|
479
|
+
return {
|
|
480
|
+
where: {
|
|
481
|
+
span,
|
|
482
|
+
path: null
|
|
483
|
+
},
|
|
484
|
+
raw: exact,
|
|
485
|
+
exact,
|
|
486
|
+
excerpt: () => excerptOf(doc.text, span.start, span.end),
|
|
487
|
+
readings: token.readings
|
|
488
|
+
};
|
|
489
|
+
});
|
|
490
|
+
/** The codes of ISO 4217 that the check knows, for the common currencies. */
|
|
491
|
+
const CODES = [
|
|
492
|
+
"USD",
|
|
493
|
+
"EUR",
|
|
494
|
+
"GBP",
|
|
495
|
+
"JPY",
|
|
496
|
+
"CNY",
|
|
497
|
+
"CAD",
|
|
498
|
+
"AUD",
|
|
499
|
+
"NZD",
|
|
500
|
+
"CHF",
|
|
501
|
+
"SEK",
|
|
502
|
+
"NOK",
|
|
503
|
+
"DKK",
|
|
504
|
+
"ISK",
|
|
505
|
+
"PLN",
|
|
506
|
+
"CZK",
|
|
507
|
+
"HUF",
|
|
508
|
+
"RON",
|
|
509
|
+
"BGN",
|
|
510
|
+
"TRY",
|
|
511
|
+
"ILS",
|
|
512
|
+
"INR",
|
|
513
|
+
"PKR",
|
|
514
|
+
"BDT",
|
|
515
|
+
"LKR",
|
|
516
|
+
"KRW",
|
|
517
|
+
"SGD",
|
|
518
|
+
"HKD",
|
|
519
|
+
"TWD",
|
|
520
|
+
"THB",
|
|
521
|
+
"MYR",
|
|
522
|
+
"IDR",
|
|
523
|
+
"PHP",
|
|
524
|
+
"VND",
|
|
525
|
+
"BRL",
|
|
526
|
+
"MXN",
|
|
527
|
+
"ARS",
|
|
528
|
+
"CLP",
|
|
529
|
+
"COP",
|
|
530
|
+
"PEN",
|
|
531
|
+
"UYU",
|
|
532
|
+
"ZAR",
|
|
533
|
+
"NGN",
|
|
534
|
+
"EGP",
|
|
535
|
+
"KES",
|
|
536
|
+
"MAD",
|
|
537
|
+
"AED",
|
|
538
|
+
"SAR",
|
|
539
|
+
"QAR",
|
|
540
|
+
"KWD",
|
|
541
|
+
"BHD",
|
|
542
|
+
"OMR",
|
|
543
|
+
"JOD",
|
|
544
|
+
"RUB",
|
|
545
|
+
"UAH",
|
|
546
|
+
"KZT",
|
|
547
|
+
"GEL"
|
|
548
|
+
];
|
|
549
|
+
const MARKERS = [...[
|
|
550
|
+
["us$", ["USD"]],
|
|
551
|
+
["ca$", ["CAD"]],
|
|
552
|
+
["c$", ["CAD"]],
|
|
553
|
+
["au$", ["AUD"]],
|
|
554
|
+
["a$", ["AUD"]],
|
|
555
|
+
["nz$", ["NZD"]],
|
|
556
|
+
["hk$", ["HKD"]],
|
|
557
|
+
["s$", ["SGD"]],
|
|
558
|
+
["r$", ["BRL"]],
|
|
559
|
+
["mx$", ["MXN"]],
|
|
560
|
+
["nt$", ["TWD"]],
|
|
561
|
+
["$", [
|
|
562
|
+
"USD",
|
|
563
|
+
"CAD",
|
|
564
|
+
"AUD",
|
|
565
|
+
"NZD",
|
|
566
|
+
"MXN",
|
|
567
|
+
"SGD",
|
|
568
|
+
"HKD",
|
|
569
|
+
"TWD",
|
|
570
|
+
"ARS",
|
|
571
|
+
"CLP",
|
|
572
|
+
"COP"
|
|
573
|
+
]],
|
|
574
|
+
["€", ["EUR"]],
|
|
575
|
+
["£", ["GBP"]],
|
|
576
|
+
["cn¥", ["CNY"]],
|
|
577
|
+
["jp¥", ["JPY"]],
|
|
578
|
+
["¥", ["JPY", "CNY"]],
|
|
579
|
+
["₹", ["INR"]],
|
|
580
|
+
["₩", ["KRW"]],
|
|
581
|
+
["₽", ["RUB"]],
|
|
582
|
+
["₺", ["TRY"]],
|
|
583
|
+
["₪", ["ILS"]],
|
|
584
|
+
["₫", ["VND"]],
|
|
585
|
+
["฿", ["THB"]],
|
|
586
|
+
["₱", ["PHP"]],
|
|
587
|
+
["₦", ["NGN"]],
|
|
588
|
+
["₴", ["UAH"]],
|
|
589
|
+
["₸", ["KZT"]],
|
|
590
|
+
["₾", ["GEL"]],
|
|
591
|
+
["zł", ["PLN"]],
|
|
592
|
+
["kč", ["CZK"]],
|
|
593
|
+
["kr", [
|
|
594
|
+
"SEK",
|
|
595
|
+
"NOK",
|
|
596
|
+
"DKK",
|
|
597
|
+
"ISK"
|
|
598
|
+
]]
|
|
599
|
+
].map(([text, codes]) => ({
|
|
600
|
+
text,
|
|
601
|
+
codes,
|
|
602
|
+
label: codes.length === 1 ? codes[0] : text,
|
|
603
|
+
alpha: /^[a-z]/.test(text),
|
|
604
|
+
code: false
|
|
605
|
+
})), ...CODES.map((code) => ({
|
|
606
|
+
text: code.toLowerCase(),
|
|
607
|
+
codes: [code],
|
|
608
|
+
label: code,
|
|
609
|
+
alpha: true,
|
|
610
|
+
code: true
|
|
611
|
+
}))].sort((a, b) => b.text.length - a.text.length);
|
|
612
|
+
/** This function finds a currency marker that ends at a position (exclusive), or undefined. */
|
|
613
|
+
const markerEndingAt = (f, end) => {
|
|
614
|
+
for (const marker of MARKERS) {
|
|
615
|
+
const from = end - marker.text.length;
|
|
616
|
+
if (from < 0 || !f.startsWith(marker.text, from)) continue;
|
|
617
|
+
if (marker.alpha && isLetterAt(f, from - 1)) continue;
|
|
618
|
+
return marker;
|
|
619
|
+
}
|
|
620
|
+
};
|
|
621
|
+
/** This function finds a currency marker that starts at a position, or undefined. */
|
|
622
|
+
const markerStartingAt = (f, from) => {
|
|
623
|
+
for (const marker of MARKERS) {
|
|
624
|
+
if (!f.startsWith(marker.text, from)) continue;
|
|
625
|
+
if (marker.alpha && isLetterAt(f, from + marker.text.length)) continue;
|
|
626
|
+
return marker;
|
|
627
|
+
}
|
|
628
|
+
};
|
|
629
|
+
const moneyText = (reading) => reading.marker === null ? reading.amount.value : `${reading.marker.label} ${reading.amount.value}`;
|
|
630
|
+
const MONEY = {
|
|
631
|
+
compare: (claim, reading) => {
|
|
632
|
+
if (claim.amount.value !== reading.amount.value || reading.marker === null) return void 0;
|
|
633
|
+
const labels = new Set(numberLabels(claim.amount, reading.amount));
|
|
634
|
+
let clean = true;
|
|
635
|
+
if (claim.marker === null) labels.add("amount-only");
|
|
636
|
+
else {
|
|
637
|
+
if (claim.marker.label !== reading.marker.label) {
|
|
638
|
+
if (!claim.marker.codes.some((code) => reading.marker.codes.includes(code))) return void 0;
|
|
639
|
+
clean = false;
|
|
640
|
+
}
|
|
641
|
+
if (claim.marker.text !== reading.marker.text) labels.add("currency-symbol");
|
|
642
|
+
if (claim.before !== reading.before) labels.add("currency-position");
|
|
643
|
+
}
|
|
644
|
+
if (labels.size === 0) labels.add("whitespace");
|
|
645
|
+
return {
|
|
646
|
+
clean,
|
|
647
|
+
key: moneyText(reading),
|
|
648
|
+
reading: moneyText(reading),
|
|
649
|
+
labels: [...labels]
|
|
650
|
+
};
|
|
651
|
+
},
|
|
652
|
+
display: moneyText
|
|
653
|
+
};
|
|
654
|
+
/** This function reads a proposed amount of money: an amount, with a currency symbol or code before or after it. */
|
|
655
|
+
const claimMoney = (value) => {
|
|
656
|
+
if (typeof value === "number") return claimNumber(value).map((amount) => ({
|
|
657
|
+
amount,
|
|
658
|
+
marker: null,
|
|
659
|
+
before: false
|
|
660
|
+
}));
|
|
661
|
+
const text = fold(value, "quote").trim();
|
|
662
|
+
const head = markerStartingAt(text, 0);
|
|
663
|
+
if (head !== void 0) return claimNumber(text.slice(head.text.length).trim()).map((amount) => ({
|
|
664
|
+
amount,
|
|
665
|
+
marker: head,
|
|
666
|
+
before: true
|
|
667
|
+
}));
|
|
668
|
+
const tail = markerEndingAt(text, text.length);
|
|
669
|
+
if (tail !== void 0) return claimNumber(text.slice(0, text.length - tail.text.length).trim()).map((amount) => ({
|
|
670
|
+
amount,
|
|
671
|
+
marker: tail,
|
|
672
|
+
before: false
|
|
673
|
+
}));
|
|
674
|
+
return claimNumber(text).map((amount) => ({
|
|
675
|
+
amount,
|
|
676
|
+
marker: null,
|
|
677
|
+
before: false
|
|
678
|
+
}));
|
|
679
|
+
};
|
|
680
|
+
/** This function finds the amounts of money of a text. An amount is a number with a currency marker before or after it, with one space or none. */
|
|
681
|
+
const moneyCands = (doc, work) => cached(doc, "money", () => {
|
|
682
|
+
const f = doc.folded.text;
|
|
683
|
+
const out = [];
|
|
684
|
+
const written = (marker, from) => {
|
|
685
|
+
if (marker === void 0 || !marker.code) return marker;
|
|
686
|
+
return doc.text.slice(doc.folded.start[from], doc.folded.end[from + marker.text.length - 1]) === marker.label ? marker : void 0;
|
|
687
|
+
};
|
|
688
|
+
for (const token of numberTokens(doc, work)) {
|
|
689
|
+
work.steps += MARKERS.length;
|
|
690
|
+
const gapBefore = f[token.i - 1] === " " ? 1 : 0;
|
|
691
|
+
const found = markerEndingAt(f, token.i - gapBefore);
|
|
692
|
+
const before = written(found, token.i - gapBefore - (found === void 0 ? 0 : found.text.length));
|
|
693
|
+
let marker = before;
|
|
694
|
+
let i = token.i;
|
|
695
|
+
let j = token.j;
|
|
696
|
+
if (before !== void 0) i = token.i - gapBefore - before.text.length;
|
|
697
|
+
else {
|
|
698
|
+
const gapAfter = f[token.j] === " " ? 1 : 0;
|
|
699
|
+
const after = written(markerStartingAt(f, token.j + gapAfter), token.j + gapAfter);
|
|
700
|
+
if (after === void 0) continue;
|
|
701
|
+
marker = after;
|
|
702
|
+
j = token.j + gapAfter + after.text.length;
|
|
703
|
+
}
|
|
704
|
+
const span = spanOf(doc, i, j);
|
|
705
|
+
const amount = spanOf(doc, token.i, token.j);
|
|
706
|
+
const exact = doc.text.slice(span.start, span.end);
|
|
707
|
+
out.push({
|
|
708
|
+
where: {
|
|
709
|
+
span,
|
|
710
|
+
path: null
|
|
711
|
+
},
|
|
712
|
+
raw: exact,
|
|
713
|
+
exact,
|
|
714
|
+
excerpt: () => excerptOf(doc.text, span.start, span.end),
|
|
715
|
+
readings: token.readings.map((reading) => ({
|
|
716
|
+
amount: reading,
|
|
717
|
+
marker: marker ?? null,
|
|
718
|
+
before: before !== void 0
|
|
719
|
+
})),
|
|
720
|
+
alt: {
|
|
721
|
+
where: {
|
|
722
|
+
span: amount,
|
|
723
|
+
path: null
|
|
724
|
+
},
|
|
725
|
+
exact: doc.text.slice(amount.start, amount.end)
|
|
726
|
+
}
|
|
727
|
+
});
|
|
728
|
+
}
|
|
729
|
+
return out;
|
|
730
|
+
});
|
|
731
|
+
const MONTHS = /* @__PURE__ */ new Map([
|
|
732
|
+
["january", 1],
|
|
733
|
+
["jan", 1],
|
|
734
|
+
["february", 2],
|
|
735
|
+
["feb", 2],
|
|
736
|
+
["march", 3],
|
|
737
|
+
["mar", 3],
|
|
738
|
+
["april", 4],
|
|
739
|
+
["apr", 4],
|
|
740
|
+
["may", 5],
|
|
741
|
+
["june", 6],
|
|
742
|
+
["jun", 6],
|
|
743
|
+
["july", 7],
|
|
744
|
+
["jul", 7],
|
|
745
|
+
["august", 8],
|
|
746
|
+
["aug", 8],
|
|
747
|
+
["september", 9],
|
|
748
|
+
["sept", 9],
|
|
749
|
+
["sep", 9],
|
|
750
|
+
["october", 10],
|
|
751
|
+
["oct", 10],
|
|
752
|
+
["november", 11],
|
|
753
|
+
["nov", 11],
|
|
754
|
+
["december", 12],
|
|
755
|
+
["dec", 12]
|
|
756
|
+
]);
|
|
757
|
+
const WEEKDAYS = /* @__PURE__ */ new Set([
|
|
758
|
+
"monday",
|
|
759
|
+
"mon",
|
|
760
|
+
"tuesday",
|
|
761
|
+
"tue",
|
|
762
|
+
"tues",
|
|
763
|
+
"wednesday",
|
|
764
|
+
"wed",
|
|
765
|
+
"thursday",
|
|
766
|
+
"thu",
|
|
767
|
+
"thur",
|
|
768
|
+
"thurs",
|
|
769
|
+
"friday",
|
|
770
|
+
"fri",
|
|
771
|
+
"saturday",
|
|
772
|
+
"sat",
|
|
773
|
+
"sunday",
|
|
774
|
+
"sun"
|
|
775
|
+
]);
|
|
776
|
+
const ORDINALS = /* @__PURE__ */ new Set([
|
|
777
|
+
"st",
|
|
778
|
+
"nd",
|
|
779
|
+
"rd",
|
|
780
|
+
"th"
|
|
781
|
+
]);
|
|
782
|
+
const daysIn = (m, y) => m === 2 ? y === null || y % 4 === 0 && (y % 100 !== 0 || y % 400 === 0) ? 29 : 28 : [
|
|
783
|
+
4,
|
|
784
|
+
6,
|
|
785
|
+
9,
|
|
786
|
+
11
|
|
787
|
+
].includes(m) ? 30 : 31;
|
|
788
|
+
const validDate = (y, m, d) => m >= 1 && m <= 12 && (y === null || y >= 1e3 && y <= 2999) && (d === null || d >= 1 && d <= daysIn(m, y));
|
|
789
|
+
const pad = (value, size) => String(value).padStart(size, "0");
|
|
790
|
+
const dateText = (reading) => reading.y === null ? `--${pad(reading.m, 2)}${reading.d === null ? "" : `-${pad(reading.d, 2)}`}` : `${pad(reading.y, 4)}-${pad(reading.m, 2)}${reading.d === null ? "" : `-${pad(reading.d, 2)}`}`;
|
|
791
|
+
const NUM = 0;
|
|
792
|
+
const ALPHA = 1;
|
|
793
|
+
const PUNCT = 2;
|
|
794
|
+
const lex = (f, work) => {
|
|
795
|
+
const out = [];
|
|
796
|
+
let gap = false;
|
|
797
|
+
let i = 0;
|
|
798
|
+
while (i < f.length) {
|
|
799
|
+
work.steps += 1;
|
|
800
|
+
if (f[i] === " ") {
|
|
801
|
+
gap = true;
|
|
802
|
+
i += 1;
|
|
803
|
+
continue;
|
|
804
|
+
}
|
|
805
|
+
let j = i + 1;
|
|
806
|
+
let type = PUNCT;
|
|
807
|
+
if (isDigitAt(f, i)) {
|
|
808
|
+
type = NUM;
|
|
809
|
+
while (isDigitAt(f, j)) j += 1;
|
|
810
|
+
} else if (isLetterAt(f, i)) {
|
|
811
|
+
type = ALPHA;
|
|
812
|
+
while (isLetterAt(f, j)) j += 1;
|
|
813
|
+
}
|
|
814
|
+
work.steps += j - i;
|
|
815
|
+
out.push({
|
|
816
|
+
type,
|
|
817
|
+
i,
|
|
818
|
+
j,
|
|
819
|
+
text: f.slice(i, j),
|
|
820
|
+
gap
|
|
821
|
+
});
|
|
822
|
+
gap = false;
|
|
823
|
+
i = j;
|
|
824
|
+
}
|
|
825
|
+
return out;
|
|
826
|
+
};
|
|
827
|
+
const isNum = (token, min, max) => token !== void 0 && token.type === NUM && token.text.length >= min && token.text.length <= max;
|
|
828
|
+
const isPunct = (token, chars) => token !== void 0 && token.type === PUNCT && !token.gap && chars.includes(token.text);
|
|
829
|
+
const glued = (token) => token !== void 0 && !token.gap;
|
|
830
|
+
/** This function reads a day after a month or before it: one or two digits, and an ordinal suffix. */
|
|
831
|
+
const dayAt = (tokens, k) => {
|
|
832
|
+
const token = tokens[k];
|
|
833
|
+
if (!isNum(token, 1, 2)) return void 0;
|
|
834
|
+
const after = tokens[k + 1];
|
|
835
|
+
if (after !== void 0 && glued(after)) {
|
|
836
|
+
if (after.type === ALPHA) {
|
|
837
|
+
if (!ORDINALS.has(after.text)) return void 0;
|
|
838
|
+
return {
|
|
839
|
+
d: Number(token.text),
|
|
840
|
+
next: k + 2
|
|
841
|
+
};
|
|
842
|
+
}
|
|
843
|
+
if (after.type === PUNCT && (after.text === ":" || after.text === ".") && isNum(tokens[k + 2], 1, 4) && glued(tokens[k + 2])) return void 0;
|
|
844
|
+
if (after.type === NUM) return void 0;
|
|
845
|
+
}
|
|
846
|
+
return {
|
|
847
|
+
d: Number(token.text),
|
|
848
|
+
next: k + 1
|
|
849
|
+
};
|
|
850
|
+
};
|
|
851
|
+
const yearAt = (tokens, k) => {
|
|
852
|
+
let at = k;
|
|
853
|
+
if (isPunct(tokens[at], ",")) at += 1;
|
|
854
|
+
const token = tokens[at];
|
|
855
|
+
if (!isNum(token, 4, 4)) return void 0;
|
|
856
|
+
if (glued(tokens[at + 1]) && tokens[at + 1].type === NUM) return void 0;
|
|
857
|
+
return {
|
|
858
|
+
y: Number(token.text),
|
|
859
|
+
next: at + 1
|
|
860
|
+
};
|
|
861
|
+
};
|
|
862
|
+
/** This function reads a date that starts at a token, or gives undefined. It looks at a constant number of tokens. */
|
|
863
|
+
const dateAt = (tokens, t) => {
|
|
864
|
+
const a = tokens[t];
|
|
865
|
+
const previous = tokens[t - 1];
|
|
866
|
+
const chained = previous !== void 0 && !a.gap && (previous.type === NUM || previous.type === PUNCT && "/.-".includes(previous.text));
|
|
867
|
+
if (a.type === NUM && !chained) {
|
|
868
|
+
const p1 = tokens[t + 1];
|
|
869
|
+
const b = tokens[t + 2];
|
|
870
|
+
const p2 = tokens[t + 3];
|
|
871
|
+
const c = tokens[t + 4];
|
|
872
|
+
const numeric = isPunct(p1, "-/.") && isNum(b, 1, 2) && !b.gap && isPunct(p2, p1.text) && glued(c) && c.type === NUM;
|
|
873
|
+
if (numeric && a.text.length === 4 && isNum(c, 1, 2)) {
|
|
874
|
+
const y = Number(a.text);
|
|
875
|
+
const m = Number(b.text);
|
|
876
|
+
const d = Number(c.text);
|
|
877
|
+
if (!validDate(y, m, d)) return void 0;
|
|
878
|
+
const iso = p1.text === "-" && b.text.length === 2 && c.text.length === 2;
|
|
879
|
+
return {
|
|
880
|
+
end: t + 5,
|
|
881
|
+
readings: [{
|
|
882
|
+
y,
|
|
883
|
+
m,
|
|
884
|
+
d,
|
|
885
|
+
form: iso ? "iso" : "ymd"
|
|
886
|
+
}]
|
|
887
|
+
};
|
|
888
|
+
}
|
|
889
|
+
if (numeric && a.text.length <= 2 && isNum(c, 4, 4)) {
|
|
890
|
+
const first = Number(a.text);
|
|
891
|
+
const second = Number(b.text);
|
|
892
|
+
const y = Number(c.text);
|
|
893
|
+
if (glued(tokens[t + 5]) && tokens[t + 5].type === NUM) return void 0;
|
|
894
|
+
const readings = [];
|
|
895
|
+
if (first === second) {
|
|
896
|
+
if (validDate(y, first, second)) readings.push({
|
|
897
|
+
y,
|
|
898
|
+
m: first,
|
|
899
|
+
d: second,
|
|
900
|
+
form: "numeric"
|
|
901
|
+
});
|
|
902
|
+
} else {
|
|
903
|
+
if (validDate(y, first, second)) readings.push({
|
|
904
|
+
y,
|
|
905
|
+
m: first,
|
|
906
|
+
d: second,
|
|
907
|
+
form: "mdy"
|
|
908
|
+
});
|
|
909
|
+
if (validDate(y, second, first)) readings.push({
|
|
910
|
+
y,
|
|
911
|
+
m: second,
|
|
912
|
+
d: first,
|
|
913
|
+
form: "dmy"
|
|
914
|
+
});
|
|
915
|
+
}
|
|
916
|
+
return readings.length === 0 ? void 0 : {
|
|
917
|
+
end: t + 5,
|
|
918
|
+
readings
|
|
919
|
+
};
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
if (a.type === ALPHA) {
|
|
923
|
+
const m = MONTHS.get(a.text);
|
|
924
|
+
if (m === void 0) return void 0;
|
|
925
|
+
let k = t + 1;
|
|
926
|
+
if (isPunct(tokens[k], ".")) k += 1;
|
|
927
|
+
const day = tokens[k]?.gap === true || k > t + 1 ? dayAt(tokens, k) : void 0;
|
|
928
|
+
if (day !== void 0) {
|
|
929
|
+
const year = yearAt(tokens, day.next);
|
|
930
|
+
const y = year?.y ?? null;
|
|
931
|
+
if (!validDate(y, m, day.d)) return void 0;
|
|
932
|
+
return {
|
|
933
|
+
end: year?.next ?? day.next,
|
|
934
|
+
readings: [{
|
|
935
|
+
y,
|
|
936
|
+
m,
|
|
937
|
+
d: day.d,
|
|
938
|
+
form: "written"
|
|
939
|
+
}]
|
|
940
|
+
};
|
|
941
|
+
}
|
|
942
|
+
const year = yearAt(tokens, k);
|
|
943
|
+
if (year === void 0 || !validDate(year.y, m, null)) return void 0;
|
|
944
|
+
return {
|
|
945
|
+
end: year.next,
|
|
946
|
+
readings: [{
|
|
947
|
+
y: year.y,
|
|
948
|
+
m,
|
|
949
|
+
d: null,
|
|
950
|
+
form: "written"
|
|
951
|
+
}]
|
|
952
|
+
};
|
|
953
|
+
}
|
|
954
|
+
if (a.type === NUM && !chained) {
|
|
955
|
+
const day = dayAt(tokens, t);
|
|
956
|
+
if (day === void 0) return void 0;
|
|
957
|
+
let k = day.next;
|
|
958
|
+
if (tokens[k]?.type === ALPHA && tokens[k].text === "of") k += 1;
|
|
959
|
+
const month = tokens[k];
|
|
960
|
+
if (month === void 0 || month.type !== ALPHA || !month.gap) return void 0;
|
|
961
|
+
const m = MONTHS.get(month.text);
|
|
962
|
+
if (m === void 0) return void 0;
|
|
963
|
+
k += 1;
|
|
964
|
+
if (isPunct(tokens[k], ".")) k += 1;
|
|
965
|
+
const year = yearAt(tokens, k);
|
|
966
|
+
const y = year?.y ?? null;
|
|
967
|
+
if (!validDate(y, m, day.d)) return void 0;
|
|
968
|
+
return {
|
|
969
|
+
end: year?.next ?? k,
|
|
970
|
+
readings: [{
|
|
971
|
+
y,
|
|
972
|
+
m,
|
|
973
|
+
d: day.d,
|
|
974
|
+
form: "written"
|
|
975
|
+
}]
|
|
976
|
+
};
|
|
977
|
+
}
|
|
978
|
+
};
|
|
979
|
+
const scanDates = (f, work) => {
|
|
980
|
+
const tokens = lex(f, work);
|
|
981
|
+
const out = [];
|
|
982
|
+
let t = 0;
|
|
983
|
+
while (t < tokens.length) {
|
|
984
|
+
work.steps += 1;
|
|
985
|
+
const found = dateAt(tokens, t);
|
|
986
|
+
if (found === void 0) {
|
|
987
|
+
t += 1;
|
|
988
|
+
continue;
|
|
989
|
+
}
|
|
990
|
+
out.push({
|
|
991
|
+
i: tokens[t].i,
|
|
992
|
+
j: tokens[found.end - 1].j,
|
|
993
|
+
readings: found.readings
|
|
994
|
+
});
|
|
995
|
+
t = found.end;
|
|
996
|
+
}
|
|
997
|
+
return out;
|
|
998
|
+
};
|
|
999
|
+
const dateCands = (doc, work) => cached(doc, "date", () => scanDates(doc.folded.text, work).map((token) => {
|
|
1000
|
+
const span = spanOf(doc, token.i, token.j);
|
|
1001
|
+
const exact = doc.text.slice(span.start, span.end);
|
|
1002
|
+
return {
|
|
1003
|
+
where: {
|
|
1004
|
+
span,
|
|
1005
|
+
path: null
|
|
1006
|
+
},
|
|
1007
|
+
raw: exact,
|
|
1008
|
+
exact,
|
|
1009
|
+
excerpt: () => excerptOf(doc.text, span.start, span.end),
|
|
1010
|
+
readings: token.readings
|
|
1011
|
+
};
|
|
1012
|
+
}));
|
|
1013
|
+
/** This function reads a proposed date: ISO 8601, written English months, or numbers. It gives no reading for a text that is not one date. */
|
|
1014
|
+
const claimDate = (value) => {
|
|
1015
|
+
if (typeof value === "number") return {
|
|
1016
|
+
readings: [],
|
|
1017
|
+
time: false
|
|
1018
|
+
};
|
|
1019
|
+
const text = fold(value, "quote").trim();
|
|
1020
|
+
const iso = /^(\d{4})-(\d{2})(?:-(\d{2}))?(t\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:z|[+-]\d{2}:?\d{2})?)?$/.exec(text);
|
|
1021
|
+
if (iso !== null) {
|
|
1022
|
+
const y = Number(iso[1]);
|
|
1023
|
+
const m = Number(iso[2]);
|
|
1024
|
+
const d = iso[3] === void 0 ? null : Number(iso[3]);
|
|
1025
|
+
if (iso[4] !== void 0 && d === null) return {
|
|
1026
|
+
readings: [],
|
|
1027
|
+
time: false
|
|
1028
|
+
};
|
|
1029
|
+
return validDate(y, m, d) ? {
|
|
1030
|
+
readings: [{
|
|
1031
|
+
y,
|
|
1032
|
+
m,
|
|
1033
|
+
d,
|
|
1034
|
+
form: "iso"
|
|
1035
|
+
}],
|
|
1036
|
+
time: iso[4] !== void 0
|
|
1037
|
+
} : {
|
|
1038
|
+
readings: [],
|
|
1039
|
+
time: false
|
|
1040
|
+
};
|
|
1041
|
+
}
|
|
1042
|
+
const tokens = lex(text, { steps: 0 });
|
|
1043
|
+
let t = 0;
|
|
1044
|
+
if (tokens[0]?.type === ALPHA && WEEKDAYS.has(tokens[0].text)) {
|
|
1045
|
+
t = 1;
|
|
1046
|
+
if (tokens[1]?.type === PUNCT && tokens[1].text === ",") t = 2;
|
|
1047
|
+
}
|
|
1048
|
+
const found = t < tokens.length ? dateAt(tokens, t) : void 0;
|
|
1049
|
+
const rest = found === void 0 ? [] : tokens.slice(found.end);
|
|
1050
|
+
if (found === void 0 || rest.some((token) => token.text !== ".")) return {
|
|
1051
|
+
readings: [],
|
|
1052
|
+
time: false
|
|
1053
|
+
};
|
|
1054
|
+
return {
|
|
1055
|
+
readings: [...found.readings],
|
|
1056
|
+
time: false
|
|
1057
|
+
};
|
|
1058
|
+
};
|
|
1059
|
+
const DATE = {
|
|
1060
|
+
compare: (claim, reading) => {
|
|
1061
|
+
if (claim.m !== reading.m) return void 0;
|
|
1062
|
+
if (claim.y !== null && reading.y !== null && claim.y !== reading.y) return void 0;
|
|
1063
|
+
if (claim.d !== null && reading.d !== null && claim.d !== reading.d) return void 0;
|
|
1064
|
+
const missing = claim.y !== null && reading.y === null || claim.d !== null && reading.d === null;
|
|
1065
|
+
const partial = missing || claim.y === null && reading.y !== null || claim.d === null && reading.d !== null;
|
|
1066
|
+
const labels = [claim.form === reading.form ? "date-form" : `date-${reading.form}`];
|
|
1067
|
+
if (partial) labels.push("date-partial");
|
|
1068
|
+
return {
|
|
1069
|
+
clean: !missing,
|
|
1070
|
+
key: dateText(reading),
|
|
1071
|
+
reading: dateText(reading),
|
|
1072
|
+
labels
|
|
1073
|
+
};
|
|
1074
|
+
},
|
|
1075
|
+
display: dateText
|
|
1076
|
+
};
|
|
1077
|
+
const phoneReading = (written) => {
|
|
1078
|
+
const text = written.replace(/\(0\)/g, "");
|
|
1079
|
+
let digits = text.replace(/\D/g, "");
|
|
1080
|
+
let intl = text.trimStart().startsWith("+");
|
|
1081
|
+
if (!intl && digits.startsWith("00")) {
|
|
1082
|
+
intl = true;
|
|
1083
|
+
digits = digits.slice(2);
|
|
1084
|
+
} else if (!intl && digits.startsWith("0")) digits = digits.slice(1);
|
|
1085
|
+
if (digits.length < 7 || digits.length > 15) return void 0;
|
|
1086
|
+
return {
|
|
1087
|
+
digits,
|
|
1088
|
+
intl
|
|
1089
|
+
};
|
|
1090
|
+
};
|
|
1091
|
+
const phoneText = (reading) => `${reading.intl ? "+" : ""}${reading.digits}`;
|
|
1092
|
+
/** The longest country code has three digits. */
|
|
1093
|
+
const COUNTRY_DIGITS = 3;
|
|
1094
|
+
const PHONE = {
|
|
1095
|
+
compare: (claim, reading) => {
|
|
1096
|
+
const key = reading.digits;
|
|
1097
|
+
if (claim.digits === reading.digits) return {
|
|
1098
|
+
clean: true,
|
|
1099
|
+
key,
|
|
1100
|
+
reading: phoneText(reading),
|
|
1101
|
+
labels: ["phone-digits"]
|
|
1102
|
+
};
|
|
1103
|
+
if (reading.digits.endsWith(claim.digits)) return {
|
|
1104
|
+
clean: true,
|
|
1105
|
+
key,
|
|
1106
|
+
reading: phoneText(reading),
|
|
1107
|
+
labels: ["phone-suffix"]
|
|
1108
|
+
};
|
|
1109
|
+
if (claim.digits.endsWith(reading.digits)) {
|
|
1110
|
+
const country = claim.digits.length - reading.digits.length;
|
|
1111
|
+
return {
|
|
1112
|
+
clean: claim.intl && !reading.intl && country <= COUNTRY_DIGITS,
|
|
1113
|
+
key,
|
|
1114
|
+
reading: phoneText(reading),
|
|
1115
|
+
labels: ["phone-suffix"]
|
|
1116
|
+
};
|
|
1117
|
+
}
|
|
1118
|
+
},
|
|
1119
|
+
display: phoneText,
|
|
1120
|
+
same: (a, b) => a.key.endsWith(b.key) || b.key.endsWith(a.key)
|
|
1121
|
+
};
|
|
1122
|
+
const PHONE_CHARS = "-.()/";
|
|
1123
|
+
/** This function finds the phone numbers of a folded text: 7 to 15 digits with spaces, dashes, points, brackets and slashes. */
|
|
1124
|
+
const phoneCands = (doc, work) => cached(doc, "phone", () => {
|
|
1125
|
+
const f = doc.folded.text;
|
|
1126
|
+
const n = f.length;
|
|
1127
|
+
const out = [];
|
|
1128
|
+
let i = 0;
|
|
1129
|
+
while (i < n) {
|
|
1130
|
+
work.steps += 1;
|
|
1131
|
+
const char = f[i];
|
|
1132
|
+
if (!((isDigitAt(f, i) || (char === "+" || char === "(") && (isDigitAt(f, i + 1) || f[i + 1] === "(")) && !isWordAt(f, i - 1) && f[i - 1] !== "+")) {
|
|
1133
|
+
i += 1;
|
|
1134
|
+
continue;
|
|
1135
|
+
}
|
|
1136
|
+
let j = i;
|
|
1137
|
+
while (j < n) {
|
|
1138
|
+
work.steps += 1;
|
|
1139
|
+
const unit = f[j];
|
|
1140
|
+
if (isDigitAt(f, j) || PHONE_CHARS.includes(unit) || unit === "+" && j === i) {
|
|
1141
|
+
j += 1;
|
|
1142
|
+
continue;
|
|
1143
|
+
}
|
|
1144
|
+
if (unit === " " && (isDigitAt(f, j + 1) || f[j + 1] === "(")) {
|
|
1145
|
+
j += 1;
|
|
1146
|
+
continue;
|
|
1147
|
+
}
|
|
1148
|
+
break;
|
|
1149
|
+
}
|
|
1150
|
+
let e = j;
|
|
1151
|
+
while (e > i && !isDigitAt(f, e - 1)) e -= 1;
|
|
1152
|
+
const next = Math.max(j, i + 1);
|
|
1153
|
+
if (e > i && !isWordAt(f, e)) {
|
|
1154
|
+
const span = spanOf(doc, i, e);
|
|
1155
|
+
const exact = doc.text.slice(span.start, span.end);
|
|
1156
|
+
const reading = phoneReading(f.slice(i, e));
|
|
1157
|
+
if (reading !== void 0) out.push({
|
|
1158
|
+
where: {
|
|
1159
|
+
span,
|
|
1160
|
+
path: null
|
|
1161
|
+
},
|
|
1162
|
+
raw: exact,
|
|
1163
|
+
exact,
|
|
1164
|
+
excerpt: () => excerptOf(doc.text, span.start, span.end),
|
|
1165
|
+
readings: [reading]
|
|
1166
|
+
});
|
|
1167
|
+
}
|
|
1168
|
+
i = next;
|
|
1169
|
+
}
|
|
1170
|
+
return out;
|
|
1171
|
+
});
|
|
1172
|
+
const claimPhone = (value) => {
|
|
1173
|
+
const text = (typeof value === "number" ? String(value) : fold(value, "quote").trim()).replace(/^tel:/, "");
|
|
1174
|
+
if (!/^\+?[\d\s\-.()/]+$/.test(text)) return [];
|
|
1175
|
+
const reading = phoneReading(text);
|
|
1176
|
+
return reading === void 0 ? [] : [reading];
|
|
1177
|
+
};
|
|
1178
|
+
const withoutScheme = (url) => url.replace(/^[a-z][a-z0-9+.-]*:/i, "");
|
|
1179
|
+
const URL_STOP = /[\s<>"`{}|\\^]/u;
|
|
1180
|
+
const URL_TRAIL = /[.,;:!?'"]/;
|
|
1181
|
+
/** This function finds the URLs of a text: `http://`, `https://` and `www.` forms. It reads the original text, because a path keeps its case. */
|
|
1182
|
+
const urlCands = (doc, work) => cached(doc, "url", () => {
|
|
1183
|
+
const text = doc.text;
|
|
1184
|
+
const n = text.length;
|
|
1185
|
+
const out = [];
|
|
1186
|
+
let i = 0;
|
|
1187
|
+
while (i < n) {
|
|
1188
|
+
work.steps += 1;
|
|
1189
|
+
const code = text.charCodeAt(i) | 32;
|
|
1190
|
+
let prefix = 0;
|
|
1191
|
+
if (code === 104 && !isWordAt(text, i - 1)) {
|
|
1192
|
+
const head = text.slice(i, i + 8).toLowerCase();
|
|
1193
|
+
prefix = head.startsWith("https://") ? 8 : head.startsWith("http://") ? 7 : 0;
|
|
1194
|
+
} else if (code === 119 && !isWordAt(text, i - 1) && text[i - 1] !== "/" && text[i - 1] !== ".") prefix = text.slice(i, i + 4).toLowerCase() === "www." ? 4 : 0;
|
|
1195
|
+
if (prefix === 0) {
|
|
1196
|
+
i += 1;
|
|
1197
|
+
continue;
|
|
1198
|
+
}
|
|
1199
|
+
const start = i;
|
|
1200
|
+
let j = start + prefix;
|
|
1201
|
+
while (j < n && !URL_STOP.test(text[j])) {
|
|
1202
|
+
j += 1;
|
|
1203
|
+
work.steps += 1;
|
|
1204
|
+
}
|
|
1205
|
+
i = j;
|
|
1206
|
+
let e = j;
|
|
1207
|
+
const balance = {
|
|
1208
|
+
")": 0,
|
|
1209
|
+
"]": 0
|
|
1210
|
+
};
|
|
1211
|
+
for (let at = start; at < j; at += 1) {
|
|
1212
|
+
const char = text[at];
|
|
1213
|
+
if (char === "(") balance[")"] += 1;
|
|
1214
|
+
else if (char === ")") balance[")"] -= 1;
|
|
1215
|
+
else if (char === "[") balance["]"] += 1;
|
|
1216
|
+
else if (char === "]") balance["]"] -= 1;
|
|
1217
|
+
}
|
|
1218
|
+
work.steps += j - start;
|
|
1219
|
+
while (e > start + prefix) {
|
|
1220
|
+
work.steps += 1;
|
|
1221
|
+
const last = text[e - 1];
|
|
1222
|
+
if (URL_TRAIL.test(last)) {
|
|
1223
|
+
e -= 1;
|
|
1224
|
+
continue;
|
|
1225
|
+
}
|
|
1226
|
+
if ((last === ")" || last === "]") && balance[last] < 0) {
|
|
1227
|
+
balance[last] += 1;
|
|
1228
|
+
e -= 1;
|
|
1229
|
+
continue;
|
|
1230
|
+
}
|
|
1231
|
+
break;
|
|
1232
|
+
}
|
|
1233
|
+
if (e <= start + prefix) continue;
|
|
1234
|
+
const written = text.slice(start, e);
|
|
1235
|
+
const schemeless = prefix === 4;
|
|
1236
|
+
let url;
|
|
1237
|
+
try {
|
|
1238
|
+
url = canonicalUrl(schemeless ? `https://${written}` : written);
|
|
1239
|
+
} catch {
|
|
1240
|
+
continue;
|
|
1241
|
+
}
|
|
1242
|
+
const span = {
|
|
1243
|
+
start,
|
|
1244
|
+
end: e,
|
|
1245
|
+
in: doc.name
|
|
1246
|
+
};
|
|
1247
|
+
out.push({
|
|
1248
|
+
where: {
|
|
1249
|
+
span,
|
|
1250
|
+
path: null
|
|
1251
|
+
},
|
|
1252
|
+
raw: written,
|
|
1253
|
+
exact: written,
|
|
1254
|
+
excerpt: () => excerptOf(text, start, e),
|
|
1255
|
+
readings: [{
|
|
1256
|
+
url,
|
|
1257
|
+
schemeless,
|
|
1258
|
+
relative: false,
|
|
1259
|
+
written
|
|
1260
|
+
}]
|
|
1261
|
+
});
|
|
1262
|
+
}
|
|
1263
|
+
return out;
|
|
1264
|
+
});
|
|
1265
|
+
/** This function gives the stored links of a page as places of a URL. The store keeps the canonical form of each link. */
|
|
1266
|
+
const linkCands = (meta, work) => {
|
|
1267
|
+
const links = meta["links"];
|
|
1268
|
+
if (!Array.isArray(links)) return [];
|
|
1269
|
+
const out = [];
|
|
1270
|
+
for (const [index, link] of links.entries()) {
|
|
1271
|
+
work.steps += 1;
|
|
1272
|
+
if (link === null || typeof link !== "object") continue;
|
|
1273
|
+
const href = link.href;
|
|
1274
|
+
const label = link.text;
|
|
1275
|
+
if (typeof href !== "string") continue;
|
|
1276
|
+
out.push({
|
|
1277
|
+
where: {
|
|
1278
|
+
span: null,
|
|
1279
|
+
path: `links[${String(index)}]`
|
|
1280
|
+
},
|
|
1281
|
+
raw: href,
|
|
1282
|
+
exact: href,
|
|
1283
|
+
excerpt: () => typeof label === "string" && label.length > 0 ? `${label} (${href})` : href,
|
|
1284
|
+
readings: [{
|
|
1285
|
+
url: href,
|
|
1286
|
+
schemeless: false,
|
|
1287
|
+
relative: false,
|
|
1288
|
+
written: href
|
|
1289
|
+
}]
|
|
1290
|
+
});
|
|
1291
|
+
}
|
|
1292
|
+
return out;
|
|
1293
|
+
};
|
|
1294
|
+
/** This function reads a proposed URL, and resolves a relative URL against the URL of the page. Only `http` and `https` URLs are URLs of this kind. */
|
|
1295
|
+
const claimUrl = (value, base) => {
|
|
1296
|
+
if (typeof value === "number") return [];
|
|
1297
|
+
const text = value.trim().replace(/^<(.*)>$/s, "$1");
|
|
1298
|
+
if (text.length === 0 || /\s/.test(text)) return [];
|
|
1299
|
+
const hasScheme = /^[a-z][a-z0-9+.-]*:/i.test(text);
|
|
1300
|
+
const written = !hasScheme && /^www\./i.test(text) ? `//${text}` : text;
|
|
1301
|
+
let resolved;
|
|
1302
|
+
try {
|
|
1303
|
+
resolved = hasScheme ? new URL(written) : new URL(written, base ?? void 0);
|
|
1304
|
+
} catch {
|
|
1305
|
+
return [];
|
|
1306
|
+
}
|
|
1307
|
+
if (resolved.protocol !== "http:" && resolved.protocol !== "https:") return [];
|
|
1308
|
+
return [{
|
|
1309
|
+
url: canonicalUrl(resolved.href),
|
|
1310
|
+
schemeless: false,
|
|
1311
|
+
relative: !hasScheme,
|
|
1312
|
+
written: text
|
|
1313
|
+
}];
|
|
1314
|
+
};
|
|
1315
|
+
/** A text with an `http` or `https` scheme, a `www.` host, or a path from the root. */
|
|
1316
|
+
const LOOKS_LIKE_URL = /^(?:https?:\/\/|www\.|\/|\.\.?\/)/i;
|
|
1317
|
+
const URL_KIND = {
|
|
1318
|
+
compare: (claim, reading) => {
|
|
1319
|
+
const labels = [];
|
|
1320
|
+
if (claim.relative || reading.relative) labels.push("url-resolved");
|
|
1321
|
+
if (reading.schemeless) {
|
|
1322
|
+
if (withoutScheme(claim.url) !== withoutScheme(reading.url)) return void 0;
|
|
1323
|
+
labels.push("url-scheme");
|
|
1324
|
+
return {
|
|
1325
|
+
clean: true,
|
|
1326
|
+
key: claim.url,
|
|
1327
|
+
reading: claim.url,
|
|
1328
|
+
labels
|
|
1329
|
+
};
|
|
1330
|
+
}
|
|
1331
|
+
if (claim.url !== reading.url) return void 0;
|
|
1332
|
+
if (labels.length === 0) labels.push("url-form");
|
|
1333
|
+
return {
|
|
1334
|
+
clean: true,
|
|
1335
|
+
key: reading.url,
|
|
1336
|
+
reading: reading.url,
|
|
1337
|
+
labels
|
|
1338
|
+
};
|
|
1339
|
+
},
|
|
1340
|
+
display: (reading) => reading.url
|
|
1341
|
+
};
|
|
1342
|
+
const LOCAL = /^[A-Za-z0-9._%+-]$/;
|
|
1343
|
+
const DOMAIN = /^[A-Za-z0-9.-]$/;
|
|
1344
|
+
const DOMAIN_NAME = /^[a-z0-9-]+(\.[a-z0-9-]+)*\.[a-z]{2,}$/i;
|
|
1345
|
+
/** This function finds the email addresses of a text. Each unit is read two times at most. */
|
|
1346
|
+
const emailCands = (doc, work) => cached(doc, "email", () => {
|
|
1347
|
+
const text = doc.text;
|
|
1348
|
+
const n = text.length;
|
|
1349
|
+
const out = [];
|
|
1350
|
+
let i = 0;
|
|
1351
|
+
while (i < n) {
|
|
1352
|
+
work.steps += 1;
|
|
1353
|
+
if (text[i] !== "@") {
|
|
1354
|
+
i += 1;
|
|
1355
|
+
continue;
|
|
1356
|
+
}
|
|
1357
|
+
let s = i;
|
|
1358
|
+
while (s > 0 && LOCAL.test(text[s - 1])) {
|
|
1359
|
+
s -= 1;
|
|
1360
|
+
work.steps += 1;
|
|
1361
|
+
}
|
|
1362
|
+
while (s < i && text[s] === ".") s += 1;
|
|
1363
|
+
let e = i + 1;
|
|
1364
|
+
while (e < n && DOMAIN.test(text[e])) {
|
|
1365
|
+
e += 1;
|
|
1366
|
+
work.steps += 1;
|
|
1367
|
+
}
|
|
1368
|
+
const next = e;
|
|
1369
|
+
while (e > i + 1 && (text[e - 1] === "." || text[e - 1] === "-")) e -= 1;
|
|
1370
|
+
if (s < i && DOMAIN_NAME.test(text.slice(i + 1, e))) {
|
|
1371
|
+
const written = text.slice(s, e);
|
|
1372
|
+
const start = s;
|
|
1373
|
+
const end = e;
|
|
1374
|
+
out.push({
|
|
1375
|
+
where: {
|
|
1376
|
+
span: {
|
|
1377
|
+
start,
|
|
1378
|
+
end,
|
|
1379
|
+
in: doc.name
|
|
1380
|
+
},
|
|
1381
|
+
path: null
|
|
1382
|
+
},
|
|
1383
|
+
raw: written,
|
|
1384
|
+
exact: written,
|
|
1385
|
+
excerpt: () => excerptOf(text, start, end),
|
|
1386
|
+
readings: [written.toLowerCase()]
|
|
1387
|
+
});
|
|
1388
|
+
}
|
|
1389
|
+
i = Math.max(next, i + 1);
|
|
1390
|
+
}
|
|
1391
|
+
return out;
|
|
1392
|
+
});
|
|
1393
|
+
const claimEmail = (value) => {
|
|
1394
|
+
if (typeof value === "number") return [];
|
|
1395
|
+
const text = value.trim().replace(/^mailto:/i, "");
|
|
1396
|
+
const at = text.lastIndexOf("@");
|
|
1397
|
+
if (at <= 0 || !/^[A-Za-z0-9._%+-]+$/.test(text.slice(0, at))) return [];
|
|
1398
|
+
return DOMAIN_NAME.test(text.slice(at + 1)) ? [text.toLowerCase()] : [];
|
|
1399
|
+
};
|
|
1400
|
+
const EMAIL = {
|
|
1401
|
+
compare: (claim, reading) => claim === reading ? {
|
|
1402
|
+
clean: true,
|
|
1403
|
+
key: reading,
|
|
1404
|
+
reading,
|
|
1405
|
+
labels: ["email-case"]
|
|
1406
|
+
} : void 0,
|
|
1407
|
+
display: (reading) => reading
|
|
1408
|
+
};
|
|
1409
|
+
/** A key of the JSON-LD goes into a path only when it is a name. Thus a path holds no other text of the page. */
|
|
1410
|
+
const PATH_KEY = /^@?[A-Za-z_][A-Za-z0-9_-]{0,63}$/;
|
|
1411
|
+
/**
|
|
1412
|
+
* This function gives the values of the JSON-LD at a path. The first part of the path names a
|
|
1413
|
+
* schema.org type (`*` is each top-level record), and the path starts at each object of that type,
|
|
1414
|
+
* at any depth. Each array on the path gives each of its values.
|
|
1415
|
+
*/
|
|
1416
|
+
const leavesAt = (records, path, work) => {
|
|
1417
|
+
const [head = "", ...rest] = path.split(".");
|
|
1418
|
+
const type = head.toLowerCase();
|
|
1419
|
+
const starts = [];
|
|
1420
|
+
const stack = [];
|
|
1421
|
+
for (let index = records.length - 1; index >= 0; index -= 1) stack.push({
|
|
1422
|
+
value: records[index],
|
|
1423
|
+
path: `jsonld[${String(index)}]`,
|
|
1424
|
+
depth: 0
|
|
1425
|
+
});
|
|
1426
|
+
while (stack.length > 0) {
|
|
1427
|
+
work.steps += 1;
|
|
1428
|
+
const { value, path: at, depth } = stack.pop();
|
|
1429
|
+
if (Array.isArray(value)) {
|
|
1430
|
+
for (let index = value.length - 1; index >= 0; index -= 1) stack.push({
|
|
1431
|
+
value: value[index],
|
|
1432
|
+
path: `${at}[${String(index)}]`,
|
|
1433
|
+
depth: depth + 1
|
|
1434
|
+
});
|
|
1435
|
+
continue;
|
|
1436
|
+
}
|
|
1437
|
+
if (value === null || typeof value !== "object") continue;
|
|
1438
|
+
const record = value;
|
|
1439
|
+
if (type === "*") {
|
|
1440
|
+
if (depth === 0) starts.push({
|
|
1441
|
+
value: record,
|
|
1442
|
+
path: at
|
|
1443
|
+
});
|
|
1444
|
+
continue;
|
|
1445
|
+
}
|
|
1446
|
+
if (typesOf(record).some((name) => name.toLowerCase() === type)) starts.push({
|
|
1447
|
+
value: record,
|
|
1448
|
+
path: at
|
|
1449
|
+
});
|
|
1450
|
+
const keys = Object.keys(record);
|
|
1451
|
+
for (let index = keys.length - 1; index >= 0; index -= 1) {
|
|
1452
|
+
const key = keys[index];
|
|
1453
|
+
if (PATH_KEY.test(key)) stack.push({
|
|
1454
|
+
value: record[key],
|
|
1455
|
+
path: `${at}.${key}`,
|
|
1456
|
+
depth: depth + 1
|
|
1457
|
+
});
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
1460
|
+
const out = [];
|
|
1461
|
+
for (const start of starts) {
|
|
1462
|
+
const walk = [{
|
|
1463
|
+
value: start.value,
|
|
1464
|
+
path: start.path,
|
|
1465
|
+
depth: 0
|
|
1466
|
+
}];
|
|
1467
|
+
while (walk.length > 0) {
|
|
1468
|
+
work.steps += 1;
|
|
1469
|
+
const { value, path: at, depth } = walk.pop();
|
|
1470
|
+
if (Array.isArray(value)) {
|
|
1471
|
+
for (let index = value.length - 1; index >= 0; index -= 1) walk.push({
|
|
1472
|
+
value: value[index],
|
|
1473
|
+
path: `${at}[${String(index)}]`,
|
|
1474
|
+
depth
|
|
1475
|
+
});
|
|
1476
|
+
continue;
|
|
1477
|
+
}
|
|
1478
|
+
if (depth === rest.length) {
|
|
1479
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") out.push({
|
|
1480
|
+
path: at,
|
|
1481
|
+
value
|
|
1482
|
+
});
|
|
1483
|
+
else if (value !== null && typeof value === "object") {
|
|
1484
|
+
const inner = value["@value"];
|
|
1485
|
+
if (typeof inner === "string" || typeof inner === "number" || typeof inner === "boolean") out.push({
|
|
1486
|
+
path: `${at}.@value`,
|
|
1487
|
+
value: inner
|
|
1488
|
+
});
|
|
1489
|
+
}
|
|
1490
|
+
continue;
|
|
1491
|
+
}
|
|
1492
|
+
if (value === null || typeof value !== "object") continue;
|
|
1493
|
+
const key = rest[depth];
|
|
1494
|
+
const next = value[key];
|
|
1495
|
+
if (next !== void 0) walk.push({
|
|
1496
|
+
value: next,
|
|
1497
|
+
path: `${at}.${key}`,
|
|
1498
|
+
depth: depth + 1
|
|
1499
|
+
});
|
|
1500
|
+
}
|
|
1501
|
+
}
|
|
1502
|
+
return out;
|
|
1503
|
+
};
|
|
1504
|
+
const leafText = (value) => typeof value === "number" ? decimalOf(value) ?? String(value) : String(value);
|
|
1505
|
+
/** This function checks a value against the JSON-LD at a path. It compares as a number, a date, a URL and a text, in that sequence. */
|
|
1506
|
+
const checkJsonLd = (kind, path, value, claimText, source, work) => {
|
|
1507
|
+
const meta = source.meta ?? {};
|
|
1508
|
+
const records = Array.isArray(meta["jsonld"]) ? meta["jsonld"] : [];
|
|
1509
|
+
const leaves = leavesAt(records, path, work);
|
|
1510
|
+
const dropped = meta["jsonld_dropped"];
|
|
1511
|
+
const missing = typeof dropped === "number" && dropped > 0 ? `the store keeps only a part of the JSON-LD of this page (${String(dropped)} records left out), so the path can be in the part that it left out` : null;
|
|
1512
|
+
if (leaves.length === 0) return empty("not-found", kind, missing);
|
|
1513
|
+
const base = source.url ?? null;
|
|
1514
|
+
const cand = (leaf, readings) => ({
|
|
1515
|
+
where: {
|
|
1516
|
+
span: null,
|
|
1517
|
+
path: leaf.path
|
|
1518
|
+
},
|
|
1519
|
+
raw: leaf.value,
|
|
1520
|
+
exact: leafText(leaf.value),
|
|
1521
|
+
excerpt: () => JSON.stringify(leaf.value).slice(0, 200),
|
|
1522
|
+
readings
|
|
1523
|
+
});
|
|
1524
|
+
const stringOf = (leaf) => typeof leaf.value === "string" ? leaf.value : void 0;
|
|
1525
|
+
const folded = fold(claimText, "quote");
|
|
1526
|
+
const TEXT = {
|
|
1527
|
+
compare: (claim, reading) => fold(reading, "quote") === folded ? {
|
|
1528
|
+
clean: true,
|
|
1529
|
+
key: folded,
|
|
1530
|
+
reading: folded,
|
|
1531
|
+
labels: textLabels(reading, claim)
|
|
1532
|
+
} : void 0,
|
|
1533
|
+
display: (reading) => reading
|
|
1534
|
+
};
|
|
1535
|
+
const attempts = [
|
|
1536
|
+
() => {
|
|
1537
|
+
const claims = claimNumber(value);
|
|
1538
|
+
if (claims.length === 0) return void 0;
|
|
1539
|
+
const cands = leaves.map((leaf) => cand(leaf, typeof leaf.value === "boolean" ? [] : claimNumber(leaf.value)));
|
|
1540
|
+
return decide(kind, claimText, claims, cands, NUMBER, work);
|
|
1541
|
+
},
|
|
1542
|
+
() => {
|
|
1543
|
+
const claim = claimDate(value);
|
|
1544
|
+
if (claim.readings.length === 0) return void 0;
|
|
1545
|
+
const cands = leaves.map((leaf) => {
|
|
1546
|
+
const text = stringOf(leaf);
|
|
1547
|
+
return cand(leaf, text === void 0 ? [] : claimDate(text).readings);
|
|
1548
|
+
});
|
|
1549
|
+
return decide(kind, claimText, claim.readings, cands, DATE, work);
|
|
1550
|
+
},
|
|
1551
|
+
() => {
|
|
1552
|
+
const claims = LOOKS_LIKE_URL.test(claimText) ? claimUrl(value, base) : [];
|
|
1553
|
+
if (claims.length === 0) return void 0;
|
|
1554
|
+
const cands = leaves.map((leaf) => {
|
|
1555
|
+
const text = stringOf(leaf);
|
|
1556
|
+
return cand(leaf, text === void 0 || !LOOKS_LIKE_URL.test(text.trim()) ? [] : claimUrl(text, base));
|
|
1557
|
+
});
|
|
1558
|
+
return decide(kind, claimText, claims, cands, URL_KIND, work);
|
|
1559
|
+
},
|
|
1560
|
+
() => {
|
|
1561
|
+
if (folded.length === 0) return void 0;
|
|
1562
|
+
const cands = leaves.map((leaf) => cand(leaf, [leafText(leaf.value)]));
|
|
1563
|
+
return decide(kind, claimText, [claimText], cands, TEXT, work);
|
|
1564
|
+
}
|
|
1565
|
+
];
|
|
1566
|
+
for (const attempt of attempts) {
|
|
1567
|
+
const result = attempt();
|
|
1568
|
+
if (result !== void 0 && result.status !== "not-found") return result;
|
|
1569
|
+
}
|
|
1570
|
+
return empty("not-found", kind, missing);
|
|
1571
|
+
};
|
|
1572
|
+
/** This function gives the whole text of a page that the reader cut, or null. It folds that text at its first use only. */
|
|
1573
|
+
const fullOf = (context) => {
|
|
1574
|
+
if (context.full === void 0) {
|
|
1575
|
+
const full = context.source.full_text;
|
|
1576
|
+
context.full = full === void 0 || full === context.source.text ? null : prepare("full_text", full, context.work);
|
|
1577
|
+
}
|
|
1578
|
+
return context.full;
|
|
1579
|
+
};
|
|
1580
|
+
/** This function checks a field in the text. When the reader cut the text and the text does not hold the value, it checks the whole text too. */
|
|
1581
|
+
const inText = (context, check) => {
|
|
1582
|
+
const first = check(context.text);
|
|
1583
|
+
if (first.status !== "not-found") return first;
|
|
1584
|
+
const full = fullOf(context);
|
|
1585
|
+
if (full === null) return first;
|
|
1586
|
+
const second = check(full);
|
|
1587
|
+
return second.status === "not-found" ? first : second;
|
|
1588
|
+
};
|
|
1589
|
+
/** This function reads the kind of a field. A kind that is not a kind throws a `RangeError`. */
|
|
1590
|
+
const kindOf = (name, claim) => {
|
|
1591
|
+
const kind = claim.kind ?? (typeof claim.value === "number" ? "number" : "text");
|
|
1592
|
+
if (FIELD_KINDS.includes(kind)) return kind;
|
|
1593
|
+
if (kind.startsWith("json-ld:") && kind.length > 8 && !/\s/.test(kind)) return kind;
|
|
1594
|
+
throw new RangeError(`the kind of the field ${name} is ${FIELD_KINDS.join(", ")} or json-ld:<path>, not ${JSON.stringify(kind)}`);
|
|
1595
|
+
};
|
|
1596
|
+
const INVALID = {
|
|
1597
|
+
number: "the value is not a number",
|
|
1598
|
+
money: "the value is not an amount of money",
|
|
1599
|
+
date: "the value is not a date",
|
|
1600
|
+
url: "the value is not an http or https URL",
|
|
1601
|
+
email: "the value is not an email address",
|
|
1602
|
+
phone: "the value is not a phone number of 7 to 15 digits"
|
|
1603
|
+
};
|
|
1604
|
+
const checkOne = (name, claim, context) => {
|
|
1605
|
+
const kind = kindOf(name, claim);
|
|
1606
|
+
const value = claim.value;
|
|
1607
|
+
if (typeof value === "number" && !Number.isFinite(value)) return empty("invalid", kind, "the value is not a finite number");
|
|
1608
|
+
const prefix = kind === "email" ? /^mailto:/i : kind === "phone" ? /^tel:/i : void 0;
|
|
1609
|
+
const trimmed = typeof value === "number" ? decimalOf(value) ?? "" : value.trim();
|
|
1610
|
+
const claimText = prefix === void 0 ? trimmed : trimmed.replace(prefix, "");
|
|
1611
|
+
if (claimText.length === 0) return empty("invalid", kind, "the value is empty");
|
|
1612
|
+
const work = context.work;
|
|
1613
|
+
const invalid = () => empty("invalid", kind, INVALID[kind] ?? "the value is not valid");
|
|
1614
|
+
switch (kind) {
|
|
1615
|
+
case "text": return inText(context, (doc) => checkText(doc, claimText, work));
|
|
1616
|
+
case "number": {
|
|
1617
|
+
const claims = claimNumber(value);
|
|
1618
|
+
if (claims.length === 0) return invalid();
|
|
1619
|
+
return inText(context, (doc) => decide(kind, claimText, claims, numberCands(doc, work), NUMBER, work));
|
|
1620
|
+
}
|
|
1621
|
+
case "money": {
|
|
1622
|
+
const claims = claimMoney(value);
|
|
1623
|
+
if (claims.length === 0) return invalid();
|
|
1624
|
+
return inText(context, (doc) => decide(kind, claimText, claims, moneyCands(doc, work), MONEY, work));
|
|
1625
|
+
}
|
|
1626
|
+
case "date": {
|
|
1627
|
+
const claim = claimDate(value);
|
|
1628
|
+
if (claim.readings.length === 0) return invalid();
|
|
1629
|
+
const result = inText(context, (doc) => decide(kind, claimText, claim.readings, dateCands(doc, work), DATE, work));
|
|
1630
|
+
if (!claim.time || result.status === "verified") return result;
|
|
1631
|
+
const timeNote = "the check compares the date of the value, not its time";
|
|
1632
|
+
return {
|
|
1633
|
+
...result,
|
|
1634
|
+
note: result.note === null ? timeNote : `${result.note}; ${timeNote}`
|
|
1635
|
+
};
|
|
1636
|
+
}
|
|
1637
|
+
case "url": {
|
|
1638
|
+
const claims = claimUrl(value, context.source.url ?? null);
|
|
1639
|
+
if (claims.length === 0) return invalid();
|
|
1640
|
+
const links = linkCands(context.source.meta ?? {}, work);
|
|
1641
|
+
return inText(context, (doc) => decide(kind, claimText, claims, [...urlCands(doc, work), ...links], URL_KIND, work));
|
|
1642
|
+
}
|
|
1643
|
+
case "email": {
|
|
1644
|
+
const claims = claimEmail(value);
|
|
1645
|
+
if (claims.length === 0) return invalid();
|
|
1646
|
+
return inText(context, (doc) => decide(kind, claimText, claims, emailCands(doc, work), EMAIL, work));
|
|
1647
|
+
}
|
|
1648
|
+
case "phone": {
|
|
1649
|
+
const claims = claimPhone(value);
|
|
1650
|
+
if (claims.length === 0) return invalid();
|
|
1651
|
+
return inText(context, (doc) => decide(kind, claimText, claims, phoneCands(doc, work), PHONE, work));
|
|
1652
|
+
}
|
|
1653
|
+
default: return checkJsonLd(kind, kind.slice(8), value, claimText, context.source, work);
|
|
1654
|
+
}
|
|
1655
|
+
};
|
|
1656
|
+
/** This function validates the fields, and gives their names and values. A field that is not valid throws a `RangeError`. */
|
|
1657
|
+
const claimsOf = (fields) => {
|
|
1658
|
+
if (fields === null || typeof fields !== "object" || Array.isArray(fields)) throw new RangeError("fields must be an object of names and values");
|
|
1659
|
+
const entries = Object.entries(fields);
|
|
1660
|
+
for (const [name, claim] of entries) {
|
|
1661
|
+
if (name.length === 0) throw new RangeError("the name of a field must not be empty");
|
|
1662
|
+
if (claim === null || typeof claim !== "object") throw new RangeError(`the field ${name} must be an object with a value`);
|
|
1663
|
+
if (typeof claim.value !== "string" && typeof claim.value !== "number") throw new RangeError(`the value of the field ${name} must be a text or a number`);
|
|
1664
|
+
kindOf(name, claim);
|
|
1665
|
+
}
|
|
1666
|
+
return entries;
|
|
1667
|
+
};
|
|
1668
|
+
const summaryOf = (statuses) => {
|
|
1669
|
+
const count = (status) => statuses.filter((one) => one === status).length;
|
|
1670
|
+
return {
|
|
1671
|
+
fields: statuses.length,
|
|
1672
|
+
verified: count("verified"),
|
|
1673
|
+
normalized: count("normalized"),
|
|
1674
|
+
ambiguous: count("ambiguous"),
|
|
1675
|
+
not_found: count("not-found"),
|
|
1676
|
+
held_back: count("held-back"),
|
|
1677
|
+
invalid: count("invalid")
|
|
1678
|
+
};
|
|
1679
|
+
};
|
|
1680
|
+
/**
|
|
1681
|
+
* This function checks the proposed values of fields against the text and the metadata of a
|
|
1682
|
+
* document. It sends no request and writes nothing. It does not make values. A field that is not
|
|
1683
|
+
* valid throws a `RangeError`. The work grows in proportion to the length of the text.
|
|
1684
|
+
*/
|
|
1685
|
+
const checkFields = (source, fields) => {
|
|
1686
|
+
const claims = claimsOf(fields);
|
|
1687
|
+
const work = { steps: 0 };
|
|
1688
|
+
const context = {
|
|
1689
|
+
source,
|
|
1690
|
+
text: prepare("text", source.text, work),
|
|
1691
|
+
full: void 0,
|
|
1692
|
+
work
|
|
1693
|
+
};
|
|
1694
|
+
const checked = {};
|
|
1695
|
+
for (const [name, claim] of claims) checked[name] = checkOne(name, claim, context);
|
|
1696
|
+
return {
|
|
1697
|
+
fields: checked,
|
|
1698
|
+
summary: summaryOf(Object.values(checked).map((field) => field.status)),
|
|
1699
|
+
steps: work.steps
|
|
1700
|
+
};
|
|
1701
|
+
};
|
|
1702
|
+
const withCapture = (fields, capture, fetchedAt) => Object.fromEntries(Object.entries(fields).map(([name, field]) => [name, {
|
|
1703
|
+
...field,
|
|
1704
|
+
capture,
|
|
1705
|
+
fetched_at: fetchedAt
|
|
1706
|
+
}]));
|
|
1707
|
+
/** This function gives the same result for each field, with no check: for a held-back document, or a time before the first capture. */
|
|
1708
|
+
const sameForEach = (claims, status, note) => Object.fromEntries(claims.map(([name, claim]) => [name, {
|
|
1709
|
+
...empty(status, kindOf(name, claim), note),
|
|
1710
|
+
capture: null,
|
|
1711
|
+
fetched_at: null
|
|
1712
|
+
}]));
|
|
1713
|
+
/**
|
|
1714
|
+
* This function checks the proposed values of fields against a stored document. It does not make
|
|
1715
|
+
* values. It applies the rules of `verify`:
|
|
1716
|
+
*
|
|
1717
|
+
* - With no option, it checks the current version. `capture` checks the version of one capture,
|
|
1718
|
+
* and `at` checks the version that was current at a time.
|
|
1719
|
+
* - A held-back document gives `held-back` for each field, with no position and no capture, unless
|
|
1720
|
+
* `userProvided`. The crawler then logs an override.
|
|
1721
|
+
* - A field that is not valid throws a `RangeError`, and an unknown document throws an error.
|
|
1722
|
+
*/
|
|
1723
|
+
const extractDocument = async (runtime, id, fields, options = {}) => {
|
|
1724
|
+
const claims = claimsOf(fields);
|
|
1725
|
+
if (options.at !== void 0 && options.capture !== void 0) throw new RangeError("give capture or at, not the two: each one chooses a version");
|
|
1726
|
+
const at = options.at === void 0 ? void 0 : isoTime("at", options.at);
|
|
1727
|
+
const doc = runtime.index.get(id);
|
|
1728
|
+
if (doc === void 0) throw new Error(`no stored document ${id}: find one with find(), or fetch its page`);
|
|
1729
|
+
let version;
|
|
1730
|
+
let first;
|
|
1731
|
+
if (options.capture !== void 0) {
|
|
1732
|
+
version = runtime.index.versionText(id, options.capture);
|
|
1733
|
+
if (version === void 0) throw new Error(`the document ${id} has no version from the capture ${options.capture}`);
|
|
1734
|
+
} else if (at !== void 0) ({version, first} = versionAtTime(runtime, id, at));
|
|
1735
|
+
else version = runtime.index.versionText(id);
|
|
1736
|
+
const aiInput = doc.ai_input === "no" || version?.ai_input === "no" ? "no" : version?.ai_input ?? doc.ai_input;
|
|
1737
|
+
const userProvided = options.userProvided ?? false;
|
|
1738
|
+
const base = {
|
|
1739
|
+
doc: doc.doc,
|
|
1740
|
+
url: doc.url,
|
|
1741
|
+
captures: doc.captures,
|
|
1742
|
+
...at === void 0 ? {} : { at }
|
|
1743
|
+
};
|
|
1744
|
+
if (isHeldBack({ ai_input: aiInput }, userProvided)) return {
|
|
1745
|
+
...base,
|
|
1746
|
+
capture: null,
|
|
1747
|
+
blob: null,
|
|
1748
|
+
fetched_at: null,
|
|
1749
|
+
held_back: true,
|
|
1750
|
+
fields: sameForEach(claims, "held-back", HELD_BACK),
|
|
1751
|
+
summary: summaryOf(claims.map(() => "held-back")),
|
|
1752
|
+
note: HELD_BACK
|
|
1753
|
+
};
|
|
1754
|
+
if (aiInput === "no" && userProvided) await recordOverride(runtime, doc, "extract");
|
|
1755
|
+
if (version === void 0) {
|
|
1756
|
+
const note = `the document ${id} has no version at ${at ?? "this time"} or before${first === void 0 ? "" : `: its first capture is from ${first}`}`;
|
|
1757
|
+
return {
|
|
1758
|
+
...base,
|
|
1759
|
+
capture: null,
|
|
1760
|
+
blob: null,
|
|
1761
|
+
fetched_at: null,
|
|
1762
|
+
held_back: false,
|
|
1763
|
+
fields: sameForEach(claims, "not-found", note),
|
|
1764
|
+
summary: summaryOf(claims.map(() => "not-found")),
|
|
1765
|
+
note,
|
|
1766
|
+
before_first: true
|
|
1767
|
+
};
|
|
1768
|
+
}
|
|
1769
|
+
const checked = checkFields({
|
|
1770
|
+
text: version.text,
|
|
1771
|
+
full_text: version.full_text,
|
|
1772
|
+
url: version.url ?? doc.url,
|
|
1773
|
+
meta: version.meta
|
|
1774
|
+
}, Object.fromEntries(claims));
|
|
1775
|
+
const incomplete = version.meta["truncated"] === true && version.full_text === void 0;
|
|
1776
|
+
const archived = archivedOf(version.meta);
|
|
1777
|
+
const notes = [...incomplete && checked.summary.not_found > 0 ? ["the reader stopped before the end of the file, so a value that is not found can be in the part that it did not read"] : [], ...archived === void 0 ? [] : [archivedNote(archived)]];
|
|
1778
|
+
return {
|
|
1779
|
+
...base,
|
|
1780
|
+
capture: version.capture,
|
|
1781
|
+
blob: version.blob,
|
|
1782
|
+
fetched_at: version.fetched_at,
|
|
1783
|
+
held_back: false,
|
|
1784
|
+
fields: withCapture(checked.fields, version.capture, version.fetched_at),
|
|
1785
|
+
summary: checked.summary,
|
|
1786
|
+
note: notes.length > 0 ? notes.join("; ") : null,
|
|
1787
|
+
...archived === void 0 ? {} : { archived }
|
|
1788
|
+
};
|
|
1789
|
+
};
|
|
1790
|
+
//#endregion
|
|
1791
|
+
export { extract_exports as i, checkFields as n, extractDocument as r, FIELD_KINDS as t };
|
|
1792
|
+
|
|
1793
|
+
//# sourceMappingURL=extract-DcwNrTK7.mjs.map
|