pi-wsearch 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -11
- package/package.json +1 -1
- package/src/cache.ts +66 -34
- package/src/config.ts +14 -1
- package/src/dom.ts +13 -12
- package/src/engines.ts +298 -308
- package/src/fetch.ts +1 -5
- package/src/fuse.ts +27 -15
- package/src/keys.ts +2 -22
- package/src/login.ts +1 -1
- package/src/scrape.ts +58 -481
- package/src/text.ts +418 -0
- package/src/tools.ts +55 -66
- package/src/transport.ts +144 -113
- package/src/view.ts +2 -7
- package/src/constants.ts +0 -6
package/src/scrape.ts
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Fetch orchestration
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* Fetch orchestration: URL validation, GitHub README shortcuts, local
|
|
3
|
+
* extraction, archive.org fallback for dead pages, reader chain dispatch.
|
|
4
|
+
* Pure text pipeline (entity decode, extraction, filters, sectioning) lives
|
|
5
|
+
* in text.ts; HTTP + reader backends in transport.ts. SSRF: private/loopback
|
|
6
|
+
* numeric literals are rejected on the local path; see isPrivateHost for the
|
|
7
|
+
* deliberate scope.
|
|
5
8
|
*/
|
|
6
9
|
import { getConfig, sanitizeError } from './config';
|
|
7
|
-
import { ABORT_ERROR } from './
|
|
10
|
+
import { ABORT_ERROR } from './config';
|
|
8
11
|
import { isPrivateHost, validateUrl } from './urls';
|
|
9
12
|
import {
|
|
10
13
|
localFetch,
|
|
@@ -14,11 +17,26 @@ import {
|
|
|
14
17
|
isLowQualityOutput,
|
|
15
18
|
stripDataUriImages,
|
|
16
19
|
withTimeout,
|
|
17
|
-
DEFAULT_SECTION_CAP,
|
|
18
20
|
type RemoteDoc,
|
|
19
21
|
type Renderer,
|
|
20
22
|
} from './transport';
|
|
21
|
-
import {
|
|
23
|
+
import {
|
|
24
|
+
cleanReaderMarkdown,
|
|
25
|
+
decodeHtmlEntities,
|
|
26
|
+
DEFAULT_SECTION_CAP,
|
|
27
|
+
extractHtml,
|
|
28
|
+
joinSections,
|
|
29
|
+
notFoundDoc,
|
|
30
|
+
sliceSections,
|
|
31
|
+
stripToAscii,
|
|
32
|
+
} from './text';
|
|
33
|
+
|
|
34
|
+
export { isPrivateHost, validateUrl } from './urls'; // public surface unchanged; the gates live in the URL leaf
|
|
35
|
+
export { cleanReaderMarkdown, decodeHtmlEntities, extractHtml, joinSections, notFoundDoc, sliceSections } from './text';
|
|
36
|
+
|
|
37
|
+
const DEFAULT_MAX_CHARS = 12000;
|
|
38
|
+
const LOCAL_MIN_TEXT = 240;
|
|
39
|
+
const LOCAL_MIN_OUTPUT = 200;
|
|
22
40
|
|
|
23
41
|
export interface ScrapeResult {
|
|
24
42
|
url: string;
|
|
@@ -35,371 +53,15 @@ export interface ScrapeResult {
|
|
|
35
53
|
error?: string;
|
|
36
54
|
}
|
|
37
55
|
|
|
38
|
-
export { isPrivateHost, validateUrl } from './urls'; // public surface unchanged; the gates live in the URL leaf
|
|
39
|
-
|
|
40
|
-
const DEFAULT_MAX_CHARS = 12000;
|
|
41
|
-
|
|
42
|
-
function stripToAscii(s: string): string {
|
|
43
|
-
return s
|
|
44
|
-
.replace(/\u00a0/g, ' ')
|
|
45
|
-
.replace(/\u200b/g, '')
|
|
46
|
-
.replace(/\r/g, '');
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
const NAMED_ENTITIES: Record<string, string> = {
|
|
50
|
-
amp: '&',
|
|
51
|
-
lt: '<',
|
|
52
|
-
gt: '>',
|
|
53
|
-
quot: '"',
|
|
54
|
-
apos: "'",
|
|
55
|
-
nbsp: '\u00a0',
|
|
56
|
-
ndash: '–',
|
|
57
|
-
mdash: '—',
|
|
58
|
-
hellip: '…',
|
|
59
|
-
copy: '©',
|
|
60
|
-
reg: '®',
|
|
61
|
-
trade: '™',
|
|
62
|
-
laquo: '«',
|
|
63
|
-
raquo: '»',
|
|
64
|
-
lsquo: '‘',
|
|
65
|
-
rsquo: '’',
|
|
66
|
-
ldquo: '“',
|
|
67
|
-
rdquo: '”',
|
|
68
|
-
bull: '•',
|
|
69
|
-
middot: '·',
|
|
70
|
-
deg: '°',
|
|
71
|
-
plusmn: '±',
|
|
72
|
-
times: '×',
|
|
73
|
-
divide: '÷',
|
|
74
|
-
frac12: '½',
|
|
75
|
-
sup2: '²',
|
|
76
|
-
sup3: '³',
|
|
77
|
-
ensp: ' ',
|
|
78
|
-
emsp: ' ',
|
|
79
|
-
thinsp: ' ',
|
|
80
|
-
};
|
|
81
|
-
|
|
82
|
-
export function decodeHtmlEntities(s: string): string {
|
|
83
|
-
return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z][a-z0-9]*);/gi, (m, body: string) => {
|
|
84
|
-
if (body[0] === '#') {
|
|
85
|
-
const hex = body[1] === 'x' || body[1] === 'X';
|
|
86
|
-
const n = hex ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
|
87
|
-
if (!Number.isFinite(n) || n <= 0 || n > 0x10ffff || (n >= 0xd800 && n <= 0xdfff)) return m;
|
|
88
|
-
try {
|
|
89
|
-
return String.fromCodePoint(n);
|
|
90
|
-
} catch {
|
|
91
|
-
return m;
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
return NAMED_ENTITIES[body.toLowerCase()] ?? m;
|
|
95
|
-
});
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
export function extractHtml(html: string): { title: string; text: string; publishedAt?: string; paywalled: boolean } {
|
|
99
|
-
const paywalled =
|
|
100
|
-
/class="[^"]*(?:fc-ab-root|metering-modal|paywall|pw-overlay|subscription-overlay|regwall)[^"]*"/i.test(html) ||
|
|
101
|
-
/id="[^"]*(?:paywall|metering|pay-overlay)[^"]*"/i.test(html) ||
|
|
102
|
-
/<meta[^>]+(?:property|name)=["'](?:isAccessibleForFree|tunnel:[^"']*paywall)["'][^>]+content=["'](?:false|yes)["']/i.test(
|
|
103
|
-
html,
|
|
104
|
-
);
|
|
105
|
-
const { title, publishedAt } = headScan(html);
|
|
106
|
-
let h = treeToMarkdownish(html).replace(/\uE000/g, '\n');
|
|
107
|
-
h = stripToAscii(decodeHtmlEntities(h));
|
|
108
|
-
const lines = h
|
|
109
|
-
.split('\n')
|
|
110
|
-
.map((l) => l.trim())
|
|
111
|
-
.filter((l) => l.length > 0 && !/^•\s*$/.test(l))
|
|
112
|
-
.filter((l) => !(/^[-*•]\s/.test(l) && isNavBullet(l)))
|
|
113
|
-
.filter((l) => !/^#{1,6}\s*$/.test(l))
|
|
114
|
-
.map((l) => (/^[-*•]\s{2,}/.test(l) ? l.replace(/^[-*•]\s+/, '').replace(/\s{2,}/g, ' ') : l))
|
|
115
|
-
.filter((l) => !(l.length < 120 && isPromo(l)));
|
|
116
|
-
const out: string[] = [];
|
|
117
|
-
let blank = 0;
|
|
118
|
-
for (const l of lines) {
|
|
119
|
-
if (l.startsWith('#')) {
|
|
120
|
-
if (out.length) out.push('');
|
|
121
|
-
out.push(l);
|
|
122
|
-
blank = 0;
|
|
123
|
-
continue;
|
|
124
|
-
}
|
|
125
|
-
if (out.length === 0 || /^#{1,6} /.test(out[out.length - 1])) {
|
|
126
|
-
out.push(l);
|
|
127
|
-
blank = 0;
|
|
128
|
-
continue;
|
|
129
|
-
}
|
|
130
|
-
if (/^[-*•]\s|^\d{1,2}[.)]\s/.test(l) && l.length < 80) {
|
|
131
|
-
const prev = out[out.length - 1];
|
|
132
|
-
if (prev && /^[-*•]\s|^\d{1,2}[.)]\s/.test(prev)) out.push(l);
|
|
133
|
-
else {
|
|
134
|
-
out.push('');
|
|
135
|
-
out.push(l);
|
|
136
|
-
}
|
|
137
|
-
blank = 0;
|
|
138
|
-
continue;
|
|
139
|
-
}
|
|
140
|
-
if (l.length < SHORT_LINE_JOIN && blank < 2) {
|
|
141
|
-
out[out.length - 1] += ' ' + l;
|
|
142
|
-
blank++;
|
|
143
|
-
continue;
|
|
144
|
-
}
|
|
145
|
-
out.push('');
|
|
146
|
-
out.push(l);
|
|
147
|
-
blank = 0;
|
|
148
|
-
}
|
|
149
|
-
return { title, publishedAt, paywalled, text: out.join('\n').replace(/\n{3,}/g, '\n\n') };
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
const NAV_LABEL_RE =
|
|
153
|
-
/^(home|menu|about( us)?|docs?|documentation|downloads?|license|licence|support|purchase|buy|pricing|search|login|log ?in|sign ?(in|up|out)|register|account|blog|news|contact|careers?|jobs|store|shop|forum|community|tutorials?|api|guide|guides|reference|examples?|faq|help|terms|privacy|cookies?|sitemap|rss|changelog|releases?|twitter|facebook|github|discord|youtube|linkedin|mastodon|x)$/i;
|
|
154
|
-
|
|
155
|
-
function isBoilerplate(block: string): boolean {
|
|
156
|
-
const t = block.trim();
|
|
157
|
-
const words = t.split(/\s{2,}|\s*[|·>]\s*|\s+/).filter(Boolean);
|
|
158
|
-
if (t.length < NAV_BLOCK_MAX && words.length >= 2 && words.every((w) => NAV_LABEL_RE.test(w.trim()))) return true;
|
|
159
|
-
const tldr = t.length < 120;
|
|
160
|
-
if (tldr) {
|
|
161
|
-
if (
|
|
162
|
-
/^(skip to|jump to|table of contents|related (articles|posts|links)|share this|back to top|sign (in|up| out)|subscribe to|newsletter|follow (us|@)|menu|navigation|breadcrumbs\b|tags:|categories:|©|all rights reserved|privacy policy|terms of (use|service)|cookie|cookies|accept )/i.test(
|
|
163
|
-
t,
|
|
164
|
-
)
|
|
165
|
-
)
|
|
166
|
-
return true;
|
|
167
|
-
}
|
|
168
|
-
const links = t.match(/\[[^\]]+\]\([^)]+\)/g) ?? [];
|
|
169
|
-
if (links.length) {
|
|
170
|
-
const rest = t.replace(/\[[^\]]+\]\([^)]+\)/g, '').trim();
|
|
171
|
-
if (rest.length === 0) return true;
|
|
172
|
-
if (rest.length < 8 && links.length >= 3) return true;
|
|
173
|
-
}
|
|
174
|
-
return false;
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
function isNavBullet(line: string): boolean {
|
|
178
|
-
const t = line
|
|
179
|
-
.replace(/^[-*•]\s*/, '')
|
|
180
|
-
.replace(/\s+/g, ' ')
|
|
181
|
-
.trim();
|
|
182
|
-
if (!t) return true;
|
|
183
|
-
if (t.length > NAV_LABEL_MAX) return false;
|
|
184
|
-
const words = t.split(/\s{2,}|\s*[|·>]\s*/).filter(Boolean);
|
|
185
|
-
if (words.length > 1) return words.every((w) => NAV_LABEL_RE.test(w.trim()));
|
|
186
|
-
return NAV_LABEL_RE.test(t);
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
function scrubNavBullets(text: string): string {
|
|
190
|
-
const lines = text.split('\n');
|
|
191
|
-
const kept = lines.filter((l) => !/^[-*•]\s/.test(l) || !isNavBullet(l));
|
|
192
|
-
return kept.join('\n').replace(/\n{3,}/g, '\n\n');
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
const normHeading = (s: string): string =>
|
|
196
|
-
s
|
|
197
|
-
.replace(/^[#\d.\s]+/, '')
|
|
198
|
-
.replace(/[^a-z0-9 ]/gi, ' ')
|
|
199
|
-
.replace(/\s+/g, ' ')
|
|
200
|
-
.trim()
|
|
201
|
-
.toLowerCase();
|
|
202
|
-
|
|
203
|
-
function isOutlineEcho(block: string, outline: string[]): boolean {
|
|
204
|
-
if (outline.length < MIN_OUTLINE_FOR_ECHO) return false;
|
|
205
|
-
const trimmed = block.trim();
|
|
206
|
-
if (/\btable of contents\b/i.test(trimmed) && trimmed.length < 80) return true;
|
|
207
|
-
const heads = outline.map(normHeading).filter(Boolean);
|
|
208
|
-
const raw = trimmed
|
|
209
|
-
.split('\n')
|
|
210
|
-
.flatMap((l) => l.split(/(?=\d+(?:\.\d+)*\.\s)/))
|
|
211
|
-
.map((l) => l.replace(/^[-*•]?\s*\d*\.?\s*/, '').trim())
|
|
212
|
-
.filter((l) => l.length > 2 && l.length < OUTLINE_LINE_MAX);
|
|
213
|
-
if (raw.length < MIN_OUTLINE_FOR_ECHO) return false;
|
|
214
|
-
const hit = (l: string): boolean => {
|
|
215
|
-
const n = normHeading(l);
|
|
216
|
-
return heads.some((h) => h === n || h.startsWith(n + ' ') || n.startsWith(h + ' '));
|
|
217
|
-
};
|
|
218
|
-
return raw.filter(hit).length / raw.length >= 0.6;
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
const PROMO_STRONG = [
|
|
222
|
-
/become a (?:premium )?member/i,
|
|
223
|
-
/unlock exclusive/i,
|
|
224
|
-
/go premium/i,
|
|
225
|
-
/upgrade to (?:the )?premium/i,
|
|
226
|
-
/your membership journey/i,
|
|
227
|
-
/stay on the cutting edge/i,
|
|
228
|
-
/enter your email below/i,
|
|
229
|
-
/by submitting your information/i,
|
|
230
|
-
/sign (?:up|you up) (?:for|to) (?:our )?(?:free )?newsletter/i,
|
|
231
|
-
/subscribe to (?:our )?newsletter/i,
|
|
232
|
-
/get the [a-z][a-z ]{2,40}newsletter/i,
|
|
233
|
-
/limited-?time (?:premium )?offer/i,
|
|
234
|
-
/choose how you want to join/i,
|
|
235
|
-
/explore (?:our )?(?:premium|pro)\b/i,
|
|
236
|
-
/^email newsletter\b/i,
|
|
237
|
-
/^your \(?[\w\s-]+\)? email$/i,
|
|
238
|
-
/^IE \d+ is not supported/i,
|
|
239
|
-
/for an optimal experience visit our site/i,
|
|
240
|
-
/^skip to (?:content|main)/i,
|
|
241
|
-
/^for subscribers$/i,
|
|
242
|
-
/^subscriber (?:only|exclusive)$/i,
|
|
243
|
-
/^[A-Z][A-Z0-9 &'’-]{5,50}$/,
|
|
244
|
-
/\|\s*[A-Z][\w'’&. ]{2,40}\s+\|\s*[A-Z][\w'’&. ]{2,40}\s+News\s*$/,
|
|
245
|
-
/^(?:[A-Z][\w'’.\- ]{2,30}\s+\/\s+)?(?:via )?(?:Getty Images(?:\s+file)?|AP(?: Photo)?|Reuters|Xinhua(?:[\s\w/]+)?|AFP|Bloomberg(?: Photo)?|The Associated Press|NBC(?: News)? via|CNN Wire|Sipa USA|AP Photo\/[^,]{2,40})[\s.]*$/i,
|
|
246
|
-
/^\d+\s*(?:s|sec|seconds?|m|min|minutes?|h|hr|hours?|d|day|days|w|weeks?)\s+ago[\s.]*$/i,
|
|
247
|
-
/^(?:updated|published)\s+(?:just\s+now|\d+\s*\w+\s+ago)$/i,
|
|
248
|
-
/^\d+\s+(?:hours?|minutes?)\s+ago$/i,
|
|
249
|
-
/get started with free access/i,
|
|
250
|
-
/premium (?:newsletter|articles?|membership|features|offer)/i,
|
|
251
|
-
/sign[- ]?up (?:was|is) successful/i,
|
|
252
|
-
/we use cookies[^.]*(?:to (?:improve|enhance|personalize|analyse|analyze))/i,
|
|
253
|
-
/^by continuing (?:to use|to browse)/i,
|
|
254
|
-
/^(?:accept|allow) (?:all|selected|necessary)/i,
|
|
255
|
-
/^(?:manage|customize|consent) (?:preferences|settings|choices)/i,
|
|
256
|
-
/^this (?:site|website) uses (?:cookies|essential cookies)/i,
|
|
257
|
-
];
|
|
258
|
-
const PROMO_CTA_RE =
|
|
259
|
-
/\b(?:newsletter|membership|club membership|premium member|sign ?up|subscribe|unlock|upgrade|cancel anytime|limited-?time|free trial|join now|join today)\b/gi;
|
|
260
|
-
|
|
261
|
-
function isPromo(t: string): boolean {
|
|
262
|
-
if (PROMO_STRONG.some((r) => r.test(t))) return true;
|
|
263
|
-
return (t.match(PROMO_CTA_RE) ?? []).length >= 2;
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
function splitSection(s: string, sectionCap: number): string[] {
|
|
267
|
-
const hm = /^(#{1,6})\s+([^\n]+)/.exec(s);
|
|
268
|
-
const partHead = hm ? `${'#'.repeat(hm[1].length)} ${hm[2].trim()}\n\n` : '';
|
|
269
|
-
const floor = Math.min(partHead.length + 1, sectionCap);
|
|
270
|
-
const parts: string[] = [];
|
|
271
|
-
let rest = s;
|
|
272
|
-
for (;;) {
|
|
273
|
-
if (rest.length <= sectionCap) {
|
|
274
|
-
parts.push(rest);
|
|
275
|
-
return parts;
|
|
276
|
-
}
|
|
277
|
-
const cut = rest.slice(0, sectionCap);
|
|
278
|
-
let at = cut.lastIndexOf('\n\n');
|
|
279
|
-
if (at < floor) at = cut.lastIndexOf(' ');
|
|
280
|
-
if (at < floor) at = sectionCap;
|
|
281
|
-
const used = Math.min(at, rest.length - 1);
|
|
282
|
-
parts.push(rest.slice(0, used).replace(/\s+$/, ''));
|
|
283
|
-
rest = partHead + rest.slice(used).trimStart();
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
export function sliceSections(
|
|
288
|
-
text: string,
|
|
289
|
-
sectionCap = DEFAULT_SECTION_CAP,
|
|
290
|
-
docTitle = '',
|
|
291
|
-
): { outline: string[]; sections: string[]; continued: number[] } {
|
|
292
|
-
const titleEcho = docTitle ? normHeading(docTitle) : '';
|
|
293
|
-
const blocks = text.replace(/\n{3,}/g, '\n\n').split('\n\n');
|
|
294
|
-
const sections: string[] = [];
|
|
295
|
-
const outlineNames: string[] = [];
|
|
296
|
-
const seenParas = new Set<string>();
|
|
297
|
-
let cur = '';
|
|
298
|
-
let lastBlock = '';
|
|
299
|
-
for (const b of blocks) {
|
|
300
|
-
const tb = b.trim();
|
|
301
|
-
if (!tb || tb === lastBlock) continue;
|
|
302
|
-
lastBlock = tb;
|
|
303
|
-
if (!tb.startsWith('#')) {
|
|
304
|
-
const paraKey = tb.replace(/\s+/g, ' ').toLowerCase();
|
|
305
|
-
if (seenParas.has(paraKey)) continue;
|
|
306
|
-
seenParas.add(paraKey);
|
|
307
|
-
} else {
|
|
308
|
-
const hm = /^(#{1,6})\s+([^\n]+)([\s\S]*)$/.exec(tb);
|
|
309
|
-
const body = (hm?.[3] ?? '').trim();
|
|
310
|
-
if (body) {
|
|
311
|
-
const paraKey = body.replace(/\s+/g, ' ').toLowerCase();
|
|
312
|
-
if (seenParas.has(paraKey)) {
|
|
313
|
-
const name = hm![2].trim();
|
|
314
|
-
if (isBoilerplateHeading(name) || isPromo(name)) continue;
|
|
315
|
-
const depth = hm![1].length;
|
|
316
|
-
if (cur.trim()) sections.push(cur.trim());
|
|
317
|
-
cur = `${'#'.repeat(depth)} ${name}\n`;
|
|
318
|
-
if (outlineNames.length < OUTLINE_CAP) outlineNames.push(name.replace(/\s+/g, ' ').slice(0, 90));
|
|
319
|
-
continue;
|
|
320
|
-
}
|
|
321
|
-
seenParas.add(paraKey);
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
if (titleEcho && normHeading(tb) === titleEcho) continue;
|
|
325
|
-
if (isBoilerplate(b) || isPromo(b)) continue;
|
|
326
|
-
if (isOutlineEcho(b, outlineNames)) continue;
|
|
327
|
-
const hm = /^(#{1,6})\s+([^\n]+)([\s\S]*)$/.exec(b);
|
|
328
|
-
if (hm) {
|
|
329
|
-
const name = hm[2].trim();
|
|
330
|
-
if (isBoilerplateHeading(name) || isPromo(name)) continue;
|
|
331
|
-
if (cur.trim()) sections.push(cur.trim());
|
|
332
|
-
const depth = hm[1].length;
|
|
333
|
-
const rest = (hm[3] ?? '').replace(/^[\n\s]+/, '');
|
|
334
|
-
cur = `${'#'.repeat(depth)} ${name}` + (rest ? `\n${rest}` : '') + '\n';
|
|
335
|
-
if (outlineNames.length < OUTLINE_CAP) outlineNames.push(name.replace(/\s+/g, ' ').slice(0, 90));
|
|
336
|
-
continue;
|
|
337
|
-
}
|
|
338
|
-
cur += scrubNavBullets(b) + '\n\n';
|
|
339
|
-
}
|
|
340
|
-
if (cur.trim()) sections.push(cur.trim());
|
|
341
|
-
const pruned = sections.map((s) => {
|
|
342
|
-
const paras = s.split(/\n\n/);
|
|
343
|
-
const body = paras.filter((p) => {
|
|
344
|
-
const nonHeading = p
|
|
345
|
-
.split('\n')
|
|
346
|
-
.filter((l) => !/^#{1,6} /.test(l))
|
|
347
|
-
.join('\n');
|
|
348
|
-
return !nonHeading.trim() || !isOutlineEcho(nonHeading, outlineNames);
|
|
349
|
-
});
|
|
350
|
-
return body.join('\n\n');
|
|
351
|
-
});
|
|
352
|
-
const capped: string[] = [];
|
|
353
|
-
const continued: number[] = [];
|
|
354
|
-
const outline: string[] = [];
|
|
355
|
-
const lead = sections.length > 0 && !/^#{1,6} /.test(sections[0]);
|
|
356
|
-
for (let i = 0; i < pruned.length; i++) {
|
|
357
|
-
const s = pruned[i].trim();
|
|
358
|
-
const headingOnly = /^#{1,6} [^\n]+$/.test(s);
|
|
359
|
-
if (s.length < SECTION_MIN && !/^#{1,6} /.test(s)) continue;
|
|
360
|
-
if (headingOnly && s.length < 120) continue;
|
|
361
|
-
const hm = /^(#{1,6})\s+([^\n]+)/.exec(s);
|
|
362
|
-
const own = hm ? hm[2].trim() : '(lead)';
|
|
363
|
-
const base = hm ? (outlineNames[i - (lead ? 1 : 0)] ?? own) : '(lead)';
|
|
364
|
-
const name = base.replace(/\s+/g, ' ').slice(0, 90);
|
|
365
|
-
const parts = s.length <= sectionCap ? [s] : splitSection(s, sectionCap);
|
|
366
|
-
const last = parts.length - 1;
|
|
367
|
-
parts.forEach((p, k) => {
|
|
368
|
-
outline.push(k === 0 ? name : `${name} (cont. ${k})`);
|
|
369
|
-
if (k < last) continued.push(capped.length);
|
|
370
|
-
const body =
|
|
371
|
-
parts.length === 1 || k === last
|
|
372
|
-
? p
|
|
373
|
-
: `${p.replace(/\s+$/, '')}\n\n…(continues at outline index ${capped.length + 1})`;
|
|
374
|
-
capped.push(body);
|
|
375
|
-
});
|
|
376
|
-
}
|
|
377
|
-
return { outline, sections: capped, continued };
|
|
378
|
-
}
|
|
379
|
-
|
|
380
56
|
export interface ScrapeOptions {
|
|
381
|
-
sectionCap?: number;
|
|
382
57
|
maxChars?: number;
|
|
383
|
-
render?: 'auto' | 'local'
|
|
58
|
+
render?: 'auto' | 'local';
|
|
384
59
|
allowPrivate?: boolean;
|
|
385
60
|
signal?: AbortSignal;
|
|
386
61
|
validators?: { etag?: string; lastModified?: string };
|
|
387
62
|
}
|
|
388
63
|
|
|
389
|
-
|
|
390
|
-
const NAV_LABEL_MAX = 40;
|
|
391
|
-
const NAV_BLOCK_MAX = 90;
|
|
392
|
-
const OUTLINE_CAP = 20;
|
|
393
|
-
const OUTLINE_LINE_MAX = 90;
|
|
394
|
-
const MIN_OUTLINE_FOR_ECHO = 3;
|
|
395
|
-
const SECTION_MIN = 12;
|
|
396
|
-
|
|
397
|
-
const LOCAL_MIN_TEXT = 240;
|
|
398
|
-
const LOCAL_MIN_OUTPUT = 200;
|
|
399
|
-
const MAX_JOINED_SECTIONS = 12;
|
|
400
|
-
const TRUNCATED_MARKER = '\n…[truncated]';
|
|
401
|
-
|
|
402
|
-
function abortResult(url: URL): ScrapeResult {
|
|
64
|
+
function emptyResult(url: URL, error: string): ScrapeResult {
|
|
403
65
|
return {
|
|
404
66
|
url: url.toString(),
|
|
405
67
|
title: '',
|
|
@@ -409,22 +71,16 @@ function abortResult(url: URL): ScrapeResult {
|
|
|
409
71
|
text: '',
|
|
410
72
|
truncated: false,
|
|
411
73
|
continued: [],
|
|
412
|
-
error
|
|
74
|
+
error,
|
|
413
75
|
};
|
|
414
76
|
}
|
|
415
77
|
|
|
416
|
-
function
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
sections: [],
|
|
423
|
-
text: '',
|
|
424
|
-
truncated: false,
|
|
425
|
-
continued: [],
|
|
426
|
-
error,
|
|
427
|
-
};
|
|
78
|
+
function usableReaderDoc(markdown: string): boolean {
|
|
79
|
+
const md = cleanReaderMarkdown(markdown);
|
|
80
|
+
if (md.replace(/\s+/g, '').length < 100) return false;
|
|
81
|
+
if (notFoundDoc(md)) return false;
|
|
82
|
+
const { sections } = sliceSections(md);
|
|
83
|
+
return sections.length > 0;
|
|
428
84
|
}
|
|
429
85
|
|
|
430
86
|
const GITHUB_REPO_RE = /^\/([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)(?:\/?|\?.*)?$/;
|
|
@@ -445,12 +101,7 @@ function readmeCandidates(url: URL): string[] {
|
|
|
445
101
|
return [];
|
|
446
102
|
}
|
|
447
103
|
|
|
448
|
-
async function scrapeReadme(
|
|
449
|
-
origUrl: URL,
|
|
450
|
-
opts: ScrapeOptions,
|
|
451
|
-
sectionCap: number,
|
|
452
|
-
maxChars: number,
|
|
453
|
-
): Promise<ScrapeResult | undefined> {
|
|
104
|
+
async function scrapeReadme(origUrl: URL, opts: ScrapeOptions, maxChars: number): Promise<ScrapeResult | undefined> {
|
|
454
105
|
const isBlob = GITHUB_BLOB_RE.test(origUrl.pathname);
|
|
455
106
|
const minLen = isBlob ? 1 : 500;
|
|
456
107
|
for (const rawUrl of readmeCandidates(origUrl)) {
|
|
@@ -459,7 +110,7 @@ async function scrapeReadme(
|
|
|
459
110
|
if (!isRawText(contentType) || body.length < minLen) continue;
|
|
460
111
|
const text = stripToAscii(decodeHtmlEntities(body));
|
|
461
112
|
if (text.length < (isBlob ? 1 : LOCAL_MIN_TEXT)) continue;
|
|
462
|
-
const { outline, sections, continued } = sliceSections(text
|
|
113
|
+
const { outline, sections, continued } = sliceSections(text);
|
|
463
114
|
const { text: out, truncated } = joinSections(outline, sections, maxChars);
|
|
464
115
|
if (!out) continue;
|
|
465
116
|
const label = isBlob ? 'source (github.com)' : 'README (github.com)';
|
|
@@ -484,10 +135,9 @@ async function tryLocal(
|
|
|
484
135
|
url: URL,
|
|
485
136
|
render: 'auto' | 'local',
|
|
486
137
|
signal: AbortSignal | undefined,
|
|
487
|
-
sectionCap: number,
|
|
488
138
|
maxChars: number,
|
|
489
139
|
validators?: { etag?: string; lastModified?: string },
|
|
490
|
-
): Promise<
|
|
140
|
+
): Promise<ScrapeResult | 'not-modified' | undefined> {
|
|
491
141
|
const cond = await localFetch(url, signal, validators);
|
|
492
142
|
if (cond.notModified) return 'not-modified';
|
|
493
143
|
const { body, contentType } = cond;
|
|
@@ -504,7 +154,7 @@ async function tryLocal(
|
|
|
504
154
|
publishedAt = extracted.publishedAt;
|
|
505
155
|
paywalled = extracted.paywalled;
|
|
506
156
|
}
|
|
507
|
-
const { outline, sections, continued } = sliceSections(text,
|
|
157
|
+
const { outline, sections, continued } = sliceSections(text, DEFAULT_SECTION_CAP, title);
|
|
508
158
|
const { text: out, truncated } = joinSections(outline, sections, maxChars);
|
|
509
159
|
if (render === 'auto' && text.length < LOCAL_MIN_TEXT) return undefined;
|
|
510
160
|
if (render === 'auto' && (sections.length === 0 || out.length < LOCAL_MIN_OUTPUT || isLowQualityOutput(out))) {
|
|
@@ -525,12 +175,7 @@ async function tryLocal(
|
|
|
525
175
|
};
|
|
526
176
|
}
|
|
527
177
|
|
|
528
|
-
function fromChain(
|
|
529
|
-
url: URL,
|
|
530
|
-
doc: RemoteDoc & { renderer: Renderer },
|
|
531
|
-
sectionCap: number,
|
|
532
|
-
maxChars: number,
|
|
533
|
-
): ScrapeResult {
|
|
178
|
+
function fromChain(url: URL, doc: RemoteDoc & { renderer: Renderer }, maxChars: number): ScrapeResult {
|
|
534
179
|
let markdown = cleanReaderMarkdown(stripDataUriImages(doc.markdown));
|
|
535
180
|
let title = doc.title;
|
|
536
181
|
if (looksLikeHtml(markdown)) {
|
|
@@ -538,24 +183,16 @@ function fromChain(
|
|
|
538
183
|
markdown = extracted.text;
|
|
539
184
|
if (extracted.title) title = extracted.title;
|
|
540
185
|
}
|
|
541
|
-
const { outline, sections, continued } = sliceSections(markdown
|
|
186
|
+
const { outline, sections, continued } = sliceSections(markdown);
|
|
542
187
|
const { text, truncated } = joinSections(outline, sections, maxChars);
|
|
543
188
|
return { url: url.toString(), title, renderer: doc.renderer, outline, sections, continued, text, truncated };
|
|
544
189
|
}
|
|
545
190
|
|
|
546
|
-
const BOILERPLATE_HEADING =
|
|
547
|
-
/^(?:site |main |global |primary )?(?:navigation(?: menu| bar| links)?|menu|sidebar|footer|header|contents?|related topics|quick search|site index|breadcrumbs?|personal tools|namespaces|views|user links|wiki tools|languages|repository navigation|in other(?: languages| projects)|print\/export|navigation menu)$/i;
|
|
548
|
-
|
|
549
|
-
function isBoilerplateHeading(name: string): boolean {
|
|
550
|
-
return BOILERPLATE_HEADING.test(name.trim());
|
|
551
|
-
}
|
|
552
|
-
|
|
553
191
|
// Archive fallback for DEAD pages only (doctrine line): 404/dead-host, never
|
|
554
192
|
// 403-walls. Resurrecting lost content is a service; bypassing walls is not.
|
|
555
193
|
async function archivedFetch(
|
|
556
194
|
url: URL,
|
|
557
195
|
signal: AbortSignal | undefined,
|
|
558
|
-
sectionCap: number,
|
|
559
196
|
maxChars: number,
|
|
560
197
|
): Promise<ScrapeResult | undefined> {
|
|
561
198
|
if (signal?.aborted) return undefined;
|
|
@@ -585,7 +222,7 @@ async function archivedFetch(
|
|
|
585
222
|
if (!/html|text/.test(contentType)) return undefined;
|
|
586
223
|
const extracted = extractHtml(body);
|
|
587
224
|
if (extracted.text.length < LOCAL_MIN_TEXT) return undefined;
|
|
588
|
-
const { outline, sections, continued } = sliceSections(extracted.text,
|
|
225
|
+
const { outline, sections, continued } = sliceSections(extracted.text, DEFAULT_SECTION_CAP, extracted.title);
|
|
589
226
|
const { text: out, truncated } = joinSections(outline, sections, maxChars);
|
|
590
227
|
if (!out) return undefined;
|
|
591
228
|
const snapDate =
|
|
@@ -611,59 +248,51 @@ async function archivedFetch(
|
|
|
611
248
|
export async function scrape(urlRaw: string, opts: ScrapeOptions = {}): Promise<ScrapeResult> {
|
|
612
249
|
const cfg = getConfig();
|
|
613
250
|
const url = validateUrl(urlRaw, opts.allowPrivate ?? cfg.allowPrivate);
|
|
614
|
-
const sectionCap = opts.sectionCap ?? DEFAULT_SECTION_CAP;
|
|
615
251
|
const maxChars = opts.maxChars ?? DEFAULT_MAX_CHARS;
|
|
616
252
|
const render = opts.render ?? 'auto';
|
|
617
253
|
|
|
618
|
-
if (opts.signal?.aborted) return
|
|
254
|
+
if (opts.signal?.aborted) return emptyResult(url, ABORT_ERROR);
|
|
619
255
|
|
|
620
256
|
if (render === 'auto' || render === 'local') {
|
|
621
|
-
const readme = await scrapeReadme(url, opts,
|
|
257
|
+
const readme = await scrapeReadme(url, opts, maxChars);
|
|
622
258
|
if (readme) return readme;
|
|
623
259
|
}
|
|
624
260
|
|
|
625
261
|
if (render === 'local') {
|
|
626
262
|
try {
|
|
627
|
-
const forced = await tryLocal(url, 'local', opts.signal,
|
|
628
|
-
if (forced === 'not-modified')
|
|
263
|
+
const forced = await tryLocal(url, 'local', opts.signal, maxChars, opts.validators);
|
|
264
|
+
if (forced === 'not-modified')
|
|
265
|
+
return { ...emptyResult(url, ABORT_ERROR), error: 'not-modified', renderer: 'local' };
|
|
629
266
|
return forced!;
|
|
630
267
|
} catch (e) {
|
|
631
|
-
return opts.signal?.aborted ?
|
|
268
|
+
return opts.signal?.aborted ? emptyResult(url, ABORT_ERROR) : emptyResult(url, sanitizeError(e));
|
|
632
269
|
}
|
|
633
270
|
}
|
|
634
271
|
|
|
635
|
-
const usable = (markdown: string): boolean => {
|
|
636
|
-
const md = cleanReaderMarkdown(markdown);
|
|
637
|
-
if (md.replace(/\s+/g, '').length < 100) return false;
|
|
638
|
-
if (notFoundDoc(md)) return false;
|
|
639
|
-
const { sections } = sliceSections(md, DEFAULT_SECTION_CAP);
|
|
640
|
-
return sections.length > 0;
|
|
641
|
-
};
|
|
642
|
-
|
|
643
272
|
const isJinaProxy = url.hostname === 'r.jina.ai' || url.hostname === 's.jina.ai';
|
|
644
273
|
|
|
645
274
|
if (render === 'auto') {
|
|
646
|
-
let local:
|
|
275
|
+
let local: ScrapeResult | 'not-modified' | undefined;
|
|
647
276
|
let localErr: string | undefined;
|
|
648
277
|
try {
|
|
649
|
-
local = await tryLocal(url, 'auto', opts.signal,
|
|
278
|
+
local = await tryLocal(url, 'auto', opts.signal, maxChars, opts.validators);
|
|
650
279
|
} catch (e) {
|
|
651
|
-
if (opts.signal?.aborted) return
|
|
280
|
+
if (opts.signal?.aborted) return emptyResult(url, ABORT_ERROR);
|
|
652
281
|
localErr = sanitizeError(e);
|
|
653
282
|
}
|
|
654
283
|
if (local === 'not-modified') {
|
|
655
|
-
return { ...
|
|
284
|
+
return { ...emptyResult(url, ABORT_ERROR), error: 'not-modified', renderer: 'local' };
|
|
656
285
|
}
|
|
657
286
|
if (local) return local;
|
|
658
287
|
if (localErr && /HTTP 404|not a readable page|page too large|empty body/.test(localErr)) {
|
|
659
288
|
if (/HTTP 404/.test(localErr)) {
|
|
660
|
-
const archived = await archivedFetch(url, opts.signal,
|
|
289
|
+
const archived = await archivedFetch(url, opts.signal, maxChars);
|
|
661
290
|
if (archived) return archived;
|
|
662
291
|
}
|
|
663
|
-
return
|
|
292
|
+
return emptyResult(url, localErr);
|
|
664
293
|
}
|
|
665
294
|
if (isJinaProxy) {
|
|
666
|
-
return
|
|
295
|
+
return emptyResult(
|
|
667
296
|
url,
|
|
668
297
|
localErr
|
|
669
298
|
? `${localErr} (jina proxy; target walled or unavailable)`
|
|
@@ -672,63 +301,11 @@ export async function scrape(urlRaw: string, opts: ScrapeOptions = {}): Promise<
|
|
|
672
301
|
}
|
|
673
302
|
}
|
|
674
303
|
|
|
675
|
-
const chainResult = await remoteChain(url, opts.signal,
|
|
304
|
+
const chainResult = await remoteChain(url, opts.signal, usableReaderDoc);
|
|
676
305
|
if (!chainResult.ok) {
|
|
677
|
-
return chainResult.aborted || opts.signal?.aborted
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
}
|
|
681
|
-
|
|
682
|
-
/** Whole-line news junk in reader output: photo credits, carousel markers,
|
|
683
|
-
* video durations, relative-time chips. Prose lines are untouched. */
|
|
684
|
-
const READER_JUNK_LINE_RE = [
|
|
685
|
-
// bare credits: "REUTERS/Sofiia Gatilova", "AP Photo/John Smith"
|
|
686
|
-
/^(?:reuters|ap photo|getty images|afp|epa)\/(?:[a-z0-9][\w.' -]{1,80})$/i,
|
|
687
|
-
// handout/file photo tags at line end: "…REUTERS/Rachel Wisniewski/File Photo"
|
|
688
|
-
/\/(?:file|handout)\s+photo\s*\.*$/i,
|
|
689
|
-
// carousel markers: "Item 1 of 15"
|
|
690
|
-
/^item\s+\d{1,3}\s+of\s+\d{1,3}\s*$/i,
|
|
691
|
-
// video durations: "0:48"
|
|
692
|
-
/^\d{1,3}:\d{2}$/,
|
|
693
|
-
// relative-time chips: "* 1 hour ago", "just now", "yesterday"
|
|
694
|
-
/^[*\-–—]?\s*(?:just now|today|yesterday|\d+\s*(?:m|min(?:ute)?s?|hrs?|h|hours?|d|days?|w|weeks?)\s*ago\.?)\s*$/i,
|
|
695
|
-
];
|
|
696
|
-
|
|
697
|
-
const NOT_FOUND_DOC_RE =
|
|
698
|
-
/(?:page|url|requested page|webpage)[^.]{0,40}(?:not found|not be found|cannot be found|does not exist|doesn't exist|no longer exists|error 404|404 error|404 not found)|we (?:couldn't|could not|can't) find(?: the)? (?:page|requested|article|that)|it looks like there isn't a page/i;
|
|
699
|
-
|
|
700
|
-
/** Best-effort scrub of reader markdown (news sites leak credits/captions). */
|
|
701
|
-
export function cleanReaderMarkdown(markdown: string): string {
|
|
702
|
-
const out: string[] = [];
|
|
703
|
-
let inFence = false;
|
|
704
|
-
for (const line of markdown.split('\n')) {
|
|
705
|
-
if (/^\s*(?:```|~~~)/.test(line)) inFence = !inFence;
|
|
706
|
-
const t = line.trim();
|
|
707
|
-
if (!inFence && t && t.length <= 200 && READER_JUNK_LINE_RE.some((re) => re.test(t))) continue;
|
|
708
|
-
out.push(line);
|
|
306
|
+
return chainResult.aborted || opts.signal?.aborted
|
|
307
|
+
? emptyResult(url, ABORT_ERROR)
|
|
308
|
+
: emptyResult(url, chainResult.error);
|
|
709
309
|
}
|
|
710
|
-
return
|
|
310
|
+
return fromChain(url, chainResult.doc, maxChars);
|
|
711
311
|
}
|
|
712
|
-
|
|
713
|
-
/** True for a 404-style reader stub; the chain skips it and tries the next backend. */
|
|
714
|
-
export function notFoundDoc(markdown: string): boolean {
|
|
715
|
-
if (markdown.replace(/\s+/g, '').length > 500) return false;
|
|
716
|
-
return NOT_FOUND_DOC_RE.test(markdown.slice(0, 300));
|
|
717
|
-
}
|
|
718
|
-
|
|
719
|
-
/** Compose outline + sections; the cap is honest (marker included). */
|
|
720
|
-
export function joinSections(
|
|
721
|
-
outline: string[],
|
|
722
|
-
sections: string[],
|
|
723
|
-
maxChars: number,
|
|
724
|
-
): { text: string; truncated: boolean } {
|
|
725
|
-
const parts: string[] = [];
|
|
726
|
-
if (outline.length) parts.push('Outline: ' + outline.join(' | '));
|
|
727
|
-
for (const s of sections.slice(0, MAX_JOINED_SECTIONS)) parts.push(s);
|
|
728
|
-
let out = parts.join('\n\n').replace(/\n{3,}/g, '\n\n');
|
|
729
|
-
const truncated = out.length > maxChars;
|
|
730
|
-
if (truncated) out = out.slice(0, maxChars - TRUNCATED_MARKER.length) + TRUNCATED_MARKER;
|
|
731
|
-
return { text: out, truncated };
|
|
732
|
-
}
|
|
733
|
-
|
|
734
|
-
/** Deterministic over the cached sections: repeated offset reads avoid re-fetching. */
|