pi-wsearch 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/scrape.ts CHANGED
@@ -1,10 +1,13 @@
1
1
  /**
2
- * Fetch orchestration + pure text pipeline (transport in transport.ts). SSRF:
3
- * private/loopback numeric literals are rejected on the local path; see
4
- * isPrivateHost for the deliberate scope.
2
+ * Fetch orchestration: URL validation, GitHub README shortcuts, local
3
+ * extraction, archive.org fallback for dead pages, reader chain dispatch.
4
+ * Pure text pipeline (entity decode, extraction, filters, sectioning) lives
5
+ * in text.ts; HTTP + reader backends in transport.ts. SSRF: private/loopback
6
+ * numeric literals are rejected on the local path; see isPrivateHost for the
7
+ * deliberate scope.
5
8
  */
6
9
  import { getConfig, sanitizeError } from './config';
7
- import { ABORT_ERROR } from './constants';
10
+ import { ABORT_ERROR } from './config';
8
11
  import { isPrivateHost, validateUrl } from './urls';
9
12
  import {
10
13
  localFetch,
@@ -14,11 +17,26 @@ import {
14
17
  isLowQualityOutput,
15
18
  stripDataUriImages,
16
19
  withTimeout,
17
- DEFAULT_SECTION_CAP,
18
20
  type RemoteDoc,
19
21
  type Renderer,
20
22
  } from './transport';
21
- import { headScan, treeToMarkdownish } from './dom';
23
+ import {
24
+ cleanReaderMarkdown,
25
+ decodeHtmlEntities,
26
+ DEFAULT_SECTION_CAP,
27
+ extractHtml,
28
+ joinSections,
29
+ notFoundDoc,
30
+ sliceSections,
31
+ stripToAscii,
32
+ } from './text';
33
+
34
+ export { isPrivateHost, validateUrl } from './urls'; // public surface unchanged; the gates live in the URL leaf
35
+ export { cleanReaderMarkdown, decodeHtmlEntities, extractHtml, joinSections, notFoundDoc, sliceSections } from './text';
36
+
37
+ const DEFAULT_MAX_CHARS = 12000;
38
+ const LOCAL_MIN_TEXT = 240;
39
+ const LOCAL_MIN_OUTPUT = 200;
22
40
 
23
41
  export interface ScrapeResult {
24
42
  url: string;
@@ -35,371 +53,15 @@ export interface ScrapeResult {
35
53
  error?: string;
36
54
  }
37
55
 
38
- export { isPrivateHost, validateUrl } from './urls'; // public surface unchanged; the gates live in the URL leaf
39
-
40
- const DEFAULT_MAX_CHARS = 12000;
41
-
42
- function stripToAscii(s: string): string {
43
- return s
44
- .replace(/\u00a0/g, ' ')
45
- .replace(/\u200b/g, '')
46
- .replace(/\r/g, '');
47
- }
48
-
49
- const NAMED_ENTITIES: Record<string, string> = {
50
- amp: '&',
51
- lt: '<',
52
- gt: '>',
53
- quot: '"',
54
- apos: "'",
55
- nbsp: '\u00a0',
56
- ndash: '–',
57
- mdash: '—',
58
- hellip: '…',
59
- copy: '©',
60
- reg: '®',
61
- trade: '™',
62
- laquo: '«',
63
- raquo: '»',
64
- lsquo: '‘',
65
- rsquo: '’',
66
- ldquo: '“',
67
- rdquo: '”',
68
- bull: '•',
69
- middot: '·',
70
- deg: '°',
71
- plusmn: '±',
72
- times: '×',
73
- divide: '÷',
74
- frac12: '½',
75
- sup2: '²',
76
- sup3: '³',
77
- ensp: ' ',
78
- emsp: ' ',
79
- thinsp: ' ',
80
- };
81
-
82
- export function decodeHtmlEntities(s: string): string {
83
- return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z][a-z0-9]*);/gi, (m, body: string) => {
84
- if (body[0] === '#') {
85
- const hex = body[1] === 'x' || body[1] === 'X';
86
- const n = hex ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
87
- if (!Number.isFinite(n) || n <= 0 || n > 0x10ffff || (n >= 0xd800 && n <= 0xdfff)) return m;
88
- try {
89
- return String.fromCodePoint(n);
90
- } catch {
91
- return m;
92
- }
93
- }
94
- return NAMED_ENTITIES[body.toLowerCase()] ?? m;
95
- });
96
- }
97
-
98
- export function extractHtml(html: string): { title: string; text: string; publishedAt?: string; paywalled: boolean } {
99
- const paywalled =
100
- /class="[^"]*(?:fc-ab-root|metering-modal|paywall|pw-overlay|subscription-overlay|regwall)[^"]*"/i.test(html) ||
101
- /id="[^"]*(?:paywall|metering|pay-overlay)[^"]*"/i.test(html) ||
102
- /<meta[^>]+(?:property|name)=["'](?:isAccessibleForFree|tunnel:[^"']*paywall)["'][^>]+content=["'](?:false|yes)["']/i.test(
103
- html,
104
- );
105
- const { title, publishedAt } = headScan(html);
106
- let h = treeToMarkdownish(html).replace(/\uE000/g, '\n');
107
- h = stripToAscii(decodeHtmlEntities(h));
108
- const lines = h
109
- .split('\n')
110
- .map((l) => l.trim())
111
- .filter((l) => l.length > 0 && !/^•\s*$/.test(l))
112
- .filter((l) => !(/^[-*•]\s/.test(l) && isNavBullet(l)))
113
- .filter((l) => !/^#{1,6}\s*$/.test(l))
114
- .map((l) => (/^[-*•]\s{2,}/.test(l) ? l.replace(/^[-*•]\s+/, '').replace(/\s{2,}/g, ' ') : l))
115
- .filter((l) => !(l.length < 120 && isPromo(l)));
116
- const out: string[] = [];
117
- let blank = 0;
118
- for (const l of lines) {
119
- if (l.startsWith('#')) {
120
- if (out.length) out.push('');
121
- out.push(l);
122
- blank = 0;
123
- continue;
124
- }
125
- if (out.length === 0 || /^#{1,6} /.test(out[out.length - 1])) {
126
- out.push(l);
127
- blank = 0;
128
- continue;
129
- }
130
- if (/^[-*•]\s|^\d{1,2}[.)]\s/.test(l) && l.length < 80) {
131
- const prev = out[out.length - 1];
132
- if (prev && /^[-*•]\s|^\d{1,2}[.)]\s/.test(prev)) out.push(l);
133
- else {
134
- out.push('');
135
- out.push(l);
136
- }
137
- blank = 0;
138
- continue;
139
- }
140
- if (l.length < SHORT_LINE_JOIN && blank < 2) {
141
- out[out.length - 1] += ' ' + l;
142
- blank++;
143
- continue;
144
- }
145
- out.push('');
146
- out.push(l);
147
- blank = 0;
148
- }
149
- return { title, publishedAt, paywalled, text: out.join('\n').replace(/\n{3,}/g, '\n\n') };
150
- }
151
-
152
- const NAV_LABEL_RE =
153
- /^(home|menu|about( us)?|docs?|documentation|downloads?|license|licence|support|purchase|buy|pricing|search|login|log ?in|sign ?(in|up|out)|register|account|blog|news|contact|careers?|jobs|store|shop|forum|community|tutorials?|api|guide|guides|reference|examples?|faq|help|terms|privacy|cookies?|sitemap|rss|changelog|releases?|twitter|facebook|github|discord|youtube|linkedin|mastodon|x)$/i;
154
-
155
- function isBoilerplate(block: string): boolean {
156
- const t = block.trim();
157
- const words = t.split(/\s{2,}|\s*[|·>]\s*|\s+/).filter(Boolean);
158
- if (t.length < NAV_BLOCK_MAX && words.length >= 2 && words.every((w) => NAV_LABEL_RE.test(w.trim()))) return true;
159
- const tldr = t.length < 120;
160
- if (tldr) {
161
- if (
162
- /^(skip to|jump to|table of contents|related (articles|posts|links)|share this|back to top|sign (in|up| out)|subscribe to|newsletter|follow (us|@)|menu|navigation|breadcrumbs\b|tags:|categories:|©|all rights reserved|privacy policy|terms of (use|service)|cookie|cookies|accept )/i.test(
163
- t,
164
- )
165
- )
166
- return true;
167
- }
168
- const links = t.match(/\[[^\]]+\]\([^)]+\)/g) ?? [];
169
- if (links.length) {
170
- const rest = t.replace(/\[[^\]]+\]\([^)]+\)/g, '').trim();
171
- if (rest.length === 0) return true;
172
- if (rest.length < 8 && links.length >= 3) return true;
173
- }
174
- return false;
175
- }
176
-
177
- function isNavBullet(line: string): boolean {
178
- const t = line
179
- .replace(/^[-*•]\s*/, '')
180
- .replace(/\s+/g, ' ')
181
- .trim();
182
- if (!t) return true;
183
- if (t.length > NAV_LABEL_MAX) return false;
184
- const words = t.split(/\s{2,}|\s*[|·>]\s*/).filter(Boolean);
185
- if (words.length > 1) return words.every((w) => NAV_LABEL_RE.test(w.trim()));
186
- return NAV_LABEL_RE.test(t);
187
- }
188
-
189
- function scrubNavBullets(text: string): string {
190
- const lines = text.split('\n');
191
- const kept = lines.filter((l) => !/^[-*•]\s/.test(l) || !isNavBullet(l));
192
- return kept.join('\n').replace(/\n{3,}/g, '\n\n');
193
- }
194
-
195
- const normHeading = (s: string): string =>
196
- s
197
- .replace(/^[#\d.\s]+/, '')
198
- .replace(/[^a-z0-9 ]/gi, ' ')
199
- .replace(/\s+/g, ' ')
200
- .trim()
201
- .toLowerCase();
202
-
203
- function isOutlineEcho(block: string, outline: string[]): boolean {
204
- if (outline.length < MIN_OUTLINE_FOR_ECHO) return false;
205
- const trimmed = block.trim();
206
- if (/\btable of contents\b/i.test(trimmed) && trimmed.length < 80) return true;
207
- const heads = outline.map(normHeading).filter(Boolean);
208
- const raw = trimmed
209
- .split('\n')
210
- .flatMap((l) => l.split(/(?=\d+(?:\.\d+)*\.\s)/))
211
- .map((l) => l.replace(/^[-*•]?\s*\d*\.?\s*/, '').trim())
212
- .filter((l) => l.length > 2 && l.length < OUTLINE_LINE_MAX);
213
- if (raw.length < MIN_OUTLINE_FOR_ECHO) return false;
214
- const hit = (l: string): boolean => {
215
- const n = normHeading(l);
216
- return heads.some((h) => h === n || h.startsWith(n + ' ') || n.startsWith(h + ' '));
217
- };
218
- return raw.filter(hit).length / raw.length >= 0.6;
219
- }
220
-
221
- const PROMO_STRONG = [
222
- /become a (?:premium )?member/i,
223
- /unlock exclusive/i,
224
- /go premium/i,
225
- /upgrade to (?:the )?premium/i,
226
- /your membership journey/i,
227
- /stay on the cutting edge/i,
228
- /enter your email below/i,
229
- /by submitting your information/i,
230
- /sign (?:up|you up) (?:for|to) (?:our )?(?:free )?newsletter/i,
231
- /subscribe to (?:our )?newsletter/i,
232
- /get the [a-z][a-z ]{2,40}newsletter/i,
233
- /limited-?time (?:premium )?offer/i,
234
- /choose how you want to join/i,
235
- /explore (?:our )?(?:premium|pro)\b/i,
236
- /^email newsletter\b/i,
237
- /^your \(?[\w\s-]+\)? email$/i,
238
- /^IE \d+ is not supported/i,
239
- /for an optimal experience visit our site/i,
240
- /^skip to (?:content|main)/i,
241
- /^for subscribers$/i,
242
- /^subscriber (?:only|exclusive)$/i,
243
- /^[A-Z][A-Z0-9 &'’-]{5,50}$/,
244
- /\|\s*[A-Z][\w'’&. ]{2,40}\s+\|\s*[A-Z][\w'’&. ]{2,40}\s+News\s*$/,
245
- /^(?:[A-Z][\w'’.\- ]{2,30}\s+\/\s+)?(?:via )?(?:Getty Images(?:\s+file)?|AP(?: Photo)?|Reuters|Xinhua(?:[\s\w/]+)?|AFP|Bloomberg(?: Photo)?|The Associated Press|NBC(?: News)? via|CNN Wire|Sipa USA|AP Photo\/[^,]{2,40})[\s.]*$/i,
246
- /^\d+\s*(?:s|sec|seconds?|m|min|minutes?|h|hr|hours?|d|day|days|w|weeks?)\s+ago[\s.]*$/i,
247
- /^(?:updated|published)\s+(?:just\s+now|\d+\s*\w+\s+ago)$/i,
248
- /^\d+\s+(?:hours?|minutes?)\s+ago$/i,
249
- /get started with free access/i,
250
- /premium (?:newsletter|articles?|membership|features|offer)/i,
251
- /sign[- ]?up (?:was|is) successful/i,
252
- /we use cookies[^.]*(?:to (?:improve|enhance|personalize|analyse|analyze))/i,
253
- /^by continuing (?:to use|to browse)/i,
254
- /^(?:accept|allow) (?:all|selected|necessary)/i,
255
- /^(?:manage|customize|consent) (?:preferences|settings|choices)/i,
256
- /^this (?:site|website) uses (?:cookies|essential cookies)/i,
257
- ];
258
- const PROMO_CTA_RE =
259
- /\b(?:newsletter|membership|club membership|premium member|sign ?up|subscribe|unlock|upgrade|cancel anytime|limited-?time|free trial|join now|join today)\b/gi;
260
-
261
- function isPromo(t: string): boolean {
262
- if (PROMO_STRONG.some((r) => r.test(t))) return true;
263
- return (t.match(PROMO_CTA_RE) ?? []).length >= 2;
264
- }
265
-
266
- function splitSection(s: string, sectionCap: number): string[] {
267
- const hm = /^(#{1,6})\s+([^\n]+)/.exec(s);
268
- const partHead = hm ? `${'#'.repeat(hm[1].length)} ${hm[2].trim()}\n\n` : '';
269
- const floor = Math.min(partHead.length + 1, sectionCap);
270
- const parts: string[] = [];
271
- let rest = s;
272
- for (;;) {
273
- if (rest.length <= sectionCap) {
274
- parts.push(rest);
275
- return parts;
276
- }
277
- const cut = rest.slice(0, sectionCap);
278
- let at = cut.lastIndexOf('\n\n');
279
- if (at < floor) at = cut.lastIndexOf(' ');
280
- if (at < floor) at = sectionCap;
281
- const used = Math.min(at, rest.length - 1);
282
- parts.push(rest.slice(0, used).replace(/\s+$/, ''));
283
- rest = partHead + rest.slice(used).trimStart();
284
- }
285
- }
286
-
287
- export function sliceSections(
288
- text: string,
289
- sectionCap = DEFAULT_SECTION_CAP,
290
- docTitle = '',
291
- ): { outline: string[]; sections: string[]; continued: number[] } {
292
- const titleEcho = docTitle ? normHeading(docTitle) : '';
293
- const blocks = text.replace(/\n{3,}/g, '\n\n').split('\n\n');
294
- const sections: string[] = [];
295
- const outlineNames: string[] = [];
296
- const seenParas = new Set<string>();
297
- let cur = '';
298
- let lastBlock = '';
299
- for (const b of blocks) {
300
- const tb = b.trim();
301
- if (!tb || tb === lastBlock) continue;
302
- lastBlock = tb;
303
- if (!tb.startsWith('#')) {
304
- const paraKey = tb.replace(/\s+/g, ' ').toLowerCase();
305
- if (seenParas.has(paraKey)) continue;
306
- seenParas.add(paraKey);
307
- } else {
308
- const hm = /^(#{1,6})\s+([^\n]+)([\s\S]*)$/.exec(tb);
309
- const body = (hm?.[3] ?? '').trim();
310
- if (body) {
311
- const paraKey = body.replace(/\s+/g, ' ').toLowerCase();
312
- if (seenParas.has(paraKey)) {
313
- const name = hm![2].trim();
314
- if (isBoilerplateHeading(name) || isPromo(name)) continue;
315
- const depth = hm![1].length;
316
- if (cur.trim()) sections.push(cur.trim());
317
- cur = `${'#'.repeat(depth)} ${name}\n`;
318
- if (outlineNames.length < OUTLINE_CAP) outlineNames.push(name.replace(/\s+/g, ' ').slice(0, 90));
319
- continue;
320
- }
321
- seenParas.add(paraKey);
322
- }
323
- }
324
- if (titleEcho && normHeading(tb) === titleEcho) continue;
325
- if (isBoilerplate(b) || isPromo(b)) continue;
326
- if (isOutlineEcho(b, outlineNames)) continue;
327
- const hm = /^(#{1,6})\s+([^\n]+)([\s\S]*)$/.exec(b);
328
- if (hm) {
329
- const name = hm[2].trim();
330
- if (isBoilerplateHeading(name) || isPromo(name)) continue;
331
- if (cur.trim()) sections.push(cur.trim());
332
- const depth = hm[1].length;
333
- const rest = (hm[3] ?? '').replace(/^[\n\s]+/, '');
334
- cur = `${'#'.repeat(depth)} ${name}` + (rest ? `\n${rest}` : '') + '\n';
335
- if (outlineNames.length < OUTLINE_CAP) outlineNames.push(name.replace(/\s+/g, ' ').slice(0, 90));
336
- continue;
337
- }
338
- cur += scrubNavBullets(b) + '\n\n';
339
- }
340
- if (cur.trim()) sections.push(cur.trim());
341
- const pruned = sections.map((s) => {
342
- const paras = s.split(/\n\n/);
343
- const body = paras.filter((p) => {
344
- const nonHeading = p
345
- .split('\n')
346
- .filter((l) => !/^#{1,6} /.test(l))
347
- .join('\n');
348
- return !nonHeading.trim() || !isOutlineEcho(nonHeading, outlineNames);
349
- });
350
- return body.join('\n\n');
351
- });
352
- const capped: string[] = [];
353
- const continued: number[] = [];
354
- const outline: string[] = [];
355
- const lead = sections.length > 0 && !/^#{1,6} /.test(sections[0]);
356
- for (let i = 0; i < pruned.length; i++) {
357
- const s = pruned[i].trim();
358
- const headingOnly = /^#{1,6} [^\n]+$/.test(s);
359
- if (s.length < SECTION_MIN && !/^#{1,6} /.test(s)) continue;
360
- if (headingOnly && s.length < 120) continue;
361
- const hm = /^(#{1,6})\s+([^\n]+)/.exec(s);
362
- const own = hm ? hm[2].trim() : '(lead)';
363
- const base = hm ? (outlineNames[i - (lead ? 1 : 0)] ?? own) : '(lead)';
364
- const name = base.replace(/\s+/g, ' ').slice(0, 90);
365
- const parts = s.length <= sectionCap ? [s] : splitSection(s, sectionCap);
366
- const last = parts.length - 1;
367
- parts.forEach((p, k) => {
368
- outline.push(k === 0 ? name : `${name} (cont. ${k})`);
369
- if (k < last) continued.push(capped.length);
370
- const body =
371
- parts.length === 1 || k === last
372
- ? p
373
- : `${p.replace(/\s+$/, '')}\n\n…(continues at outline index ${capped.length + 1})`;
374
- capped.push(body);
375
- });
376
- }
377
- return { outline, sections: capped, continued };
378
- }
379
-
380
56
  export interface ScrapeOptions {
381
- sectionCap?: number;
382
57
  maxChars?: number;
383
- render?: 'auto' | 'local' | 'firecrawl';
58
+ render?: 'auto' | 'local';
384
59
  allowPrivate?: boolean;
385
60
  signal?: AbortSignal;
386
61
  validators?: { etag?: string; lastModified?: string };
387
62
  }
388
63
 
389
- const SHORT_LINE_JOIN = 28;
390
- const NAV_LABEL_MAX = 40;
391
- const NAV_BLOCK_MAX = 90;
392
- const OUTLINE_CAP = 20;
393
- const OUTLINE_LINE_MAX = 90;
394
- const MIN_OUTLINE_FOR_ECHO = 3;
395
- const SECTION_MIN = 12;
396
-
397
- const LOCAL_MIN_TEXT = 240;
398
- const LOCAL_MIN_OUTPUT = 200;
399
- const MAX_JOINED_SECTIONS = 12;
400
- const TRUNCATED_MARKER = '\n…[truncated]';
401
-
402
- function abortResult(url: URL): ScrapeResult {
64
+ function emptyResult(url: URL, error: string): ScrapeResult {
403
65
  return {
404
66
  url: url.toString(),
405
67
  title: '',
@@ -409,22 +71,16 @@ function abortResult(url: URL): ScrapeResult {
409
71
  text: '',
410
72
  truncated: false,
411
73
  continued: [],
412
- error: ABORT_ERROR,
74
+ error,
413
75
  };
414
76
  }
415
77
 
416
- function errorResult(url: URL, error: string): ScrapeResult {
417
- return {
418
- url: url.toString(),
419
- title: '',
420
- renderer: 'local',
421
- outline: [],
422
- sections: [],
423
- text: '',
424
- truncated: false,
425
- continued: [],
426
- error,
427
- };
78
+ function usableReaderDoc(markdown: string): boolean {
79
+ const md = cleanReaderMarkdown(markdown);
80
+ if (md.replace(/\s+/g, '').length < 100) return false;
81
+ if (notFoundDoc(md)) return false;
82
+ const { sections } = sliceSections(md);
83
+ return sections.length > 0;
428
84
  }
429
85
 
430
86
  const GITHUB_REPO_RE = /^\/([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)(?:\/?|\?.*)?$/;
@@ -445,12 +101,7 @@ function readmeCandidates(url: URL): string[] {
445
101
  return [];
446
102
  }
447
103
 
448
- async function scrapeReadme(
449
- origUrl: URL,
450
- opts: ScrapeOptions,
451
- sectionCap: number,
452
- maxChars: number,
453
- ): Promise<ScrapeResult | undefined> {
104
+ async function scrapeReadme(origUrl: URL, opts: ScrapeOptions, maxChars: number): Promise<ScrapeResult | undefined> {
454
105
  const isBlob = GITHUB_BLOB_RE.test(origUrl.pathname);
455
106
  const minLen = isBlob ? 1 : 500;
456
107
  for (const rawUrl of readmeCandidates(origUrl)) {
@@ -459,7 +110,7 @@ async function scrapeReadme(
459
110
  if (!isRawText(contentType) || body.length < minLen) continue;
460
111
  const text = stripToAscii(decodeHtmlEntities(body));
461
112
  if (text.length < (isBlob ? 1 : LOCAL_MIN_TEXT)) continue;
462
- const { outline, sections, continued } = sliceSections(text, sectionCap);
113
+ const { outline, sections, continued } = sliceSections(text);
463
114
  const { text: out, truncated } = joinSections(outline, sections, maxChars);
464
115
  if (!out) continue;
465
116
  const label = isBlob ? 'source (github.com)' : 'README (github.com)';
@@ -484,10 +135,9 @@ async function tryLocal(
484
135
  url: URL,
485
136
  render: 'auto' | 'local',
486
137
  signal: AbortSignal | undefined,
487
- sectionCap: number,
488
138
  maxChars: number,
489
139
  validators?: { etag?: string; lastModified?: string },
490
- ): Promise<(ScrapeResult & { validators?: { etag?: string; lastModified?: string } }) | undefined | 'not-modified'> {
140
+ ): Promise<ScrapeResult | 'not-modified' | undefined> {
491
141
  const cond = await localFetch(url, signal, validators);
492
142
  if (cond.notModified) return 'not-modified';
493
143
  const { body, contentType } = cond;
@@ -504,7 +154,7 @@ async function tryLocal(
504
154
  publishedAt = extracted.publishedAt;
505
155
  paywalled = extracted.paywalled;
506
156
  }
507
- const { outline, sections, continued } = sliceSections(text, sectionCap, title);
157
+ const { outline, sections, continued } = sliceSections(text, DEFAULT_SECTION_CAP, title);
508
158
  const { text: out, truncated } = joinSections(outline, sections, maxChars);
509
159
  if (render === 'auto' && text.length < LOCAL_MIN_TEXT) return undefined;
510
160
  if (render === 'auto' && (sections.length === 0 || out.length < LOCAL_MIN_OUTPUT || isLowQualityOutput(out))) {
@@ -525,12 +175,7 @@ async function tryLocal(
525
175
  };
526
176
  }
527
177
 
528
- function fromChain(
529
- url: URL,
530
- doc: RemoteDoc & { renderer: Renderer },
531
- sectionCap: number,
532
- maxChars: number,
533
- ): ScrapeResult {
178
+ function fromChain(url: URL, doc: RemoteDoc & { renderer: Renderer }, maxChars: number): ScrapeResult {
534
179
  let markdown = cleanReaderMarkdown(stripDataUriImages(doc.markdown));
535
180
  let title = doc.title;
536
181
  if (looksLikeHtml(markdown)) {
@@ -538,24 +183,16 @@ function fromChain(
538
183
  markdown = extracted.text;
539
184
  if (extracted.title) title = extracted.title;
540
185
  }
541
- const { outline, sections, continued } = sliceSections(markdown, sectionCap);
186
+ const { outline, sections, continued } = sliceSections(markdown);
542
187
  const { text, truncated } = joinSections(outline, sections, maxChars);
543
188
  return { url: url.toString(), title, renderer: doc.renderer, outline, sections, continued, text, truncated };
544
189
  }
545
190
 
546
- const BOILERPLATE_HEADING =
547
- /^(?:site |main |global |primary )?(?:navigation(?: menu| bar| links)?|menu|sidebar|footer|header|contents?|related topics|quick search|site index|breadcrumbs?|personal tools|namespaces|views|user links|wiki tools|languages|repository navigation|in other(?: languages| projects)|print\/export|navigation menu)$/i;
548
-
549
- function isBoilerplateHeading(name: string): boolean {
550
- return BOILERPLATE_HEADING.test(name.trim());
551
- }
552
-
553
191
  // Archive fallback for DEAD pages only (doctrine line): 404/dead-host, never
554
192
  // 403-walls. Resurrecting lost content is a service; bypassing walls is not.
555
193
  async function archivedFetch(
556
194
  url: URL,
557
195
  signal: AbortSignal | undefined,
558
- sectionCap: number,
559
196
  maxChars: number,
560
197
  ): Promise<ScrapeResult | undefined> {
561
198
  if (signal?.aborted) return undefined;
@@ -585,7 +222,7 @@ async function archivedFetch(
585
222
  if (!/html|text/.test(contentType)) return undefined;
586
223
  const extracted = extractHtml(body);
587
224
  if (extracted.text.length < LOCAL_MIN_TEXT) return undefined;
588
- const { outline, sections, continued } = sliceSections(extracted.text, sectionCap, extracted.title);
225
+ const { outline, sections, continued } = sliceSections(extracted.text, DEFAULT_SECTION_CAP, extracted.title);
589
226
  const { text: out, truncated } = joinSections(outline, sections, maxChars);
590
227
  if (!out) return undefined;
591
228
  const snapDate =
@@ -611,59 +248,51 @@ async function archivedFetch(
611
248
  export async function scrape(urlRaw: string, opts: ScrapeOptions = {}): Promise<ScrapeResult> {
612
249
  const cfg = getConfig();
613
250
  const url = validateUrl(urlRaw, opts.allowPrivate ?? cfg.allowPrivate);
614
- const sectionCap = opts.sectionCap ?? DEFAULT_SECTION_CAP;
615
251
  const maxChars = opts.maxChars ?? DEFAULT_MAX_CHARS;
616
252
  const render = opts.render ?? 'auto';
617
253
 
618
- if (opts.signal?.aborted) return abortResult(url);
254
+ if (opts.signal?.aborted) return emptyResult(url, ABORT_ERROR);
619
255
 
620
256
  if (render === 'auto' || render === 'local') {
621
- const readme = await scrapeReadme(url, opts, sectionCap, maxChars);
257
+ const readme = await scrapeReadme(url, opts, maxChars);
622
258
  if (readme) return readme;
623
259
  }
624
260
 
625
261
  if (render === 'local') {
626
262
  try {
627
- const forced = await tryLocal(url, 'local', opts.signal, sectionCap, maxChars, opts.validators);
628
- if (forced === 'not-modified') return { ...abortResult(url), error: 'not-modified', renderer: 'local' };
263
+ const forced = await tryLocal(url, 'local', opts.signal, maxChars, opts.validators);
264
+ if (forced === 'not-modified')
265
+ return { ...emptyResult(url, ABORT_ERROR), error: 'not-modified', renderer: 'local' };
629
266
  return forced!;
630
267
  } catch (e) {
631
- return opts.signal?.aborted ? abortResult(url) : errorResult(url, sanitizeError(e));
268
+ return opts.signal?.aborted ? emptyResult(url, ABORT_ERROR) : emptyResult(url, sanitizeError(e));
632
269
  }
633
270
  }
634
271
 
635
- const usable = (markdown: string): boolean => {
636
- const md = cleanReaderMarkdown(markdown);
637
- if (md.replace(/\s+/g, '').length < 100) return false;
638
- if (notFoundDoc(md)) return false;
639
- const { sections } = sliceSections(md, DEFAULT_SECTION_CAP);
640
- return sections.length > 0;
641
- };
642
-
643
272
  const isJinaProxy = url.hostname === 'r.jina.ai' || url.hostname === 's.jina.ai';
644
273
 
645
274
  if (render === 'auto') {
646
- let local: (ScrapeResult & { validators?: { etag?: string; lastModified?: string } }) | undefined | 'not-modified';
275
+ let local: ScrapeResult | 'not-modified' | undefined;
647
276
  let localErr: string | undefined;
648
277
  try {
649
- local = await tryLocal(url, 'auto', opts.signal, sectionCap, maxChars, opts.validators);
278
+ local = await tryLocal(url, 'auto', opts.signal, maxChars, opts.validators);
650
279
  } catch (e) {
651
- if (opts.signal?.aborted) return abortResult(url);
280
+ if (opts.signal?.aborted) return emptyResult(url, ABORT_ERROR);
652
281
  localErr = sanitizeError(e);
653
282
  }
654
283
  if (local === 'not-modified') {
655
- return { ...abortResult(url), error: 'not-modified', renderer: 'local' };
284
+ return { ...emptyResult(url, ABORT_ERROR), error: 'not-modified', renderer: 'local' };
656
285
  }
657
286
  if (local) return local;
658
287
  if (localErr && /HTTP 404|not a readable page|page too large|empty body/.test(localErr)) {
659
288
  if (/HTTP 404/.test(localErr)) {
660
- const archived = await archivedFetch(url, opts.signal, sectionCap, maxChars);
289
+ const archived = await archivedFetch(url, opts.signal, maxChars);
661
290
  if (archived) return archived;
662
291
  }
663
- return errorResult(url, localErr);
292
+ return emptyResult(url, localErr);
664
293
  }
665
294
  if (isJinaProxy) {
666
- return errorResult(
295
+ return emptyResult(
667
296
  url,
668
297
  localErr
669
298
  ? `${localErr} (jina proxy; target walled or unavailable)`
@@ -672,63 +301,11 @@ export async function scrape(urlRaw: string, opts: ScrapeOptions = {}): Promise<
672
301
  }
673
302
  }
674
303
 
675
- const chainResult = await remoteChain(url, opts.signal, usable);
304
+ const chainResult = await remoteChain(url, opts.signal, usableReaderDoc);
676
305
  if (!chainResult.ok) {
677
- return chainResult.aborted || opts.signal?.aborted ? abortResult(url) : errorResult(url, chainResult.error);
678
- }
679
- return fromChain(url, chainResult.doc, sectionCap, maxChars);
680
- }
681
-
682
- /** Whole-line news junk in reader output: photo credits, carousel markers,
683
- * video durations, relative-time chips. Prose lines are untouched. */
684
- const READER_JUNK_LINE_RE = [
685
- // bare credits: "REUTERS/Sofiia Gatilova", "AP Photo/John Smith"
686
- /^(?:reuters|ap photo|getty images|afp|epa)\/(?:[a-z0-9][\w.' -]{1,80})$/i,
687
- // handout/file photo tags at line end: "…REUTERS/Rachel Wisniewski/File Photo"
688
- /\/(?:file|handout)\s+photo\s*\.*$/i,
689
- // carousel markers: "Item 1 of 15"
690
- /^item\s+\d{1,3}\s+of\s+\d{1,3}\s*$/i,
691
- // video durations: "0:48"
692
- /^\d{1,3}:\d{2}$/,
693
- // relative-time chips: "* 1 hour ago", "just now", "yesterday"
694
- /^[*\-–—]?\s*(?:just now|today|yesterday|\d+\s*(?:m|min(?:ute)?s?|hrs?|h|hours?|d|days?|w|weeks?)\s*ago\.?)\s*$/i,
695
- ];
696
-
697
- const NOT_FOUND_DOC_RE =
698
- /(?:page|url|requested page|webpage)[^.]{0,40}(?:not found|not be found|cannot be found|does not exist|doesn't exist|no longer exists|error 404|404 error|404 not found)|we (?:couldn't|could not|can't) find(?: the)? (?:page|requested|article|that)|it looks like there isn't a page/i;
699
-
700
- /** Best-effort scrub of reader markdown (news sites leak credits/captions). */
701
- export function cleanReaderMarkdown(markdown: string): string {
702
- const out: string[] = [];
703
- let inFence = false;
704
- for (const line of markdown.split('\n')) {
705
- if (/^\s*(?:```|~~~)/.test(line)) inFence = !inFence;
706
- const t = line.trim();
707
- if (!inFence && t && t.length <= 200 && READER_JUNK_LINE_RE.some((re) => re.test(t))) continue;
708
- out.push(line);
306
+ return chainResult.aborted || opts.signal?.aborted
307
+ ? emptyResult(url, ABORT_ERROR)
308
+ : emptyResult(url, chainResult.error);
709
309
  }
710
- return out.join('\n').replace(/\n{3,}/g, '\n\n');
310
+ return fromChain(url, chainResult.doc, maxChars);
711
311
  }
712
-
713
- /** True for a 404-style reader stub; the chain skips it and tries the next backend. */
714
- export function notFoundDoc(markdown: string): boolean {
715
- if (markdown.replace(/\s+/g, '').length > 500) return false;
716
- return NOT_FOUND_DOC_RE.test(markdown.slice(0, 300));
717
- }
718
-
719
- /** Compose outline + sections; the cap is honest (marker included). */
720
- export function joinSections(
721
- outline: string[],
722
- sections: string[],
723
- maxChars: number,
724
- ): { text: string; truncated: boolean } {
725
- const parts: string[] = [];
726
- if (outline.length) parts.push('Outline: ' + outline.join(' | '));
727
- for (const s of sections.slice(0, MAX_JOINED_SECTIONS)) parts.push(s);
728
- let out = parts.join('\n\n').replace(/\n{3,}/g, '\n\n');
729
- const truncated = out.length > maxChars;
730
- if (truncated) out = out.slice(0, maxChars - TRUNCATED_MARKER.length) + TRUNCATED_MARKER;
731
- return { text: out, truncated };
732
- }
733
-
734
- /** Deterministic over the cached sections: repeated offset reads avoid re-fetching. */