pi-wsearch 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/package.json +3 -3
- package/src/tools.ts +141 -115
- package/src/view.ts +223 -0
package/README.md
CHANGED
|
@@ -39,7 +39,10 @@ and reads only those. No
|
|
|
39
39
|
full reads, no 40 KB dumps. Sections over 3,000 characters split into
|
|
40
40
|
addressable continuation entries (`Section (cont. 1)`), so no tail is ever
|
|
41
41
|
unreachable; pages that overflow are capped; the final response is clipped.
|
|
42
|
-
Every cut is marked — nothing is silently lost.
|
|
42
|
+
Every cut is marked — nothing is silently lost. Codemode consumers get the same
|
|
43
|
+
view via structuredContent: per-section `{index, heading, text}`, continuation
|
|
44
|
+
cuts counted in `sections_truncated`, local budget cuts listed in
|
|
45
|
+
`sections_clipped`.
|
|
43
46
|
|
|
44
47
|
**Reads are free.** Page parsing is built in, nothing billed per read, no key
|
|
45
48
|
required. Real headings become the map, TOC echoes get dropped, so reads come
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-wsearch",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.2",
|
|
4
4
|
"description": "Pi extension: web.search + web.fetch",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
"check": "oxlint && oxfmt --check && tsc --noEmit && knip && bun test --jobs 1 test/",
|
|
37
37
|
"test": "bun test --jobs 1 test/",
|
|
38
38
|
"typecheck": "tsc --noEmit",
|
|
39
|
-
"knip": "knip",
|
|
39
|
+
"knip": "bun scripts/knip.mjs",
|
|
40
40
|
"coverage": "bun test --jobs 1 test/ --coverage"
|
|
41
41
|
},
|
|
42
42
|
"pi": {
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"knip": "^6.39.0",
|
|
58
58
|
"oxfmt": "^0.71.0",
|
|
59
59
|
"oxlint": "^1.86.0",
|
|
60
|
-
"typescript": "^
|
|
60
|
+
"typescript": "^5.9.3"
|
|
61
61
|
},
|
|
62
62
|
"dependencies": {},
|
|
63
63
|
"main": "index.ts",
|
package/src/tools.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import { Type } from 'typebox';
|
|
6
6
|
import type { ExtensionAPI, ExtensionCommandContext, Theme } from '@earendil-works/pi-coding-agent';
|
|
7
7
|
import { keyHint } from '@earendil-works/pi-coding-agent';
|
|
8
|
-
import { Text } from '@earendil-works/pi-tui';
|
|
8
|
+
import { Text, type Component } from '@earendil-works/pi-tui';
|
|
9
9
|
import { appendFileSync, mkdirSync, rmSync, writeFileSync, existsSync } from 'node:fs';
|
|
10
10
|
import { dirname, join } from 'node:path';
|
|
11
11
|
import {
|
|
@@ -20,9 +20,10 @@ import {
|
|
|
20
20
|
} from './config';
|
|
21
21
|
import { ABORT_ERROR, DEFAULT_MAX_RESULTS } from './constants';
|
|
22
22
|
import { cacheKey, openCache, CACHE_TTL_HOURS } from './cache';
|
|
23
|
-
import { clearDeadEngines, runEngines } from './engines';
|
|
24
|
-
import { fuse, diversifyByHost
|
|
25
|
-
import {
|
|
23
|
+
import { clearDeadEngines, runEngines, type EngineOutcome } from './engines';
|
|
24
|
+
import { fuse, diversifyByHost } from './fuse';
|
|
25
|
+
import { normalizeUrl } from './urls';
|
|
26
|
+
import { buildSearchText, READ_FIRST_TAG, wrapAll } from './view';
|
|
26
27
|
import { scrape } from './scrape';
|
|
27
28
|
|
|
28
29
|
const SEARCH_PARAMS = Type.Object({
|
|
@@ -70,19 +71,22 @@ const FETCH_OUTPUT = Type.Object({
|
|
|
70
71
|
title: Type.String(),
|
|
71
72
|
renderer: Type.String(),
|
|
72
73
|
outline: Type.Array(Type.String()),
|
|
74
|
+
sections: Type.Array(
|
|
75
|
+
Type.Object({
|
|
76
|
+
index: Type.Integer({ minimum: 0 }),
|
|
77
|
+
heading: Type.String(),
|
|
78
|
+
text: Type.String(),
|
|
79
|
+
}),
|
|
80
|
+
),
|
|
73
81
|
section_count: Type.Number(),
|
|
74
82
|
chars: Type.Number(),
|
|
75
83
|
sections_requested: Type.Optional(Type.Array(Type.Number())),
|
|
76
84
|
sections_missing: Type.Optional(Type.Array(Type.Number())),
|
|
77
85
|
sections_truncated: Type.Optional(Type.Number()),
|
|
86
|
+
sections_clipped: Type.Optional(Type.Array(Type.Integer({ minimum: 0 }))),
|
|
78
87
|
error: Type.Optional(Type.String()),
|
|
79
88
|
});
|
|
80
89
|
|
|
81
|
-
function fmtSnippet(s: string, max: number): string {
|
|
82
|
-
const t = s.replace(/\s+/g, ' ').trim();
|
|
83
|
-
return t.length > max ? t.slice(0, max) + '…' : t;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
90
|
function dropRarestTerm(query: string): string {
|
|
87
91
|
const words = query.split(/\s+/).filter((w) => w.length > 2 && !/^(?:site|inurl|intitle|filetype):/i.test(w));
|
|
88
92
|
if (words.length < 2) return '';
|
|
@@ -93,54 +97,6 @@ function dropRarestTerm(query: string): string {
|
|
|
93
97
|
.trim();
|
|
94
98
|
}
|
|
95
99
|
|
|
96
|
-
function stripSiteSuffix(title: string, host: string): string {
|
|
97
|
-
const m = /^(.{3,})\s+[|–—-]\s+([^|–—-]{2,40})$/.exec(title.trim());
|
|
98
|
-
if (!m) return title;
|
|
99
|
-
const suffix = m[2].trim();
|
|
100
|
-
const head = m[1].trim();
|
|
101
|
-
const hostBase = host.replace(/^www\./, '').split('.')[0];
|
|
102
|
-
if (host && (suffix.toLowerCase().includes(hostBase) || suffix.length <= 25)) return head;
|
|
103
|
-
return title;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
function normalizeHitDate(raw: string | undefined): string | undefined {
|
|
107
|
-
if (!raw) return undefined;
|
|
108
|
-
const t = Date.parse(raw);
|
|
109
|
-
if (!Number.isNaN(t)) return new Date(t).toISOString().slice(0, 10);
|
|
110
|
-
const rel = /^(\d+)\s*(m|min|minute|minutes|h|hour|hours|d|day|days|w|week|weeks|mo|month|months)\b/i.exec(
|
|
111
|
-
raw.trim(),
|
|
112
|
-
);
|
|
113
|
-
if (rel) {
|
|
114
|
-
const n = parseInt(rel[1], 10);
|
|
115
|
-
const unit = rel[2].toLowerCase();
|
|
116
|
-
// 'mo' must win over 'm' BEFORE the minutes branch: the regex captures
|
|
117
|
-
// 'months' verbatim, so a bare startsWith('m') misclassifies it ("3
|
|
118
|
-
// months ago" used to render as ~today). Divide by unit, in months>…>ms.
|
|
119
|
-
const ms = unit.startsWith('h')
|
|
120
|
-
? 3_600_000
|
|
121
|
-
: unit.startsWith('mo')
|
|
122
|
-
? 2_592_000_000
|
|
123
|
-
: unit.startsWith('m')
|
|
124
|
-
? 60_000
|
|
125
|
-
: unit.startsWith('d')
|
|
126
|
-
? 86_400_000
|
|
127
|
-
: 604_800_000;
|
|
128
|
-
return new Date(Date.now() - n * ms).toISOString().slice(0, 10);
|
|
129
|
-
}
|
|
130
|
-
if (/^yesterday$/i.test(raw.trim())) return new Date(Date.now() - 86_400_000).toISOString().slice(0, 10);
|
|
131
|
-
return undefined;
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
function cleanSnippet(s: string): string {
|
|
135
|
-
return s
|
|
136
|
-
.replace(/\\([_*`[\]()#>~.-])/g, '$1')
|
|
137
|
-
.replace(/!\[[^\]]*\]\([^)]*\)/g, '')
|
|
138
|
-
.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
|
|
139
|
-
.replace(/(^|\s)([*]{1,3}|_{1,2}|`)(\S(?:[^*`_]*\S)?)\2(?=\s|$|[.,;:!?)]|\b)/g, '$1$3')
|
|
140
|
-
.replace(/\s+/g, ' ')
|
|
141
|
-
.trim();
|
|
142
|
-
}
|
|
143
|
-
|
|
144
100
|
export function registerWebTools(pi: ExtensionAPI) {
|
|
145
101
|
const cache = openCache(getConfig().cacheDir + '/cache.json');
|
|
146
102
|
const exposure = existsSync(disabledFlagPath()) ? 'hidden' : 'direct';
|
|
@@ -166,6 +122,13 @@ export function registerWebTools(pi: ExtensionAPI) {
|
|
|
166
122
|
parameters: SEARCH_PARAMS,
|
|
167
123
|
outputSchema: SEARCH_OUTPUT,
|
|
168
124
|
|
|
125
|
+
renderCall(args, theme) {
|
|
126
|
+
const header = theme.fg('toolTitle', theme.bold('web.search'));
|
|
127
|
+
const q = typeof args.query === 'string' ? args.query : '…';
|
|
128
|
+
const tail = typeof args.max_results === 'number' ? theme.fg('muted', ` max=${args.max_results}`) : '';
|
|
129
|
+
return new Text(`${header} ${theme.fg('accent', q)}${tail}`, 0, 0);
|
|
130
|
+
},
|
|
131
|
+
|
|
169
132
|
async execute(_id, params, signal, _onUpdate, _ctx) {
|
|
170
133
|
if (signal?.aborted) throw new Error(ABORT_ERROR);
|
|
171
134
|
const outcomes = await runEngines({
|
|
@@ -205,38 +168,32 @@ export function registerWebTools(pi: ExtensionAPI) {
|
|
|
205
168
|
fused.filter((r) => !r.junk),
|
|
206
169
|
2,
|
|
207
170
|
).slice(0, Math.max((params.max_results ?? DEFAULT_MAX_RESULTS) * 2, DEFAULT_MAX_RESULTS));
|
|
208
|
-
if (candidates.length) retryNote = `
|
|
171
|
+
if (candidates.length) retryNote = `retried as "${relaxed}"`;
|
|
209
172
|
}
|
|
210
173
|
}
|
|
211
174
|
}
|
|
212
175
|
const visible = candidates.slice(0, params.max_results ?? DEFAULT_MAX_RESULTS);
|
|
213
176
|
const enginesUsed = [...new Set(fused.flatMap((r) => r.engines))];
|
|
214
|
-
const snip = 240;
|
|
215
|
-
|
|
216
|
-
let text = `${visible.length} result${visible.length === 1 ? '' : 's'} for "${params.query}": ${enginesUsed.length} engine${enginesUsed.length === 1 ? '' : 's'} contributed (${enginesUsed.join(', ')})${retryNote}\n\n`;
|
|
217
|
-
if (!visible.length) {
|
|
218
|
-
const fatal = outcomes.find((o) => o.engine === 'none' && o.error);
|
|
219
|
-
text = fatal
|
|
220
|
-
? `No results for "${params.query}": ${fatal.error}\n`
|
|
221
|
-
: `No results for "${params.query}". Broaden the query.\n`;
|
|
222
|
-
}
|
|
223
177
|
visible.forEach((r, i) => {
|
|
224
178
|
lastSearch.urlRank.set(normalizeUrl(r.url), {
|
|
225
179
|
rank: i + 1,
|
|
226
180
|
engines: r.engines,
|
|
227
181
|
query: params.query,
|
|
228
182
|
});
|
|
229
|
-
const title = fmtSnippet(stripSiteSuffix(cleanSnippet(r.title), hostOf(r.url)), 100) || r.url;
|
|
230
|
-
const snippet = fmtSnippet(cleanSnippet(r.snippet), snip);
|
|
231
|
-
const via = formatEvidence(r);
|
|
232
|
-
const when = normalizeHitDate(r.date);
|
|
233
|
-
text += `${i + 1}. [${title}](${r.url})${when ? ` — ${when}` : ''}${via}\n`;
|
|
234
|
-
if (snippet) text += ` ${snippet}\n`;
|
|
235
|
-
text += '\n';
|
|
236
183
|
});
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
184
|
+
const text = buildSearchText(
|
|
185
|
+
{
|
|
186
|
+
query: params.query,
|
|
187
|
+
hits: visible,
|
|
188
|
+
enginesUsed,
|
|
189
|
+
errored,
|
|
190
|
+
fatal: visible.length
|
|
191
|
+
? undefined
|
|
192
|
+
: outcomes.find((o): o is EngineOutcome & { error: string } => o.engine === 'none' && !!o.error),
|
|
193
|
+
retryNote,
|
|
194
|
+
},
|
|
195
|
+
{ wrap: terminalWrapWidth() },
|
|
196
|
+
);
|
|
240
197
|
|
|
241
198
|
return {
|
|
242
199
|
content: [{ type: 'text', text }],
|
|
@@ -256,9 +213,7 @@ export function registerWebTools(pi: ExtensionAPI) {
|
|
|
256
213
|
};
|
|
257
214
|
},
|
|
258
215
|
renderResult(result, options, theme) {
|
|
259
|
-
|
|
260
|
-
if (!options.expanded) return new Text(previewWithHint(lines, theme), 0, 0);
|
|
261
|
-
return new Text(colorizedResult(resultText(result), theme), 0, 0);
|
|
216
|
+
return new WrappedSearchText(resultText(result), theme, options.expanded);
|
|
262
217
|
},
|
|
263
218
|
});
|
|
264
219
|
|
|
@@ -280,7 +235,7 @@ export function registerWebTools(pi: ExtensionAPI) {
|
|
|
280
235
|
const url = typeof args.url === 'string' ? args.url : '…';
|
|
281
236
|
const s = Array.isArray(args.sections) ? args.sections.join(', ') : '…';
|
|
282
237
|
const tail = args.sections ? ` sections=[${s}]` : '';
|
|
283
|
-
return new Text(`${header} ${theme.fg('
|
|
238
|
+
return new Text(`${header} ${theme.fg('accent', url)}${theme.fg('muted', tail)}`, 0, 0);
|
|
284
239
|
},
|
|
285
240
|
|
|
286
241
|
async execute(_id, params, signal) {
|
|
@@ -353,11 +308,13 @@ type FetchDetails = {
|
|
|
353
308
|
title: string;
|
|
354
309
|
renderer: string;
|
|
355
310
|
outline: string[];
|
|
311
|
+
sections: { index: number; heading: string; text: string }[];
|
|
356
312
|
section_count: number;
|
|
357
313
|
chars?: number;
|
|
358
314
|
sections_requested?: number[];
|
|
359
315
|
sections_missing?: number[];
|
|
360
316
|
sections_truncated?: number;
|
|
317
|
+
sections_clipped?: number[];
|
|
361
318
|
error?: string;
|
|
362
319
|
};
|
|
363
320
|
|
|
@@ -369,6 +326,7 @@ interface FetchResult {
|
|
|
369
326
|
}
|
|
370
327
|
|
|
371
328
|
const PREVIEW_LINES = 10;
|
|
329
|
+
const SEARCH_PREVIEW_LINES = 8;
|
|
372
330
|
|
|
373
331
|
/** Display-only: the model-facing markers (`indices …`, `- *N*`) render as
|
|
374
332
|
* genuine ANSI italics in the TUI; the content text itself is untouched. */
|
|
@@ -378,49 +336,87 @@ function italicizeIndices(text: string, theme: Theme): string {
|
|
|
378
336
|
.replace(/^- \*(\d+)\* /gm, (_, n: string) => `- ${theme.italic(n)} `);
|
|
379
337
|
}
|
|
380
338
|
|
|
339
|
+
function italicizeRanks(text: string, theme: Theme): string {
|
|
340
|
+
return text.replace(/^(\s*)(\d+)\.(?=\s)/gm, (m, pad: string, n: string) => `${pad}${theme.italic(n)}.`);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
const ANSI_SGR = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g');
|
|
344
|
+
|
|
345
|
+
const DEFAULT_WRAP_COLS = 90;
|
|
346
|
+
const MAX_WRAP_COLS = 240;
|
|
347
|
+
|
|
348
|
+
function terminalWrapWidth(): number {
|
|
349
|
+
const cols = process.stdout.columns ?? DEFAULT_WRAP_COLS;
|
|
350
|
+
return Math.min(MAX_WRAP_COLS, Math.max(60, cols - 4));
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
class WrappedSearchText implements Component {
|
|
354
|
+
private raw: string;
|
|
355
|
+
private theme: Theme;
|
|
356
|
+
private expanded: boolean;
|
|
357
|
+
constructor(raw: string, theme: Theme, expanded: boolean) {
|
|
358
|
+
this.raw = raw;
|
|
359
|
+
this.theme = theme;
|
|
360
|
+
this.expanded = expanded;
|
|
361
|
+
}
|
|
362
|
+
invalidate(): void {}
|
|
363
|
+
render(width: number): string[] {
|
|
364
|
+
const wrapped = wrapAll(this.raw, Math.max(40, width));
|
|
365
|
+
const lines = wrapped.split('\n');
|
|
366
|
+
let shown = lines;
|
|
367
|
+
let hint = '';
|
|
368
|
+
if (!this.expanded && lines.length > SEARCH_PREVIEW_LINES) {
|
|
369
|
+
shown = lines.slice(0, SEARCH_PREVIEW_LINES);
|
|
370
|
+
hint = moreLinesHint(shown.length, lines.length, this.theme);
|
|
371
|
+
}
|
|
372
|
+
const colored = italicizeRanks(shown.join('\n'), this.theme)
|
|
373
|
+
.split('\n')
|
|
374
|
+
.map((l) => colorLine(l, this.theme))
|
|
375
|
+
.join('\n');
|
|
376
|
+
return replaceTabs(colored + hint).split('\n');
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
function colorLine(l: string, theme: Theme): string {
|
|
381
|
+
const plain = l.replace(ANSI_SGR, '');
|
|
382
|
+
if (
|
|
383
|
+
plain.includes('…[section truncated') ||
|
|
384
|
+
plain.includes('…(continues at outline index') ||
|
|
385
|
+
plain.startsWith('…[clipped by tool]') ||
|
|
386
|
+
plain.startsWith('…[truncated]')
|
|
387
|
+
)
|
|
388
|
+
return theme.fg('warning', l);
|
|
389
|
+
if (plain.startsWith('## ') && plain.endsWith(READ_FIRST_TAG))
|
|
390
|
+
return theme.fg('accent', l.slice(0, l.length - READ_FIRST_TAG.length)) + theme.bold(READ_FIRST_TAG);
|
|
391
|
+
if (/^\s*· recommended first read$/.test(plain)) return theme.bold(l.trimEnd());
|
|
392
|
+
if (plain.startsWith('## ')) return theme.fg('accent', l);
|
|
393
|
+
if (plain.startsWith('- ')) return theme.fg('dim', l);
|
|
394
|
+
if (plain.startsWith('[rendered via')) return theme.fg('muted', l);
|
|
395
|
+
if (/^\s*\d+\. /.test(plain)) return theme.fg('dim', l);
|
|
396
|
+
return l;
|
|
397
|
+
}
|
|
398
|
+
|
|
381
399
|
function colorizedResult(text: string, theme: Theme): string {
|
|
382
400
|
return text
|
|
383
401
|
.split('\n')
|
|
384
|
-
.map((l) =>
|
|
385
|
-
if (
|
|
386
|
-
l.includes('…[section truncated') ||
|
|
387
|
-
l.includes('…(continues at outline index') ||
|
|
388
|
-
l.startsWith('…[clipped by tool]') ||
|
|
389
|
-
l.startsWith('…[truncated]')
|
|
390
|
-
)
|
|
391
|
-
return theme.fg('warning', l);
|
|
392
|
-
if (l.startsWith('## [')) return theme.fg('accent', l);
|
|
393
|
-
if (l.startsWith('- ')) return theme.fg('dim', l);
|
|
394
|
-
if (l.startsWith('[rendered via')) return theme.fg('muted', l);
|
|
395
|
-
if (/^\d+\. /.test(l)) return theme.fg('toolOutput', l);
|
|
396
|
-
return l;
|
|
397
|
-
})
|
|
402
|
+
.map((l) => colorLine(l, theme))
|
|
398
403
|
.join('\n');
|
|
399
404
|
}
|
|
400
405
|
|
|
401
|
-
/** Per-hit evidence line: "⇐ serper#1, exa#2"; singles are unverified. */
|
|
402
|
-
function formatEvidence(r: FusedHit): string {
|
|
403
|
-
if (!r.engineRanks?.length) return r.engines.length > 1 ? ` ← ${r.engines.length} engines` : '';
|
|
404
|
-
if (r.engineRanks.length === 1) return ` ⇐ ${r.engineRanks[0].engine} only (single-engine: unverified)`;
|
|
405
|
-
const shown = r.engineRanks
|
|
406
|
-
.slice(0, 6)
|
|
407
|
-
.map((e) => `${e.engine}#${e.rank}`)
|
|
408
|
-
.join(', ');
|
|
409
|
-
const rest = r.engineRanks.length > 6 ? `, +${r.engineRanks.length - 6} more` : '';
|
|
410
|
-
return ` ⇐ ${shown}${rest}`;
|
|
411
|
-
}
|
|
412
|
-
|
|
413
406
|
function resultText(result: { content: { type: string; text?: string }[] }): string {
|
|
414
407
|
return (result.content[0] as { type: string; text: string } | undefined)?.text ?? '';
|
|
415
408
|
}
|
|
416
409
|
|
|
417
|
-
function
|
|
418
|
-
const
|
|
419
|
-
|
|
410
|
+
function moreLinesHint(shown: number, total: number, theme: Theme): string {
|
|
411
|
+
const remaining = total - shown;
|
|
412
|
+
if (remaining <= 0) return '';
|
|
413
|
+
return theme.fg('muted', `\n… (${remaining} more lines, ${keyHint('app.tools.expand', 'to expand')})`);
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
function previewWithHint(lines: string[], theme: Theme, maxLines = PREVIEW_LINES): string {
|
|
417
|
+
const shown = lines.slice(0, maxLines);
|
|
420
418
|
const text = shown.map((l) => theme.fg('toolOutput', replaceTabs(l))).join('\n');
|
|
421
|
-
|
|
422
|
-
return text + theme.fg('muted', `\n… (${remaining} more lines, ${keyHint('app.tools.expand', 'to expand')})`);
|
|
423
|
-
return text;
|
|
419
|
+
return text + moreLinesHint(shown.length, lines.length, theme);
|
|
424
420
|
}
|
|
425
421
|
|
|
426
422
|
function replaceTabs(text: string): string {
|
|
@@ -439,7 +435,6 @@ function fetchErrorResult(details: FetchDetails): FetchResult {
|
|
|
439
435
|
|
|
440
436
|
function buildFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: number[]): FetchResult {
|
|
441
437
|
const want = [...new Set(sections.map((i) => Math.max(0, Math.min(res.sections.length - 1, i))))];
|
|
442
|
-
const chosen = want.map((i) => res.sections[i]).filter(Boolean);
|
|
443
438
|
const body = res.outline.length
|
|
444
439
|
? `indices - 0 - ${res.outline.length - 1} section${res.outline.length === 1 ? '' : 's'}\n${res.outline.map((h, i) => `- *${i}* ${h}`).join('\n')}\n\n`
|
|
445
440
|
: '';
|
|
@@ -471,17 +466,48 @@ function buildFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: num
|
|
|
471
466
|
if (res.renderer && res.renderer !== 'local') text += `\n\n[rendered via ${res.renderer}]`;
|
|
472
467
|
const clipped = text.length > MAX_FETCH_TEXT ? text.slice(0, MAX_FETCH_TEXT) + '\n…[clipped by tool]' : text;
|
|
473
468
|
const missing = sections.filter((i) => i >= res.sections.length);
|
|
469
|
+
// continuation cuts (res.continued) vs local budget clips (clippedSections)
|
|
474
470
|
const truncatedCount = want.filter((idx) => res.continued?.includes(idx)).length;
|
|
471
|
+
|
|
472
|
+
// same budget as the text view; headings and the '…' marker count against it
|
|
473
|
+
let budget = MAX_FETCH_TEXT;
|
|
474
|
+
const structuredSections: { index: number; heading: string; text: string }[] = [];
|
|
475
|
+
const clippedSections: number[] = [];
|
|
476
|
+
for (const idx of want) {
|
|
477
|
+
if (budget <= 0) {
|
|
478
|
+
clippedSections.push(idx);
|
|
479
|
+
continue;
|
|
480
|
+
}
|
|
481
|
+
const s = res.sections[idx];
|
|
482
|
+
if (s === undefined) continue;
|
|
483
|
+
const heading = res.outline[idx] ?? '';
|
|
484
|
+
const fullCost = heading.length + s.length;
|
|
485
|
+
if (fullCost <= budget) {
|
|
486
|
+
structuredSections.push({ index: idx, heading, text: s });
|
|
487
|
+
budget -= fullCost;
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
if (heading.length >= budget) {
|
|
491
|
+
structuredSections.push({ index: idx, heading: heading.slice(0, Math.max(0, budget - 1)) + '…', text: '' });
|
|
492
|
+
} else {
|
|
493
|
+
const textLen = Math.max(0, budget - heading.length - 1);
|
|
494
|
+
structuredSections.push({ index: idx, heading, text: s.slice(0, textLen) + (textLen < s.length ? '…' : '') });
|
|
495
|
+
}
|
|
496
|
+
clippedSections.push(idx);
|
|
497
|
+
budget = 0;
|
|
498
|
+
}
|
|
475
499
|
const details = {
|
|
476
500
|
url: res.url,
|
|
477
501
|
title: res.title,
|
|
478
502
|
renderer: res.renderer,
|
|
479
503
|
outline: res.outline,
|
|
480
|
-
|
|
504
|
+
sections: structuredSections,
|
|
505
|
+
section_count: structuredSections.length,
|
|
481
506
|
chars: clipped.length,
|
|
482
507
|
sections_requested: [...new Set(sections)],
|
|
483
508
|
sections_missing: [...new Set(missing)],
|
|
484
509
|
sections_truncated: truncatedCount,
|
|
510
|
+
sections_clipped: clippedSections,
|
|
485
511
|
...(res.publishedAt ? { publishedAt: res.publishedAt } : {}),
|
|
486
512
|
...(res.paywalled ? { paywalled: true } : {}),
|
|
487
513
|
...(res.error ? { error: res.error } : {}),
|
package/src/view.ts
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import type { FusedHit } from './fuse';
|
|
2
|
+
import { hostOf } from './urls';
|
|
3
|
+
|
|
4
|
+
const SNIPPET_MAX = 240;
|
|
5
|
+
const CHAPTER_TITLE_MAX = 100;
|
|
6
|
+
const TOC_TITLE_MAX = 60;
|
|
7
|
+
const ERROR_SNIP = 40;
|
|
8
|
+
const DEFAULT_WRAP = 100;
|
|
9
|
+
const MAX_WRAP = 240;
|
|
10
|
+
|
|
11
|
+
export const READ_FIRST_TAG = ' · recommended first read';
|
|
12
|
+
|
|
13
|
+
interface ErroredEngine {
|
|
14
|
+
engine: string;
|
|
15
|
+
error: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface SearchViewOpts {
|
|
19
|
+
wrap?: number;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface SearchView {
|
|
23
|
+
query: string;
|
|
24
|
+
hits: FusedHit[];
|
|
25
|
+
enginesUsed: string[];
|
|
26
|
+
errored: ErroredEngine[];
|
|
27
|
+
fatal?: { engine: string; error: string };
|
|
28
|
+
retryNote?: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function fmtSnippet(s: string, max: number): string {
|
|
32
|
+
const t = s.replace(/\s+/g, ' ').trim();
|
|
33
|
+
return t.length > max ? t.slice(0, max) + '…' : t;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function cleanSnippet(s: string): string {
|
|
37
|
+
return s
|
|
38
|
+
.replace(/\\(.)/g, '$1')
|
|
39
|
+
.replace(/!\[[^\]]*\]\([^)]*\)/g, '')
|
|
40
|
+
.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
|
|
41
|
+
.replace(/\[\[\d+\]\]\([^)]*\)\)?/g, '')
|
|
42
|
+
.replace(/\[\[\d+\]\]/g, '')
|
|
43
|
+
.replace(/\(https?:\/\/[^)]*\)/g, ' ')
|
|
44
|
+
.replace(/#{2,6}\s+/g, '')
|
|
45
|
+
.replace(/\|/g, ' ')
|
|
46
|
+
.replace(/(^|\s)-+(?=\s|$)/g, '$1')
|
|
47
|
+
.replace(/(^|[()?!:;])\s?\d{1,2}:\d{2}\s+/g, '$1 ')
|
|
48
|
+
.replace(/\(\s*\)/g, '')
|
|
49
|
+
.replace(/\((?=[0-9+\-–·\s]*[+\-–·])[0-9+\-–·\s]*\)/g, ' ')
|
|
50
|
+
.replace(/\d{1,4}[KM]?\s*•\s*\d{1,4}(?:mo|d|w|y|h)\s+ago/gi, ' ')
|
|
51
|
+
.replace(/\s+([?:!;])/g, '$1')
|
|
52
|
+
.replace(/(^|\s)([*]{1,3}|_{1,2}|`)(\S(?:[^*`_]*\S)?)\2(?=\s|$|[.,;:!?)]|\b)/g, '$1$3')
|
|
53
|
+
.replace(/\s+/g, ' ')
|
|
54
|
+
.trim();
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function stripSiteSuffix(title: string, host: string): string {
|
|
58
|
+
const m = /^(.{3,})\s+[|–—-]\s+([^|–—-]{2,40})$/.exec(title.trim());
|
|
59
|
+
if (!m) return title;
|
|
60
|
+
const suffix = m[2].trim();
|
|
61
|
+
const head = m[1].trim();
|
|
62
|
+
const hostBase = host.replace(/^www\./, '').split('.')[0];
|
|
63
|
+
if (host && suffix.toLowerCase().includes(hostBase)) return head;
|
|
64
|
+
if (suffix.length <= 25 && !/\s/.test(suffix)) return head;
|
|
65
|
+
return title;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function normalizeHitDate(raw: string | undefined): string | undefined {
|
|
69
|
+
if (!raw) return undefined;
|
|
70
|
+
const t = Date.parse(raw);
|
|
71
|
+
if (!Number.isNaN(t)) return new Date(t).toISOString().slice(0, 10);
|
|
72
|
+
const rel = /^(\d+)\s*(m|min|minute|minutes|h|hour|hours|d|day|days|w|week|weeks|mo|month|months)\b/i.exec(
|
|
73
|
+
raw.trim(),
|
|
74
|
+
);
|
|
75
|
+
if (rel) {
|
|
76
|
+
const n = parseInt(rel[1], 10);
|
|
77
|
+
const unit = rel[2].toLowerCase();
|
|
78
|
+
const ms = unit.startsWith('h')
|
|
79
|
+
? 3_600_000
|
|
80
|
+
: unit.startsWith('mo')
|
|
81
|
+
? 2_592_000_000
|
|
82
|
+
: unit.startsWith('m')
|
|
83
|
+
? 60_000
|
|
84
|
+
: unit.startsWith('d')
|
|
85
|
+
? 86_400_000
|
|
86
|
+
: 604_800_000;
|
|
87
|
+
return new Date(Date.now() - n * ms).toISOString().slice(0, 10);
|
|
88
|
+
}
|
|
89
|
+
if (/^yesterday$/i.test(raw.trim())) return new Date(Date.now() - 86_400_000).toISOString().slice(0, 10);
|
|
90
|
+
return undefined;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function formatEvidence(r: FusedHit): string {
|
|
94
|
+
if (!r.engineRanks?.length) return r.engines.length > 1 ? ` ← ${r.engines.length} engines` : '';
|
|
95
|
+
if (r.engineRanks.length === 1) return ` ⇐ ${r.engineRanks[0].engine} only (single-engine: unverified)`;
|
|
96
|
+
const shown = r.engineRanks
|
|
97
|
+
.slice(0, 6)
|
|
98
|
+
.map((e) => `${e.engine}#${e.rank}`)
|
|
99
|
+
.join(', ');
|
|
100
|
+
const rest = r.engineRanks.length > 6 ? `, +${r.engineRanks.length - 6} more` : '';
|
|
101
|
+
return ` ⇐ ${shown}${rest}`;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function cleanTitle(s: string, host: string): string {
|
|
105
|
+
return stripSiteSuffix(cleanSnippet(s), host).replace(/\s+[—–]\s+/g, ': ');
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function tocTitle(r: FusedHit): string {
|
|
109
|
+
return fmtSnippet(cleanTitle(r.title, hostOf(r.url)), TOC_TITLE_MAX) || fmtSnippet(r.url, TOC_TITLE_MAX);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function chapterTitle(r: FusedHit): string {
|
|
113
|
+
return fmtSnippet(cleanTitle(r.title, hostOf(r.url)), CHAPTER_TITLE_MAX) || r.url;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function compactWarn(e: ErroredEngine): string {
|
|
117
|
+
const status = /HTTP\s+(\d{3})/i.exec(e.error)?.[1];
|
|
118
|
+
return status ? `${status}: ${e.engine}` : `${e.engine}: ${e.error.slice(0, ERROR_SNIP)}`;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export function buildSearchText(v: SearchView, opts: SearchViewOpts = {}): string {
|
|
122
|
+
const count = v.hits.length;
|
|
123
|
+
const width = Math.max(40, Math.min(MAX_WRAP, opts.wrap ?? DEFAULT_WRAP));
|
|
124
|
+
const engineWord = v.enginesUsed.length === 1 ? 'engine' : 'engines';
|
|
125
|
+
if (!count) {
|
|
126
|
+
return v.fatal ? `No results: ${v.fatal.error}\n` : `No results. Broaden the query.\n`;
|
|
127
|
+
}
|
|
128
|
+
let text = `${v.enginesUsed.length} ${engineWord} [${v.enginesUsed.join(', ')}]`;
|
|
129
|
+
if (v.retryNote) text += ` · ${v.retryNote}`;
|
|
130
|
+
const warns = v.errored.map(compactWarn);
|
|
131
|
+
if (warns.length) text += ` ⚠ ${warns.join(' · ')}`;
|
|
132
|
+
text += '\n';
|
|
133
|
+
const rows = v.hits.map((r, i) => {
|
|
134
|
+
const host = hostOf(r.url);
|
|
135
|
+
return `${String(i + 1).padStart(2)}. ${tocTitle(r)}${host ? ` · ${host}` : ''}`;
|
|
136
|
+
});
|
|
137
|
+
text += `\n${rows.join('\n')}\n`;
|
|
138
|
+
v.hits.forEach((r, i) => {
|
|
139
|
+
const title = chapterTitle(r);
|
|
140
|
+
const snippet = fmtSnippet(cleanSnippet(r.snippet), SNIPPET_MAX);
|
|
141
|
+
const via = formatEvidence(r).trim();
|
|
142
|
+
const when = normalizeHitDate(r.date);
|
|
143
|
+
let meta = r.url;
|
|
144
|
+
if (when) meta += ` · ${when}`;
|
|
145
|
+
if (via) meta += ` ${via}`;
|
|
146
|
+
text += `\n## ${i + 1}. ${title}${i === 0 && count > 1 ? READ_FIRST_TAG : ''}\n`;
|
|
147
|
+
text += ` ${meta}\n`;
|
|
148
|
+
if (snippet) text += ` ${snippet}\n`;
|
|
149
|
+
});
|
|
150
|
+
return wrapAll(text, width);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export function wrapAll(text: string, width: number): string {
|
|
154
|
+
return text
|
|
155
|
+
.split('\n')
|
|
156
|
+
.map((l) => wrapLine(l, width))
|
|
157
|
+
.join('\n');
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function wrapLine(line: string, width: number): string {
|
|
161
|
+
if (line.length <= width) return line;
|
|
162
|
+
const hasTag = line.endsWith(READ_FIRST_TAG);
|
|
163
|
+
const body = hasTag ? line.slice(0, line.length - READ_FIRST_TAG.length) : line;
|
|
164
|
+
const m = /^(##\s*\d+\.\s+|\s*\d+\.\s+|\s*)/.exec(body);
|
|
165
|
+
const head = m?.[1] ?? '';
|
|
166
|
+
const indent = head.replace(/\S/g, ' ');
|
|
167
|
+
const words = body.slice(head.length).split(' ');
|
|
168
|
+
const limit = width - indent.length;
|
|
169
|
+
const out: string[] = [];
|
|
170
|
+
let cur = '';
|
|
171
|
+
for (const word of words) {
|
|
172
|
+
if (!word) continue;
|
|
173
|
+
const fits = cur ? cur.length + 1 + word.length <= limit : word.length <= limit;
|
|
174
|
+
if (fits) {
|
|
175
|
+
cur = cur ? `${cur} ${word}` : word;
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
if (cur) {
|
|
179
|
+
out.push(cur);
|
|
180
|
+
cur = '';
|
|
181
|
+
}
|
|
182
|
+
if (word.length > limit) {
|
|
183
|
+
for (const chunk of breakToken(word, limit)) {
|
|
184
|
+
if (cur) {
|
|
185
|
+
out.push(cur);
|
|
186
|
+
cur = '';
|
|
187
|
+
}
|
|
188
|
+
cur = chunk;
|
|
189
|
+
}
|
|
190
|
+
} else {
|
|
191
|
+
cur = word;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
if (cur) out.push(cur);
|
|
195
|
+
const lines = out.map((l, i) => (i ? indent + l : head + l));
|
|
196
|
+
if (hasTag) {
|
|
197
|
+
const last = lines.length - 1;
|
|
198
|
+
if (last >= 0 && lines[last].length + READ_FIRST_TAG.length <= width) {
|
|
199
|
+
lines[last] += READ_FIRST_TAG;
|
|
200
|
+
} else {
|
|
201
|
+
lines.push(indent + READ_FIRST_TAG.trimStart());
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
return lines.join('\n');
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function breakToken(token: string, limit: number): string[] {
|
|
208
|
+
const out: string[] = [];
|
|
209
|
+
let from = 0;
|
|
210
|
+
while (token.length - from > limit) {
|
|
211
|
+
let cut = from + limit;
|
|
212
|
+
for (let i = cut - 1; i > from; i--) {
|
|
213
|
+
if (i < token.length && /[/\-_.?=&]/.test(token[i])) {
|
|
214
|
+
cut = i + 1;
|
|
215
|
+
break;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
out.push(token.slice(from, cut));
|
|
219
|
+
from = cut;
|
|
220
|
+
}
|
|
221
|
+
out.push(token.slice(from));
|
|
222
|
+
return out;
|
|
223
|
+
}
|