pi-unsloth-webtools 0.7.6 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -84
- package/entities.ts +2 -2
- package/html-to-md.ts +24 -9
- package/index.ts +96 -166
- package/lightpanda.ts +276 -0
- package/package.json +13 -5
- package/proxy.ts +2 -1
- package/scripts/install-lightpanda.sh +206 -0
- package/settings.ts +57 -11
- package/tls-fetch.ts +205 -0
- package/web-access.ts +4 -0
- package/web-fetch.ts +175 -51
- package/config-ui.ts +0 -105
- package/config.ts +0 -207
- package/web-render.ts +0 -100
package/web-fetch.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { lookup as dnsLookup } from "node:dns/promises";
|
|
|
2
2
|
import type { LookupAllOptions } from "node:dns";
|
|
3
3
|
import http from "node:http";
|
|
4
4
|
import https from "node:https";
|
|
5
|
+
import { isIP } from "node:net";
|
|
5
6
|
import { open } from "node:fs/promises";
|
|
6
7
|
import type { FileHandle } from "node:fs/promises";
|
|
7
8
|
import { homedir } from "node:os";
|
|
@@ -19,9 +20,11 @@ import {
|
|
|
19
20
|
isPublicIp,
|
|
20
21
|
MAX_SIGNAL_TIMEOUT_MS,
|
|
21
22
|
normalizeUrlScheme,
|
|
23
|
+
stripIpv6Brackets,
|
|
22
24
|
type WebsitePolicy,
|
|
23
25
|
} from "./web-access.ts";
|
|
24
|
-
import {
|
|
26
|
+
import { impersonatedRequest, type TlsHopOptions, type TlsHopResponse } from "./tls-fetch.ts";
|
|
27
|
+
import { collapseWhitespace, decodeHtmlEntities, feedHtml, htmlToMarkdown, visibleChars } from "./html-to-md.ts";
|
|
25
28
|
import type { AttrDict } from "./html-to-md.ts";
|
|
26
29
|
import { INVALID_CHARREFS } from "./entities.ts";
|
|
27
30
|
import { getCached, isFresh, setCached, staleNotice } from "./cache.ts";
|
|
@@ -125,6 +128,14 @@ const HTML_LEADING_RE = new RegExp(
|
|
|
125
128
|
);
|
|
126
129
|
const HTML_DOCUMENT_RE = /^<(?:!doctype\s+html\b|\/?(?:html|head|body)\b)/;
|
|
127
130
|
|
|
131
|
+
const RENDER_THIN_PROSE_CHARS = 400;
|
|
132
|
+
const RENDER_SCRIPT_SHARE = 0.5;
|
|
133
|
+
const RENDER_SCRIPT_MIN_HTML_BYTES = 8192;
|
|
134
|
+
const RENDER_MIN_SCRIPTS = 3;
|
|
135
|
+
const RENDER_NOSCRIPT_MIN_CHARS = 40;
|
|
136
|
+
const RENDER_SPA_MARKERS =
|
|
137
|
+
/(?:id\s*=\s*["']?(?:root|app|__next|__nuxt)["']?|data-reactroot|ng-version|__NEXT_DATA__|__NUXT__|__PRELOADED_STATE__)/i;
|
|
138
|
+
|
|
128
139
|
const MIN_SINGLE_BYTE_ASCII_RATIO = 3 / 4;
|
|
129
140
|
const ASCII_TEXT_BYTES = new Set<number>([
|
|
130
141
|
...Array.from({ length: 0x7f - 0x20 }, (_, i) => i + 0x20),
|
|
@@ -147,8 +158,8 @@ export class FetchTimeoutError extends Error {
|
|
|
147
158
|
}
|
|
148
159
|
const FETCH_CANCELLED_MESSAGE = "Failed to fetch URL: cancelled.";
|
|
149
160
|
const FETCH_TIMEOUT_MESSAGE = "Failed to fetch URL: timed out.";
|
|
150
|
-
const TRUNCATED_BODY_SUFFIX = "... (page truncated at the download limit)";
|
|
151
|
-
const TRUNCATED_BODY_NOTICE = "\n\n" + TRUNCATED_BODY_SUFFIX;
|
|
161
|
+
export const TRUNCATED_BODY_SUFFIX = "... (page truncated at the download limit)";
|
|
162
|
+
export const TRUNCATED_BODY_NOTICE = "\n\n" + TRUNCATED_BODY_SUFFIX;
|
|
152
163
|
|
|
153
164
|
function fetchErrorMessage(err: unknown): string {
|
|
154
165
|
if (err instanceof FetchCancelledError) return FETCH_CANCELLED_MESSAGE;
|
|
@@ -178,6 +189,7 @@ export interface FetchPageOptions {
|
|
|
178
189
|
maxBytes?: number;
|
|
179
190
|
maxPdfBytes?: number;
|
|
180
191
|
allowPrivateAddresses?: boolean;
|
|
192
|
+
transport?: FetchTransport;
|
|
181
193
|
allowLocalFiles?: boolean;
|
|
182
194
|
seams?: FetchSeams;
|
|
183
195
|
rawFetch?: (url: string, options: RawFetchOptions) => Promise<RawFetchResult>;
|
|
@@ -198,9 +210,13 @@ export interface ResolvedHost {
|
|
|
198
210
|
alternates?: { ip: string; family: number }[];
|
|
199
211
|
}
|
|
200
212
|
|
|
213
|
+
export type FetchTransport = "tls-first" | "direct-first" | "off";
|
|
214
|
+
|
|
215
|
+
type TransportKind = "direct" | "tls";
|
|
201
216
|
export interface FetchSeams {
|
|
202
217
|
resolve?: (hostname: string, signal?: AbortSignal, allowPrivateAddresses?: boolean) => Promise<ResolvedHost>;
|
|
203
218
|
request?: (opts: HopOptions) => Promise<HopResponse>;
|
|
219
|
+
impersonate?: (options: TlsHopOptions) => Promise<TlsHopResponse | null>;
|
|
204
220
|
}
|
|
205
221
|
|
|
206
222
|
export interface HopOptions {
|
|
@@ -226,6 +242,7 @@ export interface RawFetchOptions {
|
|
|
226
242
|
maxBytes?: number;
|
|
227
243
|
maxPdfBytes?: number;
|
|
228
244
|
allowPrivateAddresses?: boolean;
|
|
245
|
+
transport?: FetchTransport;
|
|
229
246
|
seams?: FetchSeams;
|
|
230
247
|
}
|
|
231
248
|
|
|
@@ -235,6 +252,16 @@ export interface RawFetchResult {
|
|
|
235
252
|
contentType: string;
|
|
236
253
|
}
|
|
237
254
|
|
|
255
|
+
export interface RenderHint {
|
|
256
|
+
reason: "empty" | "js-shell";
|
|
257
|
+
evidence: string[];
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
export interface FetchPageOutcome {
|
|
261
|
+
text: string;
|
|
262
|
+
hint: RenderHint | null;
|
|
263
|
+
}
|
|
264
|
+
|
|
238
265
|
function htmlProbe(body: string, re: RegExp): boolean {
|
|
239
266
|
let i = 0;
|
|
240
267
|
const n = body.length;
|
|
@@ -265,6 +292,46 @@ export function looksLikeHtmlDocument(body: string): boolean {
|
|
|
265
292
|
return htmlProbe(body, HTML_DOCUMENT_RE);
|
|
266
293
|
}
|
|
267
294
|
|
|
295
|
+
function noscriptText(html: string): string {
|
|
296
|
+
const parts: string[] = [];
|
|
297
|
+
for (const match of html.match(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi) ?? []) {
|
|
298
|
+
parts.push(match.replace(/<[^>]*>/g, " "));
|
|
299
|
+
}
|
|
300
|
+
return collapseWhitespace(decodeHtmlEntities(parts.join(" ")));
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
export function needsRenderHint(html: string, converted: string): RenderHint | null {
|
|
304
|
+
const prose = visibleChars(converted);
|
|
305
|
+
if (prose >= RENDER_THIN_PROSE_CHARS) return null;
|
|
306
|
+
if (prose === 0) return { reason: "empty", evidence: [] };
|
|
307
|
+
const evidence: string[] = [];
|
|
308
|
+
let score = 0;
|
|
309
|
+
if (RENDER_SPA_MARKERS.test(html)) {
|
|
310
|
+
evidence.push("spa-markers");
|
|
311
|
+
score += 2;
|
|
312
|
+
}
|
|
313
|
+
const scriptCount = (html.match(/<script\b/gi) ?? []).length;
|
|
314
|
+
let scriptBytes = 0;
|
|
315
|
+
for (const script of html.match(/<script\b[^>]*>[\s\S]*?<\/script>/gi) ?? []) scriptBytes += script.length;
|
|
316
|
+
if (html.length >= RENDER_SCRIPT_MIN_HTML_BYTES && scriptBytes >= html.length * RENDER_SCRIPT_SHARE) {
|
|
317
|
+
evidence.push("script-heavy");
|
|
318
|
+
score += 2;
|
|
319
|
+
}
|
|
320
|
+
if (scriptCount >= RENDER_MIN_SCRIPTS) {
|
|
321
|
+
evidence.push("scripts");
|
|
322
|
+
score += 1;
|
|
323
|
+
}
|
|
324
|
+
if (noscriptText(html).length >= RENDER_NOSCRIPT_MIN_CHARS) {
|
|
325
|
+
evidence.push("noscript");
|
|
326
|
+
score += 1;
|
|
327
|
+
}
|
|
328
|
+
if (extractPageMeta(html).description) {
|
|
329
|
+
evidence.push("description");
|
|
330
|
+
score += 1;
|
|
331
|
+
}
|
|
332
|
+
return score >= 2 ? { reason: "js-shell", evidence } : null;
|
|
333
|
+
}
|
|
334
|
+
|
|
268
335
|
function parseContentType(value: string | null | undefined): string {
|
|
269
336
|
return /^[\w.+-]+\/[\w.+-]+/.exec(value ?? "")?.[0] ?? "";
|
|
270
337
|
}
|
|
@@ -320,7 +387,7 @@ function hasSingleByteTextEvidence(data: Buffer): boolean {
|
|
|
320
387
|
return ascii / data.length >= MIN_SINGLE_BYTE_ASCII_RATIO;
|
|
321
388
|
}
|
|
322
389
|
|
|
323
|
-
const CHARSET_ALIASES: Record<string, string> = {
|
|
390
|
+
const CHARSET_ALIASES: Record<string, string> = Object.assign(Object.create(null), {
|
|
324
391
|
gbk: "gbk",
|
|
325
392
|
gb2312: "gbk",
|
|
326
393
|
"gb-2312": "gbk",
|
|
@@ -346,7 +413,7 @@ const CHARSET_ALIASES: Record<string, string> = {
|
|
|
346
413
|
"windows-874": "windows-874",
|
|
347
414
|
cp874: "windows-874",
|
|
348
415
|
"tis-620": "tis-620",
|
|
349
|
-
};
|
|
416
|
+
});
|
|
350
417
|
|
|
351
418
|
function normalizeCharset(name: string): string | null {
|
|
352
419
|
const n = name.trim().replace(/["']/g, "").toLowerCase();
|
|
@@ -637,9 +704,10 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
|
|
|
637
704
|
const url = opts.url;
|
|
638
705
|
const transport = url.protocol === "https:" ? https : http;
|
|
639
706
|
const port = url.port ? Number(url.port) : url.protocol === "https:" ? 443 : 80;
|
|
707
|
+
const hostname = stripIpv6Brackets(url.hostname);
|
|
640
708
|
const options: https.RequestOptions = {
|
|
641
709
|
method: "GET",
|
|
642
|
-
host:
|
|
710
|
+
host: hostname,
|
|
643
711
|
port,
|
|
644
712
|
path: url.pathname + url.search,
|
|
645
713
|
headers: opts.headers,
|
|
@@ -652,12 +720,12 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
|
|
|
652
720
|
ip: opts.pinnedIp,
|
|
653
721
|
family: opts.family,
|
|
654
722
|
port,
|
|
655
|
-
servername:
|
|
723
|
+
servername: hostname,
|
|
656
724
|
timeoutMs: Math.max(1, opts.inactivityMs),
|
|
657
725
|
signal: opts.signal,
|
|
658
726
|
});
|
|
659
727
|
} else {
|
|
660
|
-
options.servername = url.protocol === "https:" ?
|
|
728
|
+
options.servername = url.protocol === "https:" && !isIP(hostname) ? hostname : undefined;
|
|
661
729
|
options.lookup = ((_hostname: string, _options: unknown, _callback: unknown) => {
|
|
662
730
|
const done = (typeof _options === "function" ? _options : _callback) as (err: unknown, address: unknown, family?: unknown) => void;
|
|
663
731
|
if ((_options as { all?: boolean } | undefined)?.all) {
|
|
@@ -842,6 +910,10 @@ export async function fetchUrlRaw(
|
|
|
842
910
|
const resolveHost = seams.resolve ?? resolveAndValidateHost;
|
|
843
911
|
const allowPrivateAddresses = options.allowPrivateAddresses ?? true;
|
|
844
912
|
const performRequest = seams.request ?? requestHop;
|
|
913
|
+
const transportMode = options.transport ?? "tls-first";
|
|
914
|
+
const performImpersonation =
|
|
915
|
+
transportMode === "off" ? null : (seams.impersonate ?? (seams.request ? null : impersonatedRequest));
|
|
916
|
+
const directFirst = transportMode === "direct-first" || performImpersonation === null;
|
|
845
917
|
const resolveWithBudget = async (hostname: string): Promise<ResolvedHost> => {
|
|
846
918
|
const abortController = new AbortController();
|
|
847
919
|
const waitMs = clampedRemainingMs(deadline, now);
|
|
@@ -876,14 +948,16 @@ export async function fetchUrlRaw(
|
|
|
876
948
|
let pinnedFamily = resolved.family;
|
|
877
949
|
let alternates: { ip: string; family: number }[] = resolved.alternates ?? [];
|
|
878
950
|
let alternateIndex = 0;
|
|
951
|
+
const triedTransports = new Set<string>();
|
|
879
952
|
const userAgent = randomUserAgent();
|
|
880
953
|
for (let hop = 0; hop < MAX_REQUESTS; hop++) {
|
|
881
954
|
const budgetResult = checkBudget();
|
|
882
955
|
if (budgetResult !== null) return budgetResult;
|
|
883
956
|
const parsed = new URL(currentUrl);
|
|
884
|
-
const
|
|
885
|
-
|
|
886
|
-
|
|
957
|
+
const parsedHostname = stripIpv6Brackets(parsed.hostname);
|
|
958
|
+
const hostHeader = parsedHostname.includes(":")
|
|
959
|
+
? `[${parsedHostname}]${parsed.port ? `:${parsed.port}` : ""}`
|
|
960
|
+
: parsedHostname + (parsed.port ? `:${parsed.port}` : "");
|
|
887
961
|
const headers: Record<string, string> = {
|
|
888
962
|
"User-Agent": userAgent,
|
|
889
963
|
Host: hostHeader,
|
|
@@ -898,34 +972,67 @@ export async function fetchUrlRaw(
|
|
|
898
972
|
}
|
|
899
973
|
headers["Host"] = hostHeader;
|
|
900
974
|
const inactivity = remainingMs(deadline, now);
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
975
|
+
const order: TransportKind[] = socksProxyForUrl(parsed) !== null
|
|
976
|
+
? ["direct"]
|
|
977
|
+
: directFirst
|
|
978
|
+
? ["direct", "tls"]
|
|
979
|
+
: ["tls", "direct"];
|
|
980
|
+
let response: HopResponse | null = null;
|
|
981
|
+
let failure: unknown = null;
|
|
982
|
+
for (const kind of order) {
|
|
983
|
+
const key = `${kind}|${currentUrl}|${pinnedIp}`;
|
|
984
|
+
if (triedTransports.has(key)) continue;
|
|
985
|
+
triedTransports.add(key);
|
|
986
|
+
let attempt: HopResponse | null;
|
|
987
|
+
try {
|
|
988
|
+
if (kind === "tls") {
|
|
989
|
+
const impersonate = performImpersonation;
|
|
990
|
+
if (impersonate === null) continue;
|
|
991
|
+
attempt = await impersonate({
|
|
992
|
+
url: parsed,
|
|
993
|
+
pinnedIp,
|
|
994
|
+
family: pinnedFamily,
|
|
995
|
+
timeoutMs: remainingMs(deadline, now),
|
|
996
|
+
signal,
|
|
997
|
+
maxBytes,
|
|
998
|
+
maxPdfBytes,
|
|
999
|
+
extraHeaders: options.extraHeaders,
|
|
1000
|
+
});
|
|
1001
|
+
} else {
|
|
1002
|
+
attempt = await performRequest({
|
|
1003
|
+
url: parsed,
|
|
1004
|
+
pinnedIp,
|
|
1005
|
+
family: pinnedFamily,
|
|
1006
|
+
headers,
|
|
1007
|
+
maxBytes,
|
|
1008
|
+
maxPdfBytes,
|
|
1009
|
+
inactivityMs: inactivity,
|
|
1010
|
+
deadlineMs: deadline,
|
|
1011
|
+
nowMs: now,
|
|
1012
|
+
signal,
|
|
1013
|
+
});
|
|
1014
|
+
}
|
|
1015
|
+
} catch (err) {
|
|
1016
|
+
const message = err instanceof Error ? err.message : "";
|
|
1017
|
+
if (message === "cancelled" || message === "timed out") return emptyResult(fetchErrorMessage(err));
|
|
1018
|
+
if (err instanceof FetchCancelledError || err instanceof FetchTimeoutError) {
|
|
1019
|
+
return emptyResult(fetchErrorMessage(err));
|
|
1020
|
+
}
|
|
1021
|
+
failure = err;
|
|
1022
|
+
continue;
|
|
1023
|
+
}
|
|
1024
|
+
if (attempt === null) continue;
|
|
1025
|
+
if (response === null || (attempt.status < 400 && response.status >= 400)) response = attempt;
|
|
1026
|
+
if (attempt.status < 400) break;
|
|
1027
|
+
}
|
|
1028
|
+
if (response === null) {
|
|
1029
|
+
if (alternateIndex < alternates.length) {
|
|
923
1030
|
const next = alternates[alternateIndex++];
|
|
924
1031
|
pinnedIp = next.ip;
|
|
925
1032
|
pinnedFamily = next.family;
|
|
926
1033
|
continue;
|
|
927
1034
|
}
|
|
928
|
-
return emptyResult(fetchErrorMessage(
|
|
1035
|
+
return emptyResult(fetchErrorMessage(failure ?? new Error("no transport could reach the target")));
|
|
929
1036
|
}
|
|
930
1037
|
|
|
931
1038
|
if (response.status >= 300 && response.status < 400) {
|
|
@@ -1133,9 +1240,10 @@ interface PageMeta {
|
|
|
1133
1240
|
author: string;
|
|
1134
1241
|
date: string;
|
|
1135
1242
|
site: string;
|
|
1243
|
+
description: string;
|
|
1136
1244
|
}
|
|
1137
1245
|
|
|
1138
|
-
const META_KEYS: Record<string, keyof PageMeta> = {
|
|
1246
|
+
const META_KEYS: Record<string, keyof PageMeta> = Object.assign(Object.create(null), {
|
|
1139
1247
|
author: "author",
|
|
1140
1248
|
"article:author": "author",
|
|
1141
1249
|
"dc.creator": "author",
|
|
@@ -1145,7 +1253,9 @@ const META_KEYS: Record<string, keyof PageMeta> = {
|
|
|
1145
1253
|
datepublished: "date",
|
|
1146
1254
|
"og:site_name": "site",
|
|
1147
1255
|
"application-name": "site",
|
|
1148
|
-
|
|
1256
|
+
description: "description",
|
|
1257
|
+
"og:description": "description",
|
|
1258
|
+
});
|
|
1149
1259
|
|
|
1150
1260
|
function cutAtCharBoundary(text: string, maxChars: number): string {
|
|
1151
1261
|
const sliced = text.slice(0, maxChars);
|
|
@@ -1165,7 +1275,7 @@ function capMetaValue(value: string): string {
|
|
|
1165
1275
|
}
|
|
1166
1276
|
|
|
1167
1277
|
function extractPageMeta(html: string): PageMeta {
|
|
1168
|
-
const meta: PageMeta = { title: extractPageTitle(html), author: "", date: "", site: "" };
|
|
1278
|
+
const meta: PageMeta = { title: extractPageTitle(html), author: "", date: "", site: "", description: "" };
|
|
1169
1279
|
const seen = new Set<string>();
|
|
1170
1280
|
const record = (name: string, attrs: AttrDict) => {
|
|
1171
1281
|
if (name !== "meta") return;
|
|
@@ -1191,7 +1301,7 @@ function extractPageMeta(html: string): PageMeta {
|
|
|
1191
1301
|
return meta;
|
|
1192
1302
|
}
|
|
1193
1303
|
|
|
1194
|
-
function pagePrefixedMarkdown(html: string): string {
|
|
1304
|
+
export function pagePrefixedMarkdown(html: string): string {
|
|
1195
1305
|
const meta = extractPageMeta(html);
|
|
1196
1306
|
const lines: string[] = [];
|
|
1197
1307
|
if (meta.title) lines.push(`Title: ${meta.title}`);
|
|
@@ -1291,6 +1401,13 @@ function localFileFailure(err: unknown): string {
|
|
|
1291
1401
|
return `Failed to read file: ${err instanceof Error ? err.message : String(err)}`;
|
|
1292
1402
|
}
|
|
1293
1403
|
|
|
1404
|
+
const HTML_FILE_EXTENSIONS = new Set([".htm", ".html", ".xht", ".xhtml"]);
|
|
1405
|
+
|
|
1406
|
+
function localFileContentType(filePath: string): string {
|
|
1407
|
+
const dot = filePath.lastIndexOf(".");
|
|
1408
|
+
return dot !== -1 && HTML_FILE_EXTENSIONS.has(filePath.slice(dot).toLowerCase()) ? "text/html" : "";
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1294
1411
|
async function readLocalFile(
|
|
1295
1412
|
filePath: string,
|
|
1296
1413
|
options: { signal?: AbortSignal; maxChars?: number; maxBytes?: number; maxPdfBytes?: number },
|
|
@@ -1333,16 +1450,16 @@ async function readLocalFile(
|
|
|
1333
1450
|
}
|
|
1334
1451
|
const text = decodeWithCodec(body, bomCodecFor(body) ?? "utf-8");
|
|
1335
1452
|
if (looksBinary(text)) return `(binary content, ${body.length} bytes; not readable as text)`;
|
|
1336
|
-
return truncatePageText(withTruncation(renderBody(text,
|
|
1453
|
+
return truncatePageText(withTruncation(renderBody(text, localFileContentType(filePath)), truncated), options.maxChars);
|
|
1337
1454
|
} finally {
|
|
1338
1455
|
await handle.close().catch(() => {});
|
|
1339
1456
|
}
|
|
1340
1457
|
}
|
|
1341
1458
|
|
|
1342
|
-
export async function
|
|
1459
|
+
export async function fetchPageOutcome(
|
|
1343
1460
|
url: string,
|
|
1344
1461
|
options: FetchPageOptions = {},
|
|
1345
|
-
): Promise<
|
|
1462
|
+
): Promise<FetchPageOutcome> {
|
|
1346
1463
|
const timeoutMs = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
|
|
1347
1464
|
const now = options.nowMs ?? Date.now;
|
|
1348
1465
|
const deadlineMs = options.deadlineMs ?? now() + timeoutMs;
|
|
@@ -1353,18 +1470,19 @@ export async function fetchPageText(
|
|
|
1353
1470
|
if (options.allowLocalFiles !== false) {
|
|
1354
1471
|
const localPath = parseLocalPath(url);
|
|
1355
1472
|
if (localPath !== null) {
|
|
1356
|
-
if (!localPath) return "Failed to read file: invalid file URL.";
|
|
1357
|
-
|
|
1473
|
+
if (!localPath) return { text: "Failed to read file: invalid file URL.", hint: null };
|
|
1474
|
+
const localText = await readLocalFile(localPath, {
|
|
1358
1475
|
signal,
|
|
1359
1476
|
maxChars,
|
|
1360
1477
|
maxBytes: options.maxBytes,
|
|
1361
1478
|
maxPdfBytes: options.maxPdfBytes,
|
|
1362
1479
|
});
|
|
1480
|
+
return { text: localText, hint: null };
|
|
1363
1481
|
}
|
|
1364
1482
|
}
|
|
1365
1483
|
url = normalizeUrlScheme(url);
|
|
1366
1484
|
const [allowed, reason] = checkUrlAccess(url, policy);
|
|
1367
|
-
if (!allowed) return reason;
|
|
1485
|
+
if (!allowed) return { text: reason, hint: null };
|
|
1368
1486
|
const rawFetchOptions = {
|
|
1369
1487
|
deadlineMs,
|
|
1370
1488
|
signal,
|
|
@@ -1373,6 +1491,7 @@ export async function fetchPageText(
|
|
|
1373
1491
|
maxBytes: options.maxBytes,
|
|
1374
1492
|
maxPdfBytes: options.maxPdfBytes,
|
|
1375
1493
|
seams: options.seams,
|
|
1494
|
+
transport: options.transport,
|
|
1376
1495
|
};
|
|
1377
1496
|
const useCache = !options.seams && !options.rawFetch;
|
|
1378
1497
|
const githubRaw = githubRawContentUrl(url);
|
|
@@ -1381,7 +1500,7 @@ export async function fetchPageText(
|
|
|
1381
1500
|
if (rawResult.error === null) {
|
|
1382
1501
|
const out = renderBody(rawResult.body, rawResult.contentType);
|
|
1383
1502
|
await persistCache(url, rawResult.body, rawResult.contentType, useCache);
|
|
1384
|
-
return truncatePageText(out, maxChars);
|
|
1503
|
+
return { text: truncatePageText(out, maxChars), hint: null };
|
|
1385
1504
|
}
|
|
1386
1505
|
}
|
|
1387
1506
|
const readmeApiUrl = githubRepoReadmeApiUrl(url);
|
|
@@ -1397,7 +1516,7 @@ export async function fetchPageText(
|
|
|
1397
1516
|
if (apiBody.trim()) {
|
|
1398
1517
|
const rendered = `README of ${url} (fetched via the GitHub README API):\n\n` + apiBody;
|
|
1399
1518
|
await persistCache(url, rendered, "text/markdown", useCache);
|
|
1400
|
-
return truncatePageText(rendered, maxChars);
|
|
1519
|
+
return { text: truncatePageText(rendered, maxChars), hint: null };
|
|
1401
1520
|
}
|
|
1402
1521
|
const rawReadmeUrl = githubRepoRawReadmeUrl(url);
|
|
1403
1522
|
if (rawReadmeUrl) {
|
|
@@ -1406,7 +1525,7 @@ export async function fetchPageText(
|
|
|
1406
1525
|
if (rawBody.trim()) {
|
|
1407
1526
|
const rendered = `README of ${url} (fetched via the GitHub raw README URL):\n\n` + rawBody;
|
|
1408
1527
|
await persistCache(url, rendered, "text/markdown", useCache);
|
|
1409
|
-
return truncatePageText(rendered, maxChars);
|
|
1528
|
+
return { text: truncatePageText(rendered, maxChars), hint: null };
|
|
1410
1529
|
}
|
|
1411
1530
|
}
|
|
1412
1531
|
}
|
|
@@ -1421,7 +1540,7 @@ export async function fetchPageText(
|
|
|
1421
1540
|
let cachedOut = renderBody(cached.body, cached.contentType);
|
|
1422
1541
|
if (!isFresh(cached, now())) cachedOut += staleNotice(cached);
|
|
1423
1542
|
else cachedOut += "\n\n*Served from cache — network fetch failed*";
|
|
1424
|
-
return truncatePageText(cachedOut, maxChars);
|
|
1543
|
+
return { text: truncatePageText(cachedOut, maxChars), hint: null };
|
|
1425
1544
|
}
|
|
1426
1545
|
} catch {}
|
|
1427
1546
|
}
|
|
@@ -1433,14 +1552,19 @@ export async function fetchPageText(
|
|
|
1433
1552
|
const ts = wb.timestamp ? `${wb.timestamp.slice(0, 4)}-${wb.timestamp.slice(4, 6)}-${wb.timestamp.slice(6, 8)}` : "unknown date";
|
|
1434
1553
|
out = `*Fetched from Wayback Machine snapshot (${ts}) for ${originalUrl}:*\n\n` + out;
|
|
1435
1554
|
await persistCache(originalUrl, wb.body, wb.contentType, useCache);
|
|
1436
|
-
return truncatePageText(out, maxChars);
|
|
1555
|
+
return { text: truncatePageText(out, maxChars), hint: null };
|
|
1437
1556
|
}
|
|
1438
1557
|
} catch {}
|
|
1439
1558
|
}
|
|
1440
1559
|
}
|
|
1441
|
-
return result.error;
|
|
1560
|
+
return { text: result.error, hint: null };
|
|
1442
1561
|
}
|
|
1443
1562
|
const finalOut = renderBody(result.body, result.contentType);
|
|
1563
|
+
const hint = isHtmlContent(result.body, result.contentType) ? needsRenderHint(result.body, finalOut) : null;
|
|
1444
1564
|
await persistCache(originalUrl, result.body, result.contentType, useCache);
|
|
1445
|
-
return truncatePageText(finalOut, maxChars);
|
|
1565
|
+
return { text: truncatePageText(finalOut, maxChars), hint };
|
|
1566
|
+
}
|
|
1567
|
+
|
|
1568
|
+
export async function fetchPageText(url: string, options: FetchPageOptions = {}): Promise<string> {
|
|
1569
|
+
return (await fetchPageOutcome(url, options)).text;
|
|
1446
1570
|
}
|
package/config-ui.ts
DELETED
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
import type { Theme } from "@earendil-works/pi-coding-agent";
|
|
2
|
-
import { Key, matchesKey, visibleWidth } from "@earendil-works/pi-tui";
|
|
3
|
-
import { readConfig, type Config } from "./config.ts";
|
|
4
|
-
|
|
5
|
-
export type ConfigToggleKey = "webRenderEnabled";
|
|
6
|
-
|
|
7
|
-
export interface ConfigRow {
|
|
8
|
-
key: ConfigToggleKey;
|
|
9
|
-
label: string;
|
|
10
|
-
hint: string;
|
|
11
|
-
enabled: boolean;
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
export function configRows(config: Config): ConfigRow[] {
|
|
15
|
-
return [
|
|
16
|
-
{
|
|
17
|
-
key: "webRenderEnabled",
|
|
18
|
-
label: "Web render",
|
|
19
|
-
hint: "web_render tool (Jina Reader)",
|
|
20
|
-
enabled: config.webRenderEnabled !== false,
|
|
21
|
-
},
|
|
22
|
-
];
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
function padRow(theme: Theme, innerWidth: number, content: string): string {
|
|
26
|
-
const padded = content + " ".repeat(Math.max(0, innerWidth - visibleWidth(content)));
|
|
27
|
-
return theme.fg("border", "│") + padded + theme.fg("border", "│");
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
export class WebToolsConfigOverlay {
|
|
31
|
-
private rows: ConfigRow[] = [];
|
|
32
|
-
private selected = 0;
|
|
33
|
-
|
|
34
|
-
constructor(
|
|
35
|
-
private readonly opts: {
|
|
36
|
-
tui: { requestRender(force?: boolean): void };
|
|
37
|
-
theme: Theme;
|
|
38
|
-
done: () => void;
|
|
39
|
-
onToggle: (key: ConfigToggleKey) => Promise<void>;
|
|
40
|
-
},
|
|
41
|
-
) {}
|
|
42
|
-
|
|
43
|
-
async load(): Promise<void> {
|
|
44
|
-
this.rows = configRows(await readConfig());
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
private runToggle(row: ConfigRow): void {
|
|
48
|
-
this.opts.tui.requestRender(true);
|
|
49
|
-
void this.opts
|
|
50
|
-
.onToggle(row.key)
|
|
51
|
-
.then(async () => {
|
|
52
|
-
this.rows = configRows(await readConfig());
|
|
53
|
-
this.opts.tui.requestRender(true);
|
|
54
|
-
})
|
|
55
|
-
.catch((error: unknown) => {
|
|
56
|
-
console.error("Failed to toggle web tools setting:", error);
|
|
57
|
-
});
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
private toggleSelected(): void {
|
|
61
|
-
const row = this.rows[this.selected];
|
|
62
|
-
if (!row) return;
|
|
63
|
-
row.enabled = !row.enabled;
|
|
64
|
-
this.runToggle(row);
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
handleInput(data: string): void {
|
|
68
|
-
if (matchesKey(data, Key.up) || data === "k") {
|
|
69
|
-
this.selected = (this.selected + this.rows.length - 1) % this.rows.length;
|
|
70
|
-
return;
|
|
71
|
-
}
|
|
72
|
-
if (matchesKey(data, Key.down) || data === "j") {
|
|
73
|
-
this.selected = (this.selected + 1) % this.rows.length;
|
|
74
|
-
return;
|
|
75
|
-
}
|
|
76
|
-
if (matchesKey(data, Key.space) || matchesKey(data, Key.enter) || data === " " || data === "\r" || data === "\n") {
|
|
77
|
-
this.toggleSelected();
|
|
78
|
-
return;
|
|
79
|
-
}
|
|
80
|
-
if (matchesKey(data, Key.escape) || data === "q") {
|
|
81
|
-
this.opts.done();
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
invalidate(): void {}
|
|
86
|
-
|
|
87
|
-
render(width: number): string[] {
|
|
88
|
-
const theme = this.opts.theme;
|
|
89
|
-
const innerWidth = width - 2;
|
|
90
|
-
const lines: string[] = [];
|
|
91
|
-
lines.push(theme.fg("border", `╭${"─".repeat(innerWidth)}╮`));
|
|
92
|
-
lines.push(padRow(theme, innerWidth, ` ${theme.fg("accent", theme.bold("Web Tools Config"))}`));
|
|
93
|
-
lines.push(theme.fg("border", `├${"─".repeat(innerWidth)}┤`));
|
|
94
|
-
this.rows.forEach((row, index) => {
|
|
95
|
-
const cursor = index === this.selected ? theme.fg("accent", "> ") : " ";
|
|
96
|
-
const box = row.enabled ? theme.fg("success", "[x]") : theme.fg("dim", "[ ]");
|
|
97
|
-
const label = index === this.selected ? theme.fg("accent", theme.bold(row.label)) : row.label;
|
|
98
|
-
lines.push(padRow(theme, innerWidth, `${cursor}${box} ${label} ${theme.fg("dim", `— ${row.hint}`)}`));
|
|
99
|
-
});
|
|
100
|
-
lines.push(theme.fg("border", `├${"─".repeat(innerWidth)}┤`));
|
|
101
|
-
lines.push(padRow(theme, innerWidth, theme.fg("dim", " ↑↓ navigate · space toggle · q close")));
|
|
102
|
-
lines.push(theme.fg("border", `╰${"─".repeat(innerWidth)}╯`));
|
|
103
|
-
return lines;
|
|
104
|
-
}
|
|
105
|
-
}
|