pi-unsloth-webtools 0.7.6 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/web-fetch.ts CHANGED
@@ -2,6 +2,7 @@ import { lookup as dnsLookup } from "node:dns/promises";
2
2
  import type { LookupAllOptions } from "node:dns";
3
3
  import http from "node:http";
4
4
  import https from "node:https";
5
+ import { isIP } from "node:net";
5
6
  import { open } from "node:fs/promises";
6
7
  import type { FileHandle } from "node:fs/promises";
7
8
  import { homedir } from "node:os";
@@ -19,9 +20,11 @@ import {
19
20
  isPublicIp,
20
21
  MAX_SIGNAL_TIMEOUT_MS,
21
22
  normalizeUrlScheme,
23
+ stripIpv6Brackets,
22
24
  type WebsitePolicy,
23
25
  } from "./web-access.ts";
24
- import { collapseWhitespace, decodeHtmlEntities, feedHtml, htmlToMarkdown } from "./html-to-md.ts";
26
+ import { impersonatedRequest, type TlsHopOptions, type TlsHopResponse } from "./tls-fetch.ts";
27
+ import { collapseWhitespace, decodeHtmlEntities, feedHtml, htmlToMarkdown, visibleChars } from "./html-to-md.ts";
25
28
  import type { AttrDict } from "./html-to-md.ts";
26
29
  import { INVALID_CHARREFS } from "./entities.ts";
27
30
  import { getCached, isFresh, setCached, staleNotice } from "./cache.ts";
@@ -125,6 +128,14 @@ const HTML_LEADING_RE = new RegExp(
125
128
  );
126
129
  const HTML_DOCUMENT_RE = /^<(?:!doctype\s+html\b|\/?(?:html|head|body)\b)/;
127
130
 
131
+ const RENDER_THIN_PROSE_CHARS = 400;
132
+ const RENDER_SCRIPT_SHARE = 0.5;
133
+ const RENDER_SCRIPT_MIN_HTML_BYTES = 8192;
134
+ const RENDER_MIN_SCRIPTS = 3;
135
+ const RENDER_NOSCRIPT_MIN_CHARS = 40;
136
+ const RENDER_SPA_MARKERS =
137
+ /(?:id\s*=\s*["']?(?:root|app|__next|__nuxt)["']?|data-reactroot|ng-version|__NEXT_DATA__|__NUXT__|__PRELOADED_STATE__)/i;
138
+
128
139
  const MIN_SINGLE_BYTE_ASCII_RATIO = 3 / 4;
129
140
  const ASCII_TEXT_BYTES = new Set<number>([
130
141
  ...Array.from({ length: 0x7f - 0x20 }, (_, i) => i + 0x20),
@@ -147,8 +158,8 @@ export class FetchTimeoutError extends Error {
147
158
  }
148
159
  const FETCH_CANCELLED_MESSAGE = "Failed to fetch URL: cancelled.";
149
160
  const FETCH_TIMEOUT_MESSAGE = "Failed to fetch URL: timed out.";
150
- const TRUNCATED_BODY_SUFFIX = "... (page truncated at the download limit)";
151
- const TRUNCATED_BODY_NOTICE = "\n\n" + TRUNCATED_BODY_SUFFIX;
161
+ export const TRUNCATED_BODY_SUFFIX = "... (page truncated at the download limit)";
162
+ export const TRUNCATED_BODY_NOTICE = "\n\n" + TRUNCATED_BODY_SUFFIX;
152
163
 
153
164
  function fetchErrorMessage(err: unknown): string {
154
165
  if (err instanceof FetchCancelledError) return FETCH_CANCELLED_MESSAGE;
@@ -178,6 +189,7 @@ export interface FetchPageOptions {
178
189
  maxBytes?: number;
179
190
  maxPdfBytes?: number;
180
191
  allowPrivateAddresses?: boolean;
192
+ transport?: FetchTransport;
181
193
  allowLocalFiles?: boolean;
182
194
  seams?: FetchSeams;
183
195
  rawFetch?: (url: string, options: RawFetchOptions) => Promise<RawFetchResult>;
@@ -198,9 +210,13 @@ export interface ResolvedHost {
198
210
  alternates?: { ip: string; family: number }[];
199
211
  }
200
212
 
213
+ export type FetchTransport = "tls-first" | "direct-first" | "off";
214
+
215
+ type TransportKind = "direct" | "tls";
201
216
  export interface FetchSeams {
202
217
  resolve?: (hostname: string, signal?: AbortSignal, allowPrivateAddresses?: boolean) => Promise<ResolvedHost>;
203
218
  request?: (opts: HopOptions) => Promise<HopResponse>;
219
+ impersonate?: (options: TlsHopOptions) => Promise<TlsHopResponse | null>;
204
220
  }
205
221
 
206
222
  export interface HopOptions {
@@ -226,6 +242,7 @@ export interface RawFetchOptions {
226
242
  maxBytes?: number;
227
243
  maxPdfBytes?: number;
228
244
  allowPrivateAddresses?: boolean;
245
+ transport?: FetchTransport;
229
246
  seams?: FetchSeams;
230
247
  }
231
248
 
@@ -235,6 +252,16 @@ export interface RawFetchResult {
235
252
  contentType: string;
236
253
  }
237
254
 
255
+ export interface RenderHint {
256
+ reason: "empty" | "js-shell";
257
+ evidence: string[];
258
+ }
259
+
260
+ export interface FetchPageOutcome {
261
+ text: string;
262
+ hint: RenderHint | null;
263
+ }
264
+
238
265
  function htmlProbe(body: string, re: RegExp): boolean {
239
266
  let i = 0;
240
267
  const n = body.length;
@@ -265,6 +292,46 @@ export function looksLikeHtmlDocument(body: string): boolean {
265
292
  return htmlProbe(body, HTML_DOCUMENT_RE);
266
293
  }
267
294
 
295
+ function noscriptText(html: string): string {
296
+ const parts: string[] = [];
297
+ for (const match of html.match(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi) ?? []) {
298
+ parts.push(match.replace(/<[^>]*>/g, " "));
299
+ }
300
+ return collapseWhitespace(decodeHtmlEntities(parts.join(" ")));
301
+ }
302
+
303
+ export function needsRenderHint(html: string, converted: string): RenderHint | null {
304
+ const prose = visibleChars(converted);
305
+ if (prose >= RENDER_THIN_PROSE_CHARS) return null;
306
+ if (prose === 0) return { reason: "empty", evidence: [] };
307
+ const evidence: string[] = [];
308
+ let score = 0;
309
+ if (RENDER_SPA_MARKERS.test(html)) {
310
+ evidence.push("spa-markers");
311
+ score += 2;
312
+ }
313
+ const scriptCount = (html.match(/<script\b/gi) ?? []).length;
314
+ let scriptBytes = 0;
315
+ for (const script of html.match(/<script\b[^>]*>[\s\S]*?<\/script>/gi) ?? []) scriptBytes += script.length;
316
+ if (html.length >= RENDER_SCRIPT_MIN_HTML_BYTES && scriptBytes >= html.length * RENDER_SCRIPT_SHARE) {
317
+ evidence.push("script-heavy");
318
+ score += 2;
319
+ }
320
+ if (scriptCount >= RENDER_MIN_SCRIPTS) {
321
+ evidence.push("scripts");
322
+ score += 1;
323
+ }
324
+ if (noscriptText(html).length >= RENDER_NOSCRIPT_MIN_CHARS) {
325
+ evidence.push("noscript");
326
+ score += 1;
327
+ }
328
+ if (extractPageMeta(html).description) {
329
+ evidence.push("description");
330
+ score += 1;
331
+ }
332
+ return score >= 2 ? { reason: "js-shell", evidence } : null;
333
+ }
334
+
268
335
  function parseContentType(value: string | null | undefined): string {
269
336
  return /^[\w.+-]+\/[\w.+-]+/.exec(value ?? "")?.[0] ?? "";
270
337
  }
@@ -320,7 +387,7 @@ function hasSingleByteTextEvidence(data: Buffer): boolean {
320
387
  return ascii / data.length >= MIN_SINGLE_BYTE_ASCII_RATIO;
321
388
  }
322
389
 
323
- const CHARSET_ALIASES: Record<string, string> = {
390
+ const CHARSET_ALIASES: Record<string, string> = Object.assign(Object.create(null), {
324
391
  gbk: "gbk",
325
392
  gb2312: "gbk",
326
393
  "gb-2312": "gbk",
@@ -346,7 +413,7 @@ const CHARSET_ALIASES: Record<string, string> = {
346
413
  "windows-874": "windows-874",
347
414
  cp874: "windows-874",
348
415
  "tis-620": "tis-620",
349
- };
416
+ });
350
417
 
351
418
  function normalizeCharset(name: string): string | null {
352
419
  const n = name.trim().replace(/["']/g, "").toLowerCase();
@@ -637,9 +704,10 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
637
704
  const url = opts.url;
638
705
  const transport = url.protocol === "https:" ? https : http;
639
706
  const port = url.port ? Number(url.port) : url.protocol === "https:" ? 443 : 80;
707
+ const hostname = stripIpv6Brackets(url.hostname);
640
708
  const options: https.RequestOptions = {
641
709
  method: "GET",
642
- host: url.hostname,
710
+ host: hostname,
643
711
  port,
644
712
  path: url.pathname + url.search,
645
713
  headers: opts.headers,
@@ -652,12 +720,12 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
652
720
  ip: opts.pinnedIp,
653
721
  family: opts.family,
654
722
  port,
655
- servername: url.hostname,
723
+ servername: hostname,
656
724
  timeoutMs: Math.max(1, opts.inactivityMs),
657
725
  signal: opts.signal,
658
726
  });
659
727
  } else {
660
- options.servername = url.protocol === "https:" ? url.hostname : undefined;
728
+ options.servername = url.protocol === "https:" && !isIP(hostname) ? hostname : undefined;
661
729
  options.lookup = ((_hostname: string, _options: unknown, _callback: unknown) => {
662
730
  const done = (typeof _options === "function" ? _options : _callback) as (err: unknown, address: unknown, family?: unknown) => void;
663
731
  if ((_options as { all?: boolean } | undefined)?.all) {
@@ -842,6 +910,10 @@ export async function fetchUrlRaw(
842
910
  const resolveHost = seams.resolve ?? resolveAndValidateHost;
843
911
  const allowPrivateAddresses = options.allowPrivateAddresses ?? true;
844
912
  const performRequest = seams.request ?? requestHop;
913
+ const transportMode = options.transport ?? "tls-first";
914
+ const performImpersonation =
915
+ transportMode === "off" ? null : (seams.impersonate ?? (seams.request ? null : impersonatedRequest));
916
+ const directFirst = transportMode === "direct-first" || performImpersonation === null;
845
917
  const resolveWithBudget = async (hostname: string): Promise<ResolvedHost> => {
846
918
  const abortController = new AbortController();
847
919
  const waitMs = clampedRemainingMs(deadline, now);
@@ -876,14 +948,16 @@ export async function fetchUrlRaw(
876
948
  let pinnedFamily = resolved.family;
877
949
  let alternates: { ip: string; family: number }[] = resolved.alternates ?? [];
878
950
  let alternateIndex = 0;
951
+ const triedTransports = new Set<string>();
879
952
  const userAgent = randomUserAgent();
880
953
  for (let hop = 0; hop < MAX_REQUESTS; hop++) {
881
954
  const budgetResult = checkBudget();
882
955
  if (budgetResult !== null) return budgetResult;
883
956
  const parsed = new URL(currentUrl);
884
- const hostHeader = parsed.hostname.includes(":")
885
- ? `[${parsed.hostname}]${parsed.port ? `:${parsed.port}` : ""}`
886
- : parsed.hostname + (parsed.port ? `:${parsed.port}` : "");
957
+ const parsedHostname = stripIpv6Brackets(parsed.hostname);
958
+ const hostHeader = parsedHostname.includes(":")
959
+ ? `[${parsedHostname}]${parsed.port ? `:${parsed.port}` : ""}`
960
+ : parsedHostname + (parsed.port ? `:${parsed.port}` : "");
887
961
  const headers: Record<string, string> = {
888
962
  "User-Agent": userAgent,
889
963
  Host: hostHeader,
@@ -898,34 +972,67 @@ export async function fetchUrlRaw(
898
972
  }
899
973
  headers["Host"] = hostHeader;
900
974
  const inactivity = remainingMs(deadline, now);
901
- let response: HopResponse;
902
- try {
903
- response = await performRequest({
904
- url: parsed,
905
- pinnedIp,
906
- family: pinnedFamily,
907
- headers,
908
- maxBytes,
909
- maxPdfBytes,
910
- inactivityMs: inactivity,
911
- deadlineMs: deadline,
912
- nowMs: now,
913
- signal,
914
- });
915
- } catch (err) {
916
- const message = err instanceof Error ? err.message : "";
917
- if (message === "cancelled" || message === "timed out") return emptyResult(fetchErrorMessage(err));
918
- if (
919
- !(err instanceof FetchCancelledError) &&
920
- !(err instanceof FetchTimeoutError) &&
921
- alternateIndex < alternates.length
922
- ) {
975
+ const order: TransportKind[] = socksProxyForUrl(parsed) !== null
976
+ ? ["direct"]
977
+ : directFirst
978
+ ? ["direct", "tls"]
979
+ : ["tls", "direct"];
980
+ let response: HopResponse | null = null;
981
+ let failure: unknown = null;
982
+ for (const kind of order) {
983
+ const key = `${kind}|${currentUrl}|${pinnedIp}`;
984
+ if (triedTransports.has(key)) continue;
985
+ triedTransports.add(key);
986
+ let attempt: HopResponse | null;
987
+ try {
988
+ if (kind === "tls") {
989
+ const impersonate = performImpersonation;
990
+ if (impersonate === null) continue;
991
+ attempt = await impersonate({
992
+ url: parsed,
993
+ pinnedIp,
994
+ family: pinnedFamily,
995
+ timeoutMs: remainingMs(deadline, now),
996
+ signal,
997
+ maxBytes,
998
+ maxPdfBytes,
999
+ extraHeaders: options.extraHeaders,
1000
+ });
1001
+ } else {
1002
+ attempt = await performRequest({
1003
+ url: parsed,
1004
+ pinnedIp,
1005
+ family: pinnedFamily,
1006
+ headers,
1007
+ maxBytes,
1008
+ maxPdfBytes,
1009
+ inactivityMs: inactivity,
1010
+ deadlineMs: deadline,
1011
+ nowMs: now,
1012
+ signal,
1013
+ });
1014
+ }
1015
+ } catch (err) {
1016
+ const message = err instanceof Error ? err.message : "";
1017
+ if (message === "cancelled" || message === "timed out") return emptyResult(fetchErrorMessage(err));
1018
+ if (err instanceof FetchCancelledError || err instanceof FetchTimeoutError) {
1019
+ return emptyResult(fetchErrorMessage(err));
1020
+ }
1021
+ failure = err;
1022
+ continue;
1023
+ }
1024
+ if (attempt === null) continue;
1025
+ if (response === null || (attempt.status < 400 && response.status >= 400)) response = attempt;
1026
+ if (attempt.status < 400) break;
1027
+ }
1028
+ if (response === null) {
1029
+ if (alternateIndex < alternates.length) {
923
1030
  const next = alternates[alternateIndex++];
924
1031
  pinnedIp = next.ip;
925
1032
  pinnedFamily = next.family;
926
1033
  continue;
927
1034
  }
928
- return emptyResult(fetchErrorMessage(err));
1035
+ return emptyResult(fetchErrorMessage(failure ?? new Error("no transport could reach the target")));
929
1036
  }
930
1037
 
931
1038
  if (response.status >= 300 && response.status < 400) {
@@ -1133,9 +1240,10 @@ interface PageMeta {
1133
1240
  author: string;
1134
1241
  date: string;
1135
1242
  site: string;
1243
+ description: string;
1136
1244
  }
1137
1245
 
1138
- const META_KEYS: Record<string, keyof PageMeta> = {
1246
+ const META_KEYS: Record<string, keyof PageMeta> = Object.assign(Object.create(null), {
1139
1247
  author: "author",
1140
1248
  "article:author": "author",
1141
1249
  "dc.creator": "author",
@@ -1145,7 +1253,9 @@ const META_KEYS: Record<string, keyof PageMeta> = {
1145
1253
  datepublished: "date",
1146
1254
  "og:site_name": "site",
1147
1255
  "application-name": "site",
1148
- };
1256
+ description: "description",
1257
+ "og:description": "description",
1258
+ });
1149
1259
 
1150
1260
  function cutAtCharBoundary(text: string, maxChars: number): string {
1151
1261
  const sliced = text.slice(0, maxChars);
@@ -1165,7 +1275,7 @@ function capMetaValue(value: string): string {
1165
1275
  }
1166
1276
 
1167
1277
  function extractPageMeta(html: string): PageMeta {
1168
- const meta: PageMeta = { title: extractPageTitle(html), author: "", date: "", site: "" };
1278
+ const meta: PageMeta = { title: extractPageTitle(html), author: "", date: "", site: "", description: "" };
1169
1279
  const seen = new Set<string>();
1170
1280
  const record = (name: string, attrs: AttrDict) => {
1171
1281
  if (name !== "meta") return;
@@ -1191,7 +1301,7 @@ function extractPageMeta(html: string): PageMeta {
1191
1301
  return meta;
1192
1302
  }
1193
1303
 
1194
- function pagePrefixedMarkdown(html: string): string {
1304
+ export function pagePrefixedMarkdown(html: string): string {
1195
1305
  const meta = extractPageMeta(html);
1196
1306
  const lines: string[] = [];
1197
1307
  if (meta.title) lines.push(`Title: ${meta.title}`);
@@ -1291,6 +1401,13 @@ function localFileFailure(err: unknown): string {
1291
1401
  return `Failed to read file: ${err instanceof Error ? err.message : String(err)}`;
1292
1402
  }
1293
1403
 
1404
+ const HTML_FILE_EXTENSIONS = new Set([".htm", ".html", ".xht", ".xhtml"]);
1405
+
1406
+ function localFileContentType(filePath: string): string {
1407
+ const dot = filePath.lastIndexOf(".");
1408
+ return dot !== -1 && HTML_FILE_EXTENSIONS.has(filePath.slice(dot).toLowerCase()) ? "text/html" : "";
1409
+ }
1410
+
1294
1411
  async function readLocalFile(
1295
1412
  filePath: string,
1296
1413
  options: { signal?: AbortSignal; maxChars?: number; maxBytes?: number; maxPdfBytes?: number },
@@ -1333,16 +1450,16 @@ async function readLocalFile(
1333
1450
  }
1334
1451
  const text = decodeWithCodec(body, bomCodecFor(body) ?? "utf-8");
1335
1452
  if (looksBinary(text)) return `(binary content, ${body.length} bytes; not readable as text)`;
1336
- return truncatePageText(withTruncation(renderBody(text, ""), truncated), options.maxChars);
1453
+ return truncatePageText(withTruncation(renderBody(text, localFileContentType(filePath)), truncated), options.maxChars);
1337
1454
  } finally {
1338
1455
  await handle.close().catch(() => {});
1339
1456
  }
1340
1457
  }
1341
1458
 
1342
- export async function fetchPageText(
1459
+ export async function fetchPageOutcome(
1343
1460
  url: string,
1344
1461
  options: FetchPageOptions = {},
1345
- ): Promise<string> {
1462
+ ): Promise<FetchPageOutcome> {
1346
1463
  const timeoutMs = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
1347
1464
  const now = options.nowMs ?? Date.now;
1348
1465
  const deadlineMs = options.deadlineMs ?? now() + timeoutMs;
@@ -1353,18 +1470,19 @@ export async function fetchPageText(
1353
1470
  if (options.allowLocalFiles !== false) {
1354
1471
  const localPath = parseLocalPath(url);
1355
1472
  if (localPath !== null) {
1356
- if (!localPath) return "Failed to read file: invalid file URL.";
1357
- return readLocalFile(localPath, {
1473
+ if (!localPath) return { text: "Failed to read file: invalid file URL.", hint: null };
1474
+ const localText = await readLocalFile(localPath, {
1358
1475
  signal,
1359
1476
  maxChars,
1360
1477
  maxBytes: options.maxBytes,
1361
1478
  maxPdfBytes: options.maxPdfBytes,
1362
1479
  });
1480
+ return { text: localText, hint: null };
1363
1481
  }
1364
1482
  }
1365
1483
  url = normalizeUrlScheme(url);
1366
1484
  const [allowed, reason] = checkUrlAccess(url, policy);
1367
- if (!allowed) return reason;
1485
+ if (!allowed) return { text: reason, hint: null };
1368
1486
  const rawFetchOptions = {
1369
1487
  deadlineMs,
1370
1488
  signal,
@@ -1373,6 +1491,7 @@ export async function fetchPageText(
1373
1491
  maxBytes: options.maxBytes,
1374
1492
  maxPdfBytes: options.maxPdfBytes,
1375
1493
  seams: options.seams,
1494
+ transport: options.transport,
1376
1495
  };
1377
1496
  const useCache = !options.seams && !options.rawFetch;
1378
1497
  const githubRaw = githubRawContentUrl(url);
@@ -1381,7 +1500,7 @@ export async function fetchPageText(
1381
1500
  if (rawResult.error === null) {
1382
1501
  const out = renderBody(rawResult.body, rawResult.contentType);
1383
1502
  await persistCache(url, rawResult.body, rawResult.contentType, useCache);
1384
- return truncatePageText(out, maxChars);
1503
+ return { text: truncatePageText(out, maxChars), hint: null };
1385
1504
  }
1386
1505
  }
1387
1506
  const readmeApiUrl = githubRepoReadmeApiUrl(url);
@@ -1397,7 +1516,7 @@ export async function fetchPageText(
1397
1516
  if (apiBody.trim()) {
1398
1517
  const rendered = `README of ${url} (fetched via the GitHub README API):\n\n` + apiBody;
1399
1518
  await persistCache(url, rendered, "text/markdown", useCache);
1400
- return truncatePageText(rendered, maxChars);
1519
+ return { text: truncatePageText(rendered, maxChars), hint: null };
1401
1520
  }
1402
1521
  const rawReadmeUrl = githubRepoRawReadmeUrl(url);
1403
1522
  if (rawReadmeUrl) {
@@ -1406,7 +1525,7 @@ export async function fetchPageText(
1406
1525
  if (rawBody.trim()) {
1407
1526
  const rendered = `README of ${url} (fetched via the GitHub raw README URL):\n\n` + rawBody;
1408
1527
  await persistCache(url, rendered, "text/markdown", useCache);
1409
- return truncatePageText(rendered, maxChars);
1528
+ return { text: truncatePageText(rendered, maxChars), hint: null };
1410
1529
  }
1411
1530
  }
1412
1531
  }
@@ -1421,7 +1540,7 @@ export async function fetchPageText(
1421
1540
  let cachedOut = renderBody(cached.body, cached.contentType);
1422
1541
  if (!isFresh(cached, now())) cachedOut += staleNotice(cached);
1423
1542
  else cachedOut += "\n\n*Served from cache — network fetch failed*";
1424
- return truncatePageText(cachedOut, maxChars);
1543
+ return { text: truncatePageText(cachedOut, maxChars), hint: null };
1425
1544
  }
1426
1545
  } catch {}
1427
1546
  }
@@ -1433,14 +1552,19 @@ export async function fetchPageText(
1433
1552
  const ts = wb.timestamp ? `${wb.timestamp.slice(0, 4)}-${wb.timestamp.slice(4, 6)}-${wb.timestamp.slice(6, 8)}` : "unknown date";
1434
1553
  out = `*Fetched from Wayback Machine snapshot (${ts}) for ${originalUrl}:*\n\n` + out;
1435
1554
  await persistCache(originalUrl, wb.body, wb.contentType, useCache);
1436
- return truncatePageText(out, maxChars);
1555
+ return { text: truncatePageText(out, maxChars), hint: null };
1437
1556
  }
1438
1557
  } catch {}
1439
1558
  }
1440
1559
  }
1441
- return result.error;
1560
+ return { text: result.error, hint: null };
1442
1561
  }
1443
1562
  const finalOut = renderBody(result.body, result.contentType);
1563
+ const hint = isHtmlContent(result.body, result.contentType) ? needsRenderHint(result.body, finalOut) : null;
1444
1564
  await persistCache(originalUrl, result.body, result.contentType, useCache);
1445
- return truncatePageText(finalOut, maxChars);
1565
+ return { text: truncatePageText(finalOut, maxChars), hint };
1566
+ }
1567
+
1568
+ export async function fetchPageText(url: string, options: FetchPageOptions = {}): Promise<string> {
1569
+ return (await fetchPageOutcome(url, options)).text;
1446
1570
  }
package/config-ui.ts DELETED
@@ -1,105 +0,0 @@
1
- import type { Theme } from "@earendil-works/pi-coding-agent";
2
- import { Key, matchesKey, visibleWidth } from "@earendil-works/pi-tui";
3
- import { readConfig, type Config } from "./config.ts";
4
-
5
- export type ConfigToggleKey = "webRenderEnabled";
6
-
7
- export interface ConfigRow {
8
- key: ConfigToggleKey;
9
- label: string;
10
- hint: string;
11
- enabled: boolean;
12
- }
13
-
14
- export function configRows(config: Config): ConfigRow[] {
15
- return [
16
- {
17
- key: "webRenderEnabled",
18
- label: "Web render",
19
- hint: "web_render tool (Jina Reader)",
20
- enabled: config.webRenderEnabled !== false,
21
- },
22
- ];
23
- }
24
-
25
- function padRow(theme: Theme, innerWidth: number, content: string): string {
26
- const padded = content + " ".repeat(Math.max(0, innerWidth - visibleWidth(content)));
27
- return theme.fg("border", "│") + padded + theme.fg("border", "│");
28
- }
29
-
30
- export class WebToolsConfigOverlay {
31
- private rows: ConfigRow[] = [];
32
- private selected = 0;
33
-
34
- constructor(
35
- private readonly opts: {
36
- tui: { requestRender(force?: boolean): void };
37
- theme: Theme;
38
- done: () => void;
39
- onToggle: (key: ConfigToggleKey) => Promise<void>;
40
- },
41
- ) {}
42
-
43
- async load(): Promise<void> {
44
- this.rows = configRows(await readConfig());
45
- }
46
-
47
- private runToggle(row: ConfigRow): void {
48
- this.opts.tui.requestRender(true);
49
- void this.opts
50
- .onToggle(row.key)
51
- .then(async () => {
52
- this.rows = configRows(await readConfig());
53
- this.opts.tui.requestRender(true);
54
- })
55
- .catch((error: unknown) => {
56
- console.error("Failed to toggle web tools setting:", error);
57
- });
58
- }
59
-
60
- private toggleSelected(): void {
61
- const row = this.rows[this.selected];
62
- if (!row) return;
63
- row.enabled = !row.enabled;
64
- this.runToggle(row);
65
- }
66
-
67
- handleInput(data: string): void {
68
- if (matchesKey(data, Key.up) || data === "k") {
69
- this.selected = (this.selected + this.rows.length - 1) % this.rows.length;
70
- return;
71
- }
72
- if (matchesKey(data, Key.down) || data === "j") {
73
- this.selected = (this.selected + 1) % this.rows.length;
74
- return;
75
- }
76
- if (matchesKey(data, Key.space) || matchesKey(data, Key.enter) || data === " " || data === "\r" || data === "\n") {
77
- this.toggleSelected();
78
- return;
79
- }
80
- if (matchesKey(data, Key.escape) || data === "q") {
81
- this.opts.done();
82
- }
83
- }
84
-
85
- invalidate(): void {}
86
-
87
- render(width: number): string[] {
88
- const theme = this.opts.theme;
89
- const innerWidth = width - 2;
90
- const lines: string[] = [];
91
- lines.push(theme.fg("border", `╭${"─".repeat(innerWidth)}╮`));
92
- lines.push(padRow(theme, innerWidth, ` ${theme.fg("accent", theme.bold("Web Tools Config"))}`));
93
- lines.push(theme.fg("border", `├${"─".repeat(innerWidth)}┤`));
94
- this.rows.forEach((row, index) => {
95
- const cursor = index === this.selected ? theme.fg("accent", "> ") : " ";
96
- const box = row.enabled ? theme.fg("success", "[x]") : theme.fg("dim", "[ ]");
97
- const label = index === this.selected ? theme.fg("accent", theme.bold(row.label)) : row.label;
98
- lines.push(padRow(theme, innerWidth, `${cursor}${box} ${label} ${theme.fg("dim", `— ${row.hint}`)}`));
99
- });
100
- lines.push(theme.fg("border", `├${"─".repeat(innerWidth)}┤`));
101
- lines.push(padRow(theme, innerWidth, theme.fg("dim", " ↑↓ navigate · space toggle · q close")));
102
- lines.push(theme.fg("border", `╰${"─".repeat(innerWidth)}╯`));
103
- return lines;
104
- }
105
- }