pi-unsloth-webtools 0.8.0 → 0.9.1

package/settings.ts

@@ -1,6 +1,7 @@
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { agentDir } from "./agent-dir.ts";
+import type { FetchTransport } from "./web-fetch.ts";
 
 async function readJson(file: string): Promise<Record<string, unknown> | undefined> {
   try {
@@ -49,6 +50,23 @@ function pickBoolean(data: Record<string, unknown>, paths: string[][]): boolean
   return undefined;
 }
 
+function pickStringArray(data: Record<string, unknown>, paths: string[][]): string[] | undefined {
+  for (const path of paths) {
+    let cur: unknown = data;
+    for (const key of path) {
+      if (cur && typeof cur === "object" && !Array.isArray(cur)) cur = (cur as Record<string, unknown>)[key];
+      else {
+        cur = undefined;
+        break;
+      }
+    }
+    if (!Array.isArray(cur)) continue;
+    const values = cur.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0);
+    if (values.length) return values;
+  }
+  return undefined;
+}
+
 function pickString(data: Record<string, unknown>, paths: string[][]): string | undefined {
   for (const path of paths) {
     let cur: unknown = data;
@@ -94,11 +112,27 @@ const ALLOW_LOCAL_FILES_PATHS: string[][] = [
   ["webFetch", "allowLocalFiles"],
 ];
 
-const JINA_API_KEY_PATHS: string[][] = [
-  ["unslothWebTools", "jinaApiKey"],
-  ["webRender", "jinaApiKey"],
+const TRANSPORT_PATHS: string[][] = [
+  ["unslothWebTools", "transport"],
+  ["webFetch", "transport"],
+];
+
+const TRANSPORT_VALUES: ReadonlySet<string> = new Set(["tls-first", "direct-first", "off"]);
+
+const LIGHTPANDA_ENABLED_PATHS: string[][] = [
+  ["unslothWebTools", "lightpandaEnabled"],
+  ["webRender", "lightpandaEnabled"],
+];
+
+const LIGHTPANDA_PATH_PATHS: string[][] = [
+  ["unslothWebTools", "lightpandaPath"],
+  ["webRender", "lightpandaPath"],
 ];
 
+const LIGHTPANDA_COMMAND_PATHS: string[][] = [
+  ["unslothWebTools", "lightpandaCommand"],
+  ["webRender", "lightpandaCommand"],
+];
 function clampMaxResults(value: number): number {
   return Math.min(20, Math.max(1, value));
 }
@@ -151,11 +185,13 @@ export async function loadDefaultFetchSettings(cwd?: string): Promise<{
   timeoutMs?: number;
   allowPrivateAddresses: boolean;
   allowLocalFiles: boolean;
+  transport: FetchTransport;
 }> {
   let maxChars: number | undefined;
   let timeoutMs: number | undefined;
   let allowPrivateAddresses = true;
   let allowLocalFiles = true;
+  let transport: FetchTransport = "tls-first";
   for (const data of await settingsEntries(cwd)) {
     const c = pickNumber(data, FETCH_MAX_CHARS_PATHS);
     if (c !== undefined && c > 0) maxChars = c;
@@ -165,17 +201,27 @@ export async function loadDefaultFetchSettings(cwd?: string): Promise<{
     if (p !== undefined) allowPrivateAddresses = p;
     const l = pickBoolean(data, ALLOW_LOCAL_FILES_PATHS);
     if (l !== undefined) allowLocalFiles = l;
+    const mode = pickString(data, TRANSPORT_PATHS);
+    if (mode !== undefined && TRANSPORT_VALUES.has(mode)) transport = mode as FetchTransport;
   }
-  return { maxChars, timeoutMs, allowPrivateAddresses, allowLocalFiles };
+  return { maxChars, timeoutMs, allowPrivateAddresses, allowLocalFiles, transport };
 }
 
-export async function loadJinaApiKey(cwd?: string): Promise<string | undefined> {
-  let result: string | undefined;
+export async function loadLightpandaSettings(cwd?: string): Promise<{
+  enabled: boolean;
+  binaryPath: string | null;
+  command: string[] | null;
+}> {
+  let enabled = true;
+  let binaryPath: string | null = null;
+  let command: string[] | null = null;
   for (const data of await settingsEntries(cwd)) {
-    const candidate = pickString(data, JINA_API_KEY_PATHS);
-    if (candidate !== undefined) result = candidate;
+    const e = pickBoolean(data, LIGHTPANDA_ENABLED_PATHS);
+    if (e !== undefined) enabled = e;
+    const p = pickString(data, LIGHTPANDA_PATH_PATHS);
+    if (p !== undefined) binaryPath = p;
+    const c = pickStringArray(data, LIGHTPANDA_COMMAND_PATHS);
+    if (c !== undefined) command = c;
   }
-  if (result !== undefined) return result;
-  const env = process.env.JINA_API_KEY?.trim();
-  return env ? env : undefined;
+  return { enabled, binaryPath, command };
 }

package/tls-fetch.ts

@@ -0,0 +1,205 @@
+import { MAX_SIGNAL_TIMEOUT_MS } from "./web-access.ts";
+
+const MODULE_NAME = "wreq-js";
+const DEFAULT_PROFILE = "chrome_145";
+const DEFAULT_OS = "windows";
+const TRANSPORT_CACHE_LIMIT = 8;
+
+const HOP_HEADER_EXCLUSIONS = new Set(["host", "user-agent", "accept-encoding", "content-length"]);
+const PDF_MAGIC = Buffer.from("%PDF-");
+const MAGIC_HEAD_BYTES = 1024;
+const SKIPPED_MAGIC_PREFIX_BYTES = new Set([0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x20]);
+const BOM_PREFIXES = [
+  Buffer.from([0xff, 0xfe, 0x00, 0x00]),
+  Buffer.from([0x00, 0x00, 0xfe, 0xff]),
+  Buffer.from([0xff, 0xfe]),
+  Buffer.from([0xfe, 0xff]),
+  Buffer.from([0xef, 0xbb, 0xbf]),
+];
+
+function looksLikePdf(head: Buffer): boolean {
+  let start = 0;
+  while (start < head.length && SKIPPED_MAGIC_PREFIX_BYTES.has(head[start])) start++;
+  for (const bom of BOM_PREFIXES) {
+    if (head.subarray(start).length >= bom.length && head.subarray(start, start + bom.length).equals(bom)) {
+      start += bom.length;
+      while (start < head.length && SKIPPED_MAGIC_PREFIX_BYTES.has(head[start])) start++;
+      break;
+    }
+  }
+  const magic = head.subarray(start);
+  return magic.length >= PDF_MAGIC.length && magic.subarray(0, PDF_MAGIC.length).equals(PDF_MAGIC);
+}
+
+interface ImpersonationResponse {
+  status: number;
+  url: string;
+  headers: { get(name: string): string | null };
+  body: AsyncIterable<Uint8Array> | null;
+}
+
+interface ImpersonationTransport {
+  close?(): Promise<void>;
+}
+
+interface ImpersonationModule {
+  createTransport(options: Record<string, unknown>): Promise<ImpersonationTransport>;
+  fetch(input: string, init: Record<string, unknown>): Promise<ImpersonationResponse>;
+}
+
+export type ImpersonationLoader = () => Promise<ImpersonationModule | null>;
+
+export interface TlsHopOptions {
+  url: URL;
+  pinnedIp: string;
+  family: number;
+  timeoutMs: number;
+  signal?: AbortSignal;
+  maxBytes: number;
+  maxPdfBytes?: number;
+  profile?: string;
+  extraHeaders?: Record<string, string>;
+  loader?: ImpersonationLoader;
+}
+
+export interface TlsHopResponse {
+  status: number;
+  headers: Record<string, string>;
+  body: Buffer;
+  truncated: boolean;
+}
+
+let cachedModule: Promise<ImpersonationModule | null> | null = null;
+
+async function loadImpersonationModule(): Promise<ImpersonationModule | null> {
+  cachedModule ??= (async () => {
+    try {
+      return (await import(MODULE_NAME)) as unknown as ImpersonationModule;
+    } catch {
+      return null;
+    }
+  })();
+  return cachedModule;
+}
+
+const transports = new Map<string, Promise<ImpersonationTransport>>();
+
+function transportKey(options: TlsHopOptions): string {
+  return `${options.profile ?? DEFAULT_PROFILE}|${options.url.hostname}|${options.pinnedIp}|${options.family}`;
+}
+
+function evictTransports(): void {
+  while (transports.size > TRANSPORT_CACHE_LIMIT) {
+    const oldestKey = transports.keys().next().value as string | undefined;
+    if (oldestKey === undefined) return;
+    const oldest = transports.get(oldestKey);
+    transports.delete(oldestKey);
+    void oldest
+      ?.then((transport) => transport.close?.())
+      .catch(() => {});
+  }
+}
+
+async function transportFor(
+  module: ImpersonationModule,
+  options: TlsHopOptions,
+): Promise<ImpersonationTransport> {
+  const key = transportKey(options);
+  const existing = transports.get(key);
+  if (existing) return existing;
+  const created = module
+    .createTransport({
+      resolve: { [options.url.hostname]: options.pinnedIp },
+      browser: options.profile ?? DEFAULT_PROFILE,
+      os: DEFAULT_OS,
+    })
+    .catch((error: unknown) => {
+      transports.delete(key);
+      throw error;
+    });
+  transports.set(key, created);
+  evictTransports();
+  return created;
+}
+
+async function readCapped(
+  body: AsyncIterable<Uint8Array> | null,
+  maxBytes: number,
+  maxPdfBytes: number,
+): Promise<{ buffer: Buffer; truncated: boolean }> {
+  if (!body) return { buffer: Buffer.alloc(0), truncated: false };
+  const chunks: Buffer[] = [];
+  let total = 0;
+  let truncated = false;
+  let head = Buffer.alloc(0);
+  let limit = maxBytes;
+  for await (const chunk of body) {
+    const buffer = Buffer.from(chunk);
+    if (head.length < MAGIC_HEAD_BYTES) {
+      head = Buffer.concat([head, buffer.subarray(0, MAGIC_HEAD_BYTES - head.length)]);
+      if (limit === maxBytes && looksLikePdf(head)) limit = maxPdfBytes;
+    }
+    if (total + buffer.length > limit) {
+      const take = buffer.subarray(0, limit - total);
+      chunks.push(take);
+      total += take.length;
+      truncated = true;
+      break;
+    }
+    chunks.push(buffer);
+    total += buffer.length;
+  }
+  return { buffer: Buffer.concat(chunks), truncated };
+}
+
+function failureMessage(error: unknown, caller: AbortSignal | undefined, attempt: AbortSignal): string {
+  if (caller?.aborted) return "cancelled";
+  if (attempt.aborted) return "timed out";
+  if (error instanceof Error && error.name === "TimeoutError") return "timed out";
+  if (error instanceof Error && error.name === "AbortError") return "cancelled";
+  return error instanceof Error ? error.message : String(error);
+}
+
+function hopHeaders(extraHeaders: Record<string, string> | undefined): Record<string, string> {
+  const headers: Record<string, string> = {};
+  for (const [key, value] of Object.entries(extraHeaders ?? {})) {
+    if (HOP_HEADER_EXCLUSIONS.has(key.toLowerCase())) continue;
+    headers[key] = value;
+  }
+  return headers;
+}
+
+export async function impersonatedRequest(options: TlsHopOptions): Promise<TlsHopResponse | null> {
+  const load = options.loader ?? loadImpersonationModule;
+  const module = await load();
+  if (!module) return null;
+  const transport = await transportFor(module, options);
+  const timeoutMs = Math.min(MAX_SIGNAL_TIMEOUT_MS, Math.max(1, Math.floor(options.timeoutMs)));
+  const timeoutSignal = AbortSignal.timeout(timeoutMs);
+  const signal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;
+  let response: ImpersonationResponse;
+  try {
+    response = await module.fetch(options.url.toString(), {
+      transport,
+      redirect: "manual",
+      signal,
+      headers: hopHeaders(options.extraHeaders),
+    });
+  } catch (error) {
+    throw new Error(failureMessage(error, options.signal, signal));
+  }
+  if (response.status < 200) return null;
+  const { buffer, truncated } = await readCapped(
+    response.body,
+    options.maxBytes,
+    options.maxPdfBytes ?? options.maxBytes,
+  );
+  const headers: Record<string, string> = {};
+  const contentType = response.headers.get("content-type");
+  if (contentType) headers["content-type"] = contentType;
+  if (response.status >= 300 && response.status < 400) {
+    const location = response.headers.get("location");
+    if (location) headers.location = location;
+  }
+  return { status: response.status, headers, body: buffer, truncated };
+}

package/web-fetch.ts

@@ -23,6 +23,7 @@ import {
   stripIpv6Brackets,
   type WebsitePolicy,
 } from "./web-access.ts";
+import { impersonatedRequest, type TlsHopOptions, type TlsHopResponse } from "./tls-fetch.ts";
 import { collapseWhitespace, decodeHtmlEntities, feedHtml, htmlToMarkdown, visibleChars } from "./html-to-md.ts";
 import type { AttrDict } from "./html-to-md.ts";
 import { INVALID_CHARREFS } from "./entities.ts";
@@ -157,8 +158,8 @@ export class FetchTimeoutError extends Error {
 }
 const FETCH_CANCELLED_MESSAGE = "Failed to fetch URL: cancelled.";
 const FETCH_TIMEOUT_MESSAGE = "Failed to fetch URL: timed out.";
-const TRUNCATED_BODY_SUFFIX = "... (page truncated at the download limit)";
-const TRUNCATED_BODY_NOTICE = "\n\n" + TRUNCATED_BODY_SUFFIX;
+export const TRUNCATED_BODY_SUFFIX = "... (page truncated at the download limit)";
+export const TRUNCATED_BODY_NOTICE = "\n\n" + TRUNCATED_BODY_SUFFIX;
 
 function fetchErrorMessage(err: unknown): string {
   if (err instanceof FetchCancelledError) return FETCH_CANCELLED_MESSAGE;
@@ -188,6 +189,7 @@ export interface FetchPageOptions {
   maxBytes?: number;
   maxPdfBytes?: number;
   allowPrivateAddresses?: boolean;
+  transport?: FetchTransport;
   allowLocalFiles?: boolean;
   seams?: FetchSeams;
   rawFetch?: (url: string, options: RawFetchOptions) => Promise<RawFetchResult>;
@@ -208,9 +210,13 @@ export interface ResolvedHost {
   alternates?: { ip: string; family: number }[];
 }
 
+export type FetchTransport = "tls-first" | "direct-first" | "off";
+
+type TransportKind = "direct" | "tls";
 export interface FetchSeams {
   resolve?: (hostname: string, signal?: AbortSignal, allowPrivateAddresses?: boolean) => Promise<ResolvedHost>;
   request?: (opts: HopOptions) => Promise<HopResponse>;
+  impersonate?: (options: TlsHopOptions) => Promise<TlsHopResponse | null>;
 }
 
 export interface HopOptions {
@@ -236,6 +242,7 @@ export interface RawFetchOptions {
   maxBytes?: number;
   maxPdfBytes?: number;
   allowPrivateAddresses?: boolean;
+  transport?: FetchTransport;
   seams?: FetchSeams;
 }
 
@@ -903,6 +910,10 @@ export async function fetchUrlRaw(
   const resolveHost = seams.resolve ?? resolveAndValidateHost;
   const allowPrivateAddresses = options.allowPrivateAddresses ?? true;
   const performRequest = seams.request ?? requestHop;
+  const transportMode = options.transport ?? "tls-first";
+  const performImpersonation =
+    transportMode === "off" ? null : (seams.impersonate ?? (seams.request ? null : impersonatedRequest));
+  const directFirst = transportMode === "direct-first" || performImpersonation === null;
   const resolveWithBudget = async (hostname: string): Promise<ResolvedHost> => {
     const abortController = new AbortController();
     const waitMs = clampedRemainingMs(deadline, now);
@@ -937,6 +948,7 @@ export async function fetchUrlRaw(
   let pinnedFamily = resolved.family;
   let alternates: { ip: string; family: number }[] = resolved.alternates ?? [];
   let alternateIndex = 0;
+  const triedTransports = new Set<string>();
   const userAgent = randomUserAgent();
   for (let hop = 0; hop < MAX_REQUESTS; hop++) {
     const budgetResult = checkBudget();
@@ -960,34 +972,67 @@ export async function fetchUrlRaw(
     }
     headers["Host"] = hostHeader;
     const inactivity = remainingMs(deadline, now);
-    let response: HopResponse;
-    try {
-      response = await performRequest({
-        url: parsed,
-        pinnedIp,
-        family: pinnedFamily,
-        headers,
-        maxBytes,
-        maxPdfBytes,
-        inactivityMs: inactivity,
-        deadlineMs: deadline,
-        nowMs: now,
-        signal,
-      });
-    } catch (err) {
-      const message = err instanceof Error ? err.message : "";
-      if (message === "cancelled" || message === "timed out") return emptyResult(fetchErrorMessage(err));
-      if (
-        !(err instanceof FetchCancelledError) &&
-        !(err instanceof FetchTimeoutError) &&
-        alternateIndex < alternates.length
-      ) {
+    const order: TransportKind[] = socksProxyForUrl(parsed) !== null
+      ? ["direct"]
+      : directFirst
+        ? ["direct", "tls"]
+        : ["tls", "direct"];
+    let response: HopResponse | null = null;
+    let failure: unknown = null;
+    for (const kind of order) {
+      const key = `${kind}|${currentUrl}|${pinnedIp}`;
+      if (triedTransports.has(key)) continue;
+      triedTransports.add(key);
+      let attempt: HopResponse | null;
+      try {
+        if (kind === "tls") {
+          const impersonate = performImpersonation;
+          if (impersonate === null) continue;
+          attempt = await impersonate({
+            url: parsed,
+            pinnedIp,
+            family: pinnedFamily,
+            timeoutMs: remainingMs(deadline, now),
+            signal,
+            maxBytes,
+            maxPdfBytes,
+            extraHeaders: options.extraHeaders,
+          });
+        } else {
+          attempt = await performRequest({
+            url: parsed,
+            pinnedIp,
+            family: pinnedFamily,
+            headers,
+            maxBytes,
+            maxPdfBytes,
+            inactivityMs: inactivity,
+            deadlineMs: deadline,
+            nowMs: now,
+            signal,
+          });
+        }
+      } catch (err) {
+        const message = err instanceof Error ? err.message : "";
+        if (message === "cancelled" || message === "timed out") return emptyResult(fetchErrorMessage(err));
+        if (err instanceof FetchCancelledError || err instanceof FetchTimeoutError) {
+          return emptyResult(fetchErrorMessage(err));
+        }
+        failure = err;
+        continue;
+      }
+      if (attempt === null) continue;
+      if (response === null || (attempt.status < 400 && response.status >= 400)) response = attempt;
+      if (attempt.status < 400) break;
+    }
+    if (response === null) {
+      if (alternateIndex < alternates.length) {
         const next = alternates[alternateIndex++];
         pinnedIp = next.ip;
         pinnedFamily = next.family;
         continue;
       }
-      return emptyResult(fetchErrorMessage(err));
+      return emptyResult(fetchErrorMessage(failure ?? new Error("no transport could reach the target")));
     }
 
     if (response.status >= 300 && response.status < 400) {
@@ -1256,7 +1301,7 @@ function extractPageMeta(html: string): PageMeta {
   return meta;
 }
 
-function pagePrefixedMarkdown(html: string): string {
+export function pagePrefixedMarkdown(html: string): string {
   const meta = extractPageMeta(html);
   const lines: string[] = [];
   if (meta.title) lines.push(`Title: ${meta.title}`);
@@ -1446,6 +1491,7 @@ export async function fetchPageOutcome(
     maxBytes: options.maxBytes,
     maxPdfBytes: options.maxPdfBytes,
     seams: options.seams,
+    transport: options.transport,
   };
   const useCache = !options.seams && !options.rawFetch;
   const githubRaw = githubRawContentUrl(url);

package/config-ui.ts

@@ -1,105 +0,0 @@
-import type { Theme } from "@earendil-works/pi-coding-agent";
-import { Key, matchesKey, visibleWidth } from "@earendil-works/pi-tui";
-import { readConfig, type Config } from "./config.ts";
-
-export type ConfigToggleKey = "webRenderEnabled";
-
-export interface ConfigRow {
-  key: ConfigToggleKey;
-  label: string;
-  hint: string;
-  enabled: boolean;
-}
-
-export function configRows(config: Config): ConfigRow[] {
-  return [
-    {
-      key: "webRenderEnabled",
-      label: "JavaScript rendering",
-      hint: "Jina Reader fallback",
-      enabled: config.webRenderEnabled !== false,
-    },
-  ];
-}
-
-function padRow(theme: Theme, innerWidth: number, content: string): string {
-  const padded = content + " ".repeat(Math.max(0, innerWidth - visibleWidth(content)));
-  return theme.fg("border", "│") + padded + theme.fg("border", "│");
-}
-
-export class WebToolsConfigOverlay {
-  private rows: ConfigRow[] = [];
-  private selected = 0;
-
-  constructor(
-    private readonly opts: {
-      tui: { requestRender(force?: boolean): void };
-      theme: Theme;
-      done: () => void;
-      onToggle: (key: ConfigToggleKey) => Promise<void>;
-    },
-  ) {}
-
-  async load(): Promise<void> {
-    this.rows = configRows(await readConfig());
-  }
-
-  private runToggle(row: ConfigRow): void {
-    this.opts.tui.requestRender(true);
-    void this.opts
-      .onToggle(row.key)
-      .then(async () => {
-        this.rows = configRows(await readConfig());
-        this.opts.tui.requestRender(true);
-      })
-      .catch((error: unknown) => {
-        console.error("Failed to toggle web tools setting:", error);
-      });
-  }
-
-  private toggleSelected(): void {
-    const row = this.rows[this.selected];
-    if (!row) return;
-    row.enabled = !row.enabled;
-    this.runToggle(row);
-  }
-
-  handleInput(data: string): void {
-    if (matchesKey(data, Key.up) || data === "k") {
-      this.selected = (this.selected + this.rows.length - 1) % this.rows.length;
-      return;
-    }
-    if (matchesKey(data, Key.down) || data === "j") {
-      this.selected = (this.selected + 1) % this.rows.length;
-      return;
-    }
-    if (matchesKey(data, Key.space) || matchesKey(data, Key.enter) || data === " " || data === "\r" || data === "\n") {
-      this.toggleSelected();
-      return;
-    }
-    if (matchesKey(data, Key.escape) || data === "q") {
-      this.opts.done();
-    }
-  }
-
-  invalidate(): void {}
-
-  render(width: number): string[] {
-    const theme = this.opts.theme;
-    const innerWidth = width - 2;
-    const lines: string[] = [];
-    lines.push(theme.fg("border", `╭${"─".repeat(innerWidth)}╮`));
-    lines.push(padRow(theme, innerWidth, ` ${theme.fg("accent", theme.bold("Web Tools Config"))}`));
-    lines.push(theme.fg("border", `├${"─".repeat(innerWidth)}┤`));
-    this.rows.forEach((row, index) => {
-      const cursor = index === this.selected ? theme.fg("accent", "> ") : "  ";
-      const box = row.enabled ? theme.fg("success", "[x]") : theme.fg("dim", "[ ]");
-      const label = index === this.selected ? theme.fg("accent", theme.bold(row.label)) : row.label;
-      lines.push(padRow(theme, innerWidth, `${cursor}${box} ${label} ${theme.fg("dim", `— ${row.hint}`)}`));
-    });
-    lines.push(theme.fg("border", `├${"─".repeat(innerWidth)}┤`));
-    lines.push(padRow(theme, innerWidth, theme.fg("dim", " ↑↓ navigate · space toggle · q close")));
-    lines.push(theme.fg("border", `╰${"─".repeat(innerWidth)}╯`));
-    return lines;
-  }
-}