pi-unsloth-webtools 0.4.0 → 0.4.2

package/README.md

@@ -61,6 +61,9 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   When a host publishes both IPv4 and IPv6 addresses, IPv4 is preferred (broken IPv6 routes
   cannot stall a fetch), and a connection failure falls back to the next validated address for
   the same host before giving up.
+- Transient DNS failures (EAI_AGAIN, resolver timeouts, connection refusals) are retried once
+  with a short backoff inside the same deadline, so a brief resolver hiccup does not fail a
+  fetch; the deadline abort cuts a retry short when no budget remains.
 - GitHub repo root pages are rewritten to the unauthenticated README API
   (`Accept: application/vnd.github.raw+json`), falling back to the raw README URL
   (`raw.githubusercontent.com`, no API rate limit) and then to the HTML page on failure.

package/engines.ts

@@ -216,6 +216,10 @@ function parsePredExpr(input: string): Pred {
       pos += "position()=last()".length;
       return { op: "last" };
     }
+    if (input.startsWith("last()", pos)) {
+      pos += "last()".length;
+      return { op: "last" };
+    }
     if (input.startsWith("contains(@class,", pos)) {
       pos += "contains(@class,".length;
       const value = quoted();
@@ -359,7 +363,38 @@ function matchesPred(pred: Pred, el: DomNode, index: number, total: number): boo
   }
 }
 
+function predContainsLast(pred: Pred): boolean {
+  switch (pred.op) {
+    case "last":
+      return true;
+    case "or":
+    case "and":
+      return predContainsLast(pred.a) || predContainsLast(pred.b);
+    case "child":
+      return pred.preds.some(predContainsLast);
+    default:
+      return false;
+  }
+}
+
 function applyStep(step: XStep, nodes: DomNode[]): DomNode[] {
+  if (step.preds.some(predContainsLast)) {
+    const out: DomNode[] = [];
+    const seen = new Set<DomNode>();
+    for (const node of nodes) {
+      const source = step.axis === "child" ? node.children : descendantsOf(node);
+      const list = source.filter((c) => !step.name || c.tag === step.name);
+      const total = list.length;
+      list.forEach((el, index) => {
+        if (seen.has(el)) return;
+        if (step.preds.every((p) => matchesPred(p, el, index, total))) {
+          seen.add(el);
+          out.push(el);
+        }
+      });
+    }
+    return out;
+  }
   const candidates: DomNode[] = [];
   for (const node of nodes) {
     const list = step.axis === "child" ? node.children : descendantsOf(node);
@@ -427,6 +462,7 @@ export function extractResults(
       if (!data) continue;
       result[key] = key === "href" ? normalizeUrl(data) : normalizeText(data);
     }
+    if (!result.title && !result.href && !result.body) continue;
     results.push(result);
   }
   return results;
@@ -445,8 +481,7 @@ function googleUserAgent(): string {
   return (
     `Mozilla/5.0 (Linux; Android ${androidVer}; ${device}) ` +
     `AppleWebKit/537.36 (KHTML, like Gecko) ` +
-    `Chrome/${chromeMajor}.0.${chromeBuild}.${chromePatch} Mobile Safari/537.36` +
-    "NSTNWV"
+    `Chrome/${chromeMajor}.0.${chromeBuild}.${chromePatch} Mobile Safari/537.36`
   );
 }
 
@@ -794,7 +829,9 @@ export class ResultsAggregator {
   }
 
   append(item: SearchResult): void {
+    if (typeof item.href !== "string" || !item.href.trim()) return;
     const key = canonicalizeHref(item.href);
+    if (!key) return;
     const existing = this.cache.get(key);
     if (!existing || item.body.length > existing.body.length) {
       this.cache.set(key, { ...item, href: key });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.4.0",
+  "version": "0.4.2",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools ported from the Unsloth Studio codebase (DuckDuckGo search, SSRF-safe direct fetching, HTML-to-Markdown extraction)",
   "main": "index.ts",

package/pdf.ts

@@ -344,7 +344,7 @@ function findLink(links: LinkInfo[], span: SpanData): string | null {
     if (midX < link.x0 || midX > link.x1 || midY < link.y0 || midY > link.y1) continue;
     let uri = link.uri;
     for (const c of "()\n") {
-      uri = uri.replaceAll(c, "%0x" + c.charCodeAt(0).toString(16));
+      uri = uri.replaceAll(c, "%" + c.charCodeAt(0).toString(16).padStart(2, "0"));
     }
     return `[${span.text.trim()}](${uri})`;
   }

package/web-access.ts

@@ -137,7 +137,7 @@ export function normalizeWebsitePolicy(value: unknown): WebsitePolicy {
   }
   const normalized: WebsitePolicy = { allowedDomains: [], blockedDomains: [] };
   for (const key of ["allowedDomains", "blockedDomains"] as const) {
-    const rawDomains = raw[key];
+    const rawDomains = raw[key] ?? [];
     if (!Array.isArray(rawDomains)) throw new Error(`${key} must be a list`);
     if (rawDomains.length > MAX_DOMAINS_PER_LIST) {
       throw new Error(`${key} supports at most ${MAX_DOMAINS_PER_LIST} domains`);
@@ -408,7 +408,7 @@ export function isPublicIp(ip: string): boolean {
   }
   if (canonical === "::1" || canonical.startsWith("::")) return false;
   if (canonical.startsWith("fc") || canonical.startsWith("fd")) return false;
-  if (/^fe[89ab][0-9a-f]:/.test(canonical)) return false;
+  if (/^fe[89a-f][0-9a-f]:/.test(canonical)) return false;
   if (canonical.startsWith("ff")) return false;
   if (canonical.startsWith("2001:db8")) return false;
   if (canonical.startsWith("64:ff9b:")) return false;

package/web-fetch.ts

@@ -28,6 +28,8 @@ const MAX_DECOMPRESSED_BYTES = 64 * 1024 * 1024;
 const META_VALUE_MAX_CHARS = 300;
 const MAX_PDF_FETCH_BYTES = 10 * 1024 * 1024;
 const MAX_REQUESTS = 5;
+const MAX_DNS_ATTEMPTS = 2;
+const DNS_RETRY_BACKOFF_MS = 250;
 export const DEFAULT_FETCH_TIMEOUT_MS = 60_000;
 export { loadDefaultFetchMaxChars, loadDefaultFetchTimeoutMs, loadDefaultFetchSettings } from "./settings.ts";
 function remainingMs(deadline: number, now: () => number): number {
@@ -522,34 +524,73 @@ function decodeTis620(bytes: Buffer): string {
 }
 
 
+const TRANSIENT_DNS_ERROR_CODES = new Set([
+  "EAI_AGAIN",
+  "ETIMEDOUT",
+  "ECONNREFUSED",
+  "ENETUNREACH",
+  "EHOSTUNREACH",
+]);
+
+function isTransientDnsError(err: unknown): boolean {
+  if (!(err instanceof Error)) return false;
+  const code = (err as NodeJS.ErrnoException).code;
+  return typeof code === "string" && TRANSIENT_DNS_ERROR_CODES.has(code);
+}
+
+function sleepAbortable(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve) => {
+    if (signal?.aborted) return resolve();
+    const timer = setTimeout(resolve, ms);
+    signal?.addEventListener("abort", () => {
+      clearTimeout(timer);
+      resolve();
+    }, { once: true });
+  });
+}
+
 async function resolveAndValidate(hostname: string, signal?: AbortSignal): Promise<ResolvedHost> {
-  let addresses: { address: string; family: number }[];
-  try {
-    const lookupOptions: LookupAllOptions & { signal?: AbortSignal } = {
-      all: true,
-      verbatim: true,
-      signal,
-    };
-    addresses = await dnsLookup(hostname, lookupOptions);
-  } catch (err) {
-    return { ok: false, reason: `Failed to resolve host: ${err}`, ip: "", family: 0 };
-  }
-  if (!addresses.length) {
-    return { ok: false, reason: `Failed to resolve host: no addresses for '${hostname}'`, ip: "", family: 0 };
-  }
-  for (const entry of addresses) {
-    if (!isPublicIp(entry.address)) {
-      return { ok: false, reason: `Blocked: refusing to fetch the non-public address ${entry.address}.`, ip: "", family: 0 };
+  let lastError: unknown = null;
+  for (let attempt = 0; attempt < MAX_DNS_ATTEMPTS; attempt++) {
+    if (signal?.aborted) break;
+    if (attempt > 0) {
+      await sleepAbortable(DNS_RETRY_BACKOFF_MS, signal);
+      if (signal?.aborted) break;
+    }
+    try {
+      const lookupOptions: LookupAllOptions & { signal?: AbortSignal } = {
+        all: true,
+        verbatim: true,
+        signal,
+      };
+      const addresses = await dnsLookup(hostname, lookupOptions);
+      if (!addresses.length) {
+        return { ok: false, reason: `Failed to resolve host: no addresses for '${hostname}'`, ip: "", family: 0 };
+      }
+      for (const entry of addresses) {
+        if (!isPublicIp(entry.address)) {
+          return { ok: false, reason: `Blocked: refusing to fetch the non-public address ${entry.address}.`, ip: "", family: 0 };
+        }
+      }
+      addresses.sort((a, b) => (a.family === 4 ? 0 : 1) - (b.family === 4 ? 0 : 1));
+      const first = addresses[0];
+      return {
+        ok: true,
+        reason: "",
+        ip: first.address,
+        family: first.family,
+        alternates: addresses.slice(1).map((entry) => ({ ip: entry.address, family: entry.family })),
+      };
+    } catch (err) {
+      lastError = err;
+      if (!isTransientDnsError(err)) break;
     }
   }
-  addresses.sort((a, b) => (a.family === 4 ? 0 : 1) - (b.family === 4 ? 0 : 1));
-  const first = addresses[0];
   return {
-    ok: true,
-    reason: "",
-    ip: first.address,
-    family: first.family,
-    alternates: addresses.slice(1).map((entry) => ({ ip: entry.address, family: entry.family })),
+    ok: false,
+    reason: lastError === null ? "Failed to resolve host: cancelled" : `Failed to resolve host: ${lastError}`,
+    ip: "",
+    family: 0,
   };
 }
 function budgetExceededResult(
@@ -769,16 +810,22 @@ export async function fetchUrlRaw(
   const performRequest = seams.request ?? requestHop;
   const resolveWithBudget = async (hostname: string): Promise<ResolvedHost> => {
     const abortController = new AbortController();
-    const timeoutId = setTimeout(() => abortController.abort(), clampedRemainingMs(deadline, now));
+    const waitMs = clampedRemainingMs(deadline, now);
+    const timeoutId = setTimeout(() => abortController.abort(), waitMs);
     const resolveSignal = signal ? AbortSignal.any([signal, abortController.signal]) : abortController.signal;
+    let deadlineId: ReturnType<typeof setTimeout> | undefined;
+    const deadlinePromise = new Promise<ResolvedHost>((resolve) => {
+      deadlineId = setTimeout(() => resolve({ ok: false, reason: FETCH_TIMEOUT_MESSAGE, ip: "", family: 0 }), waitMs);
+    });
     try {
-      const resolved = await resolveHost(hostname, resolveSignal);
+      const resolved = await Promise.race([resolveHost(hostname, resolveSignal), deadlinePromise]);
       if (resolved.ok) return resolved;
       if (signal?.aborted) return { ...resolved, reason: FETCH_CANCELLED_MESSAGE };
       if (resolveSignal.aborted) return { ...resolved, reason: FETCH_TIMEOUT_MESSAGE };
       return resolved;
     } finally {
       clearTimeout(timeoutId);
+      if (deadlineId !== undefined) clearTimeout(deadlineId);
     }
   };
   const checkBudget = (contentType = ""): RawFetchResult | null => budgetExceededResult(deadline, signal, now, contentType);

package/web-search.ts

@@ -23,15 +23,16 @@ export type SearchClient = (
   query: string,
   maxResults: number,
   signal?: AbortSignal,
+  timeoutMs?: number,
 ) => Promise<SearchResult[]>;
-
 export async function ddgSearch(
   query: string,
   maxResults = MAX_RESULTS,
   signal?: AbortSignal,
+  timeoutMs = SEARCH_TIMEOUT_MS,
 ): Promise<SearchResult[]> {
   if (signal?.aborted) throw new SearchCancelled();
-  return autoTextSearch(query, maxResults, SEARCH_TIMEOUT_MS, signal);
+  return autoTextSearch(query, maxResults, timeoutMs, signal);
 }
 
 const POLICY_OVERFETCH = 4;
@@ -51,8 +52,11 @@ export async function webSearch(
   query: string | undefined,
   options: WebSearchOptions = {},
 ): Promise<string> {
-  const maxResults = options.maxResults ?? (await loadDefaultMaxResults(options.cwd));
-  const timeoutMs = options.timeoutMs ?? SEARCH_TIMEOUT_MS;
+  const rawMaxResults = options.maxResults ?? (await loadDefaultMaxResults(options.cwd));
+  const saneMax = Number.isFinite(rawMaxResults) ? Math.floor(rawMaxResults as number) : MAX_RESULTS;
+  const maxResults = Math.min(20, Math.max(1, saneMax));
+  const rawTimeout = options.timeoutMs ?? SEARCH_TIMEOUT_MS;
+  const timeoutMs = Number.isFinite(rawTimeout) ? (rawTimeout as number) : SEARCH_TIMEOUT_MS;
   const signal = options.signal;
   const policy = options.websitePolicy ?? null;
   const client = options.client ?? ddgSearch;
@@ -66,14 +70,15 @@ export async function webSearch(
         (policy?.blockedDomains?.length ?? 0) > 0,
     );
     const wanted = restricted ? maxResults * POLICY_OVERFETCH : maxResults;
-    const results = await client(effectiveQuery, wanted, signal);
+    const results = await client(effectiveQuery, wanted, signal, timeoutMs);
     if (signal?.aborted) return "Search cancelled.";
     if (!results.length) return EMPTY_SEARCH_RESULTS[0];
     const allowed: SearchResult[] = [];
     for (const result of results) {
       if (allowed.length >= maxResults) break;
       const href = String(result.href ?? "").trim();
-      if (href && !checkUrlAccess(href, policy)[0]) continue;
+      if (!href) continue;
+      if (!checkUrlAccess(href, policy)[0]) continue;
       allowed.push(result);
     }
     if (!allowed.length) return EMPTY_SEARCH_RESULTS[1];