pi-unsloth-webtools 0.4.0 → 0.4.1

package/README.md

@@ -61,6 +61,9 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   When a host publishes both IPv4 and IPv6 addresses, IPv4 is preferred (broken IPv6 routes
   cannot stall a fetch), and a connection failure falls back to the next validated address for
   the same host before giving up.
+- Transient DNS failures (EAI_AGAIN, resolver timeouts, connection refusals) are retried once
+  with a short backoff inside the same deadline, so a brief resolver hiccup does not fail a
+  fetch; the deadline abort cuts a retry short when no budget remains.
 - GitHub repo root pages are rewritten to the unauthenticated README API
   (`Accept: application/vnd.github.raw+json`), falling back to the raw README URL
   (`raw.githubusercontent.com`, no API rate limit) and then to the HTML page on failure.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools ported from the Unsloth Studio codebase (DuckDuckGo search, SSRF-safe direct fetching, HTML-to-Markdown extraction)",
   "main": "index.ts",

package/web-fetch.ts

@@ -28,6 +28,8 @@ const MAX_DECOMPRESSED_BYTES = 64 * 1024 * 1024;
 const META_VALUE_MAX_CHARS = 300;
 const MAX_PDF_FETCH_BYTES = 10 * 1024 * 1024;
 const MAX_REQUESTS = 5;
+const MAX_DNS_ATTEMPTS = 2;
+const DNS_RETRY_BACKOFF_MS = 250;
 export const DEFAULT_FETCH_TIMEOUT_MS = 60_000;
 export { loadDefaultFetchMaxChars, loadDefaultFetchTimeoutMs, loadDefaultFetchSettings } from "./settings.ts";
 function remainingMs(deadline: number, now: () => number): number {
@@ -522,34 +524,73 @@ function decodeTis620(bytes: Buffer): string {
 }
 
 
+const TRANSIENT_DNS_ERROR_CODES = new Set([
+  "EAI_AGAIN",
+  "ETIMEDOUT",
+  "ECONNREFUSED",
+  "ENETUNREACH",
+  "EHOSTUNREACH",
+]);
+
+function isTransientDnsError(err: unknown): boolean {
+  if (!(err instanceof Error)) return false;
+  const code = (err as NodeJS.ErrnoException).code;
+  return typeof code === "string" && TRANSIENT_DNS_ERROR_CODES.has(code);
+}
+
+function sleepAbortable(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve) => {
+    if (signal?.aborted) return resolve();
+    const timer = setTimeout(resolve, ms);
+    signal?.addEventListener("abort", () => {
+      clearTimeout(timer);
+      resolve();
+    }, { once: true });
+  });
+}
+
 async function resolveAndValidate(hostname: string, signal?: AbortSignal): Promise<ResolvedHost> {
-  let addresses: { address: string; family: number }[];
-  try {
-    const lookupOptions: LookupAllOptions & { signal?: AbortSignal } = {
-      all: true,
-      verbatim: true,
-      signal,
-    };
-    addresses = await dnsLookup(hostname, lookupOptions);
-  } catch (err) {
-    return { ok: false, reason: `Failed to resolve host: ${err}`, ip: "", family: 0 };
-  }
-  if (!addresses.length) {
-    return { ok: false, reason: `Failed to resolve host: no addresses for '${hostname}'`, ip: "", family: 0 };
-  }
-  for (const entry of addresses) {
-    if (!isPublicIp(entry.address)) {
-      return { ok: false, reason: `Blocked: refusing to fetch the non-public address ${entry.address}.`, ip: "", family: 0 };
+  let lastError: unknown = null;
+  for (let attempt = 0; attempt < MAX_DNS_ATTEMPTS; attempt++) {
+    if (signal?.aborted) break;
+    if (attempt > 0) {
+      await sleepAbortable(DNS_RETRY_BACKOFF_MS, signal);
+      if (signal?.aborted) break;
+    }
+    try {
+      const lookupOptions: LookupAllOptions & { signal?: AbortSignal } = {
+        all: true,
+        verbatim: true,
+        signal,
+      };
+      const addresses = await dnsLookup(hostname, lookupOptions);
+      if (!addresses.length) {
+        return { ok: false, reason: `Failed to resolve host: no addresses for '${hostname}'`, ip: "", family: 0 };
+      }
+      for (const entry of addresses) {
+        if (!isPublicIp(entry.address)) {
+          return { ok: false, reason: `Blocked: refusing to fetch the non-public address ${entry.address}.`, ip: "", family: 0 };
+        }
+      }
+      addresses.sort((a, b) => (a.family === 4 ? 0 : 1) - (b.family === 4 ? 0 : 1));
+      const first = addresses[0];
+      return {
+        ok: true,
+        reason: "",
+        ip: first.address,
+        family: first.family,
+        alternates: addresses.slice(1).map((entry) => ({ ip: entry.address, family: entry.family })),
+      };
+    } catch (err) {
+      lastError = err;
+      if (!isTransientDnsError(err)) break;
     }
   }
-  addresses.sort((a, b) => (a.family === 4 ? 0 : 1) - (b.family === 4 ? 0 : 1));
-  const first = addresses[0];
   return {
-    ok: true,
-    reason: "",
-    ip: first.address,
-    family: first.family,
-    alternates: addresses.slice(1).map((entry) => ({ ip: entry.address, family: entry.family })),
+    ok: false,
+    reason: lastError === null ? "Failed to resolve host: cancelled" : `Failed to resolve host: ${lastError}`,
+    ip: "",
+    family: 0,
   };
 }
 function budgetExceededResult(