pi-unsloth-webtools 0.9.1 → 0.10.0

package/README.md

@@ -38,13 +38,15 @@ Both tools display their target in the TUI tool row: `web_search "query"` and `w
 
 Mirrors Unsloth Studio's `web_search` tool:
 
-- Searches exactly like Studio's pinned `ddgs==9.14.4` `DDGS.text()`: the same seven engines
-  (duckduckgo, brave, google, mojeek, yahoo, yandex, wikipedia; bing is disabled upstream),
-  the same provider deduplication, href-dedupe aggregator with frequency ordering (hrefs are
+- Searches like Studio's pinned `ddgs==9.14.4` `DDGS.text()`: the Studio engines that still work
+  here (duckduckgo, yandex; the others are behind bot walls, see
+  [Known differences from Studio](#known-differences-from-studio)) plus startpage,
+  the same provider deduplication, and an href-dedupe aggregator (hrefs are
   canonicalized first — `utm_*`/tracking parameters and fragments are dropped and the URL is
   re-serialized, collapsing host-case, default-port, and trailing-slash variants — so the
-  same page found via different tracking links collapses), and the same `SimpleFilterRanker`
-  re-ranking. Formats results identically: `Title:` / `URL:` /
+  same page found via different tracking links collapses). Results are fused with weighted
+  reciprocal-rank fusion: each engine's own ordering, not a keyword guess, decides relevance,
+  and the fused list is capped per registrable domain. Formats results identically:
   `Snippet:` blocks separated by `---`, ending with the hint to call `web_fetch` to
   read a full page.
 - Rate-limit, timeout, and empty-result messages mirror Studio's `_search_failure_message`.
@@ -53,6 +55,11 @@ Mirrors Unsloth Studio's `web_search` tool:
   skipped); timeouts and cancellations are never retried.
 - Sweeps stop as soon as enough results are gathered: engines still in flight are aborted
   instead of being allowed to run to their timeout.
+- Engine requests go through the browser-fingerprint transport first (`webFetch.transport`,
+  default `tls-first`) and fall back to the plain Node transport; a refusal on one transport is
+  retried on the other. Redirects are followed manually, so every hop is re-checked against the
+  website policy and private-address literals are refused. When a SOCKS5 proxy is configured the
+  sweep stays on the plain transport, keeping agent and Tor routing intact.
 
 ### web_fetch
 
@@ -218,11 +225,31 @@ third-party rendering service.
   any numeric-only line at a fixed edge position where page numbers appear on at least
   half the pages (so a one-off number sharing that position is dropped too, while fused
   labels like `Page 3 of 12` survive). Studio and pymupdf4llm return them verbatim.
-- Search engines: Node's `fetch` TLS fingerprint differs from ddgs's `primp`
-  impersonation, so Google/Brave/Yahoo/Yandex may block or serve consent pages more
-  aggressively (a blocked engine simply contributes no results). User agents are a
-  fixed browser set plus ddgs's Android Google UA generator, not `fake_useragent`'s
-  database.
+- Search engines: the sweep speaks a browser's TLS/HTTP2 shape through `wreq-js` first (the same
+  transport `web_fetch` uses), so engine fingerprints match Chrome rather than Node's `fetch`;
+  the plain Node transport is the fallback. User agents on that fallback are a fixed browser set,
+  not `fake_useragent`'s database.
+- Startpage: an extra engine beyond Studio's set, matching newer ddgs (its Google-backed index
+  means it fills the `google` provider slot). It answers a plain `GET /sp/search?query=` and
+  serves an Anubis proof-of-work challenge to clients that do not look like a browser, so it works
+  through the browser-fingerprint transport and the fallback's browser headers, but not a bare
+  Node `fetch`. Startpage's POST endpoint and its safesearch parameter are challenge-gated and are
+  not used.
+- Unused engines: mojeek, yahoo, google, brave and wikipedia are not part of the sweep. A
+  non-JavaScript client cannot get past mojeek (JavaScript challenge) or yahoo (`_bv` bot beacon),
+  google and brave answer rate limits and bot interstitials instead of results (HTTP 429 in
+  testing), and wikipedia is an API for a source the other engines already return — so the sweep
+  no longer spends a slot on it, and `wikipedia.org` results are ranked like any other instead of
+  being forced to the top. The remaining three engines can be narrowed further per machine with
+  `webSearch.engines` (see Configuration).
+- Ranking: Studio's `SimpleFilterRanker` (keyword buckets with `wikipedia.org` pinned first) is
+  replaced by weighted reciprocal-rank fusion over each engine's own ordering. On the
+  `npm run engine:eval` query set that lifts mean precision@5 from 0.28 (keyword buckets) to 0.35 —
+  the fusion keeps the ranking signal the engines already computed instead of guessing from query
+  substrings. Weights default to uniform because the three engines overlap so little (mean Jaccard
+  0.07–0.22) that weighting mostly decides which engine dominates the list rather than which result
+  is better. A per-domain cap exists (`webSearch.maxPerHost`) but is off by default: it measurably
+  costs precision, and the uncapped fused top-5 already spans 4.3 of 5 distinct domains.
 - Empty sweeps: ddgs 9.14.4 raises the last engine exception; this port reports a
   timeout whenever any engine timed out, so the timeout message is not masked by later
   generic engine failures. The timeout budget bounds the entire sweep: per-engine
@@ -231,16 +258,16 @@ third-party rendering service.
 - Proxies: Studio routes through environment proxies; this port resolves and pins the target IP and
   tunnels that connection through `HTTPS_PROXY` / `HTTP_PROXY` / `ALL_PROXY` when the proxy is a
   SOCKS5 proxy (`NO_PROXY` exclusions respected; DNS stays local for the guard). Other proxy
-  schemes fall back to a direct connection. The search path uses the process-wide `fetch`, so an
-  agent-level proxy dispatcher applies there too — see
-  [Companion: rotating exit IPs](#companion-rotating-exit-ips).
+  schemes fall back to a direct connection. The search sweep stays on the process-wide `fetch`
+  whenever a SOCKS5 proxy is configured, so an agent-level proxy dispatcher still applies there —
+  see [Companion: rotating exit IPs](#companion-rotating-exit-ips).
 - Dedup and titles: the aggregator keys on canonicalized hrefs (`utm_*`/tracking parameters
   and fragments stripped, then the URL re-serialized); fetched HTML pages are prefixed with
   the document `<title>`. Studio keys on raw hrefs and returns the converted body alone.
 - Upstream drift: current ddgs ships ten backends (adding bing, startpage, grokipedia),
   requires a `vqd` token for DuckDuckGo, and exposes an `extract()` mode. This port
-  deliberately pins the Studio snapshot — seven engines, bing disabled upstream, no vqd,
-  no pagination — so engine behavior matches Studio rather than ddgs head.
+  uses duckduckgo and yandex from the Studio snapshot plus startpage — bing stays
+  disabled, no vqd, no pagination — so engine behavior matches Studio rather than ddgs head.
 
 ## When to use alternatives
 
@@ -266,8 +293,9 @@ choose the best tool per URL. No need to fork this package to add those features
 
 [`pi-tor-proxy`](https://github.com/YuGiMob/pi-tor-proxy) routes pi's in-process `fetch` traffic
 through Tor (it downloads and manages its own Tor binary) and gives each pi instance its own
-circuit and exit IP. The search sweep uses the process-wide `fetch`, so it leaves through the
-current Tor exit, and many search engines rate-limit or challenge per outgoing IP —
+circuit and exit IP. With a SOCKS5 proxy configured, the search sweep stays on the process-wide
+`fetch`, so it leaves through the current Tor exit, and many search engines rate-limit or
+challenge per outgoing IP —
 `/tor-cycle` swaps the exit those limits are counted against, while `/tor-country` and
 `/tor-exclude` constrain which exits are used.
 
@@ -310,7 +338,10 @@ Optional settings in `~/.pi/agent/settings.json` or `.pi/settings.json` (project
 | `websitePolicy` | none | Not read from settings. Tools run unrestricted by default; `websitePolicy` is a programmatic option the host passes to `webSearch` / `fetchPageText` |
 | `unslothWebTools.allowPrivateAddresses` / `webFetch.allowPrivateAddresses` | `true` | Opt out to restore the resolved-IP SSRF guard: private/loopback/link-local hosts (localhost, LAN IPs) are refused again. Non-canonical numeric IP encodings stay blocked either way |
 | `unslothWebTools.allowLocalFiles` / `webFetch.allowLocalFiles` | `true` | Opt out to refuse local files in `web_fetch` (`file://` URLs, absolute, `~/`, or `./` paths); when enabled, PDFs are extracted and HTML converted |
-| `webFetch.transport` / `unslothWebTools.transport` | `tls-first` | Fetch transport order: `tls-first` (default), `direct-first`, or `off` to disable the browser-fingerprint transport entirely |
+| `webFetch.transport` / `unslothWebTools.transport` | `tls-first` | Transport order for `web_fetch` and `web_search` engine requests: `tls-first` (default), `direct-first`, or `off` to disable the browser-fingerprint transport entirely |
+| `webSearch.engines` / `unslothWebTools.engines` | all three (duckduckgo, yandex, startpage) | Restrict `web_search` to a subset of engine names, e.g. `["duckduckgo", "yandex"]`; unknown names are ignored, and a list that matches nothing falls back to every engine |
+| `webSearch.engineWeights` / `unslothWebTools.engineWeights` | all `1` | Per-engine fusion weight, e.g. `{"startpage": 2, "yandex": 0.5}`; only positive numbers are read |
+| `webSearch.maxPerHost` / `unslothWebTools.maxPerHost` | `0` (no cap) | Most results one registrable domain may contribute; `0` disables the cap |
 | `webRender.lightpandaEnabled` / `unslothWebTools.lightpandaEnabled` | `true` | Opt out to disable local Lightpanda rendering |
 | `webRender.lightpandaPath` / `unslothWebTools.lightpandaPath` | launcher installed by `scripts/install-lightpanda.sh`, else `lightpanda` on `PATH` | Path to the Lightpanda binary used for local rendering |
 | `webRender.lightpandaCommand` / `unslothWebTools.lightpandaCommand` | none | Command prefix that launches the renderer, for WSL (`["wsl.exe","-e","<path>"]`) or containers; overrides `lightpandaPath`. The fetch flags are appended to it |
@@ -358,6 +389,7 @@ npm run test:unit
 npm run test:smoke
 bash scripts/install-lightpanda.sh
 npm run camoufox:warmup
+npm run engine:eval
 ```
 
 `npm run camoufox:warmup` measures what a warm Camoufox costs and buys: launch time, idle CPU and
@@ -366,6 +398,11 @@ RSS, per-fetch latency with the browser already running, and whether a persisten
 `--virtual` for a virtual display, `--pin=0` to let Camoufox rotate fingerprints, `--seconds=N`,
 `--idle=N`.
 
+`npm run engine:eval` measures each search engine against a small hand-labelled developer query
+set: yield, latency and failures, pairwise URL overlap, unique contribution, and offline fusion
+simulations with candidate weight vectors and per-host caps. `--cache=FILE` reuses a previous run
+without touching the network; `--delay`, `--timeout`, `--max` and `--transport` tune the sweep.
+
 ## Tests
 
 `npm test` runs the full suite. `npm run test:unit` skips the live-network smoke tests,
@@ -386,7 +423,7 @@ The suite ports Unsloth Studio's own tests for these tools:
 - `test/fetch-flow.test.ts`: GitHub README rewrite, deadline/cancellation, HTML sniffing
   (from `test_web_fetch_extraction.py`; the fetch client is injected via seams)
 - `test/engines.test.ts`: the ddgs engine port, normalizers, the XPath subset, the
-  aggregator, the ranker, and the Wikipedia engine with a stubbed fetch
+  reciprocal-rank aggregator, the per-host cap, and the Startpage engine with a stubbed fetch
 - `test/pdf-parity.test.ts`: MuPDF engine capabilities, PDF 1.5 object streams,
   ASCII85Decode, font `/Differences` encodings, pymupdf4llm-style headings/links/tables
 - `test/entities.test.ts`: `decodeHtmlEntities` parity with CPython `html.unescape`,

package/engines.ts

@@ -1,11 +1,12 @@
-import { randomBytes } from "node:crypto";
 import { appendFile, chmod, mkdir } from "node:fs/promises";
 import { dirname, isAbsolute, join } from "node:path";
 import { collapseWhitespace, decodeHtmlEntities, feedHtml } from "./html-to-md.ts";
 import type { AttrDict } from "./html-to-md.ts";
 import { randomUserAgent } from "./user-agents.ts";
 import { agentDir } from "./agent-dir.ts";
-import { MAX_SIGNAL_TIMEOUT_MS } from "./web-access.ts";
+import { impersonatedRequest, type FetchTransport, type TlsHopOptions, type TlsHopResponse } from "./tls-fetch.ts";
+import { checkUrlAccess, isPublicIp, MAX_SIGNAL_TIMEOUT_MS, type WebsitePolicy } from "./web-access.ts";
+import { socksProxyForUrl } from "./proxy.ts";
 export class EmptySweepError extends Error {
   constructor() {
     super("No results found");
@@ -247,6 +248,13 @@ function parsePredExpr(input: string): Pred {
       const name = word();
       return { op: "desc", tag: name };
     }
+    if (input.startsWith("./", pos)) {
+      pos += 2;
+      const name = word();
+      const { preds, next } = parsePredicateBlocks(input, pos);
+      pos = next;
+      return { op: "child", tag: name, preds };
+    }
     const name = word();
     const { preds, next } = parsePredicateBlocks(input, pos);
     pos = next;
@@ -471,42 +479,18 @@ export function extractResults(
   return results;
 }
 
-function googleUserAgent(): string {
-  const devices: [string, string, number, number][] = [
-    ["5.0", "SM-G900P Build/LRX21T", 39, 60],
-    ["6.0", "Nexus 5 Build/MRA58N", 39, 60],
-    ["8.0", "Pixel 2 Build/OPD3.170816.012", 39, 60],
-  ];
-  const [androidVer, device, chromeMin, chromeMax] = devices[Math.floor(Math.random() * devices.length)];
-  const chromeMajor = chromeMin + Math.floor(Math.random() * (chromeMax - chromeMin + 1));
-  const chromeBuild = 1000 + Math.floor(Math.random() * 9000);
-  const chromePatch = 1000 + Math.floor(Math.random() * 1000);
-  return (
-    `Mozilla/5.0 (Linux; Android ${androidVer}; ${device}) ` +
-    `AppleWebKit/537.36 (KHTML, like Gecko) ` +
-    `Chrome/${chromeMajor}.0.${chromeBuild}.${chromePatch} Mobile Safari/537.36`
-  );
-}
-
-function tokenUrlSafe(byteLength: number): string {
-  return randomBytes(byteLength).toString("base64url");
-}
-
-function unquotePlus(value: string): string {
-  try {
-    return decodeURIComponent(value.replace(/\+/g, "%20"));
-  } catch {
-    return value.replace(/\+/g, " ");
-  }
-}
+export type EngineImpersonation = (options: TlsHopOptions) => Promise<TlsHopResponse | null>;
 
-function yahooExtractUrl(raw: string): string {
-  const afterRu = raw.split("/RU=", 2)[1] ?? "";
-  const t = afterRu.split("/RK=", 1)[0].split("/RS=", 1)[0];
-  return unquotePlus(t);
+export interface SearchEngineOptions {
+  transport?: FetchTransport;
+  policy?: WebsitePolicy | null;
+  impersonate?: EngineImpersonation;
+  engines?: string[];
+  engineWeights?: Record<string, number>;
+  maxPerHost?: number;
 }
 
-interface EngineContext {
+export interface EngineContext extends SearchEngineOptions {
   region: string;
   safesearch: string;
 }
@@ -514,7 +498,6 @@ interface EngineContext {
 export interface Engine {
   name: string;
   provider: string;
-  priority?: number;
   search(
     query: string,
     ctx: EngineContext,
@@ -528,6 +511,7 @@ interface HttpRequestOptions {
   cookies?: Record<string, string>;
   timeoutMs: number;
   signal?: AbortSignal;
+  ctx?: EngineContext;
 }
 
 interface HttpOptions extends HttpRequestOptions {
@@ -555,6 +539,8 @@ async function httpPost(
 
 const MAX_ENGINE_RESPONSE_BYTES = 5 * 1024 * 1024;
 const ENGINE_RETRY_BACKOFF_MS = 250;
+const MAX_ENGINE_HOPS = 5;
+const DEFAULT_ENGINE_TRANSPORT: FetchTransport = "off";
 
 const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
 
@@ -578,10 +564,84 @@ async function readBodyCapped(response: Response): Promise<string | null> {
   return new TextDecoder("utf-8").decode(Buffer.concat(chunks));
 }
 
-function mapFetchError(err: unknown): never {
-  if (err instanceof DOMException && err.name === "TimeoutError") throw new SearchTimeoutError();
-  if (err instanceof DOMException && err.name === "AbortError") throw new SearchCancelled();
-  throw err;
+interface EngineHopResponse {
+  status: number;
+  location: string | null;
+  body: string | null;
+}
+
+type EngineTransportKind = "direct" | "tls";
+
+function engineTransportOrder(transport: FetchTransport, target: URL): EngineTransportKind[] {
+  if (transport === "off" || socksProxyForUrl(target) !== null) return ["direct"];
+  return transport === "direct-first" ? ["direct", "tls"] : ["tls", "direct"];
+}
+
+function engineTargetAllowed(target: string, policy: WebsitePolicy | null): boolean {
+  const [allowed, , hostname] = checkUrlAccess(target, policy);
+  if (!allowed) return false;
+  const isLiteral = hostname.includes(":") || /^\d+\.\d+\.\d+\.\d+$/.test(hostname);
+  return !isLiteral || isPublicIp(hostname);
+}
+
+function classifyRequestError(
+  err: unknown,
+  caller: AbortSignal | undefined,
+  hopSignal: AbortSignal,
+): Error | null {
+  if (caller?.aborted) return new SearchCancelled();
+  if (hopSignal.aborted) return new SearchTimeoutError();
+  if (err instanceof DOMException && err.name === "TimeoutError") return new SearchTimeoutError();
+  if (err instanceof DOMException && err.name === "AbortError") return new SearchTimeoutError();
+  if (err instanceof Error && err.message === "timed out") return new SearchTimeoutError();
+  if (err instanceof Error && err.message === "cancelled") return new SearchCancelled();
+  return null;
+}
+
+async function directEngineHop(
+  target: URL,
+  headers: Record<string, string>,
+  method: string,
+  body: string | undefined,
+  signal: AbortSignal,
+): Promise<EngineHopResponse | null> {
+  const response = await fetch(target.toString(), { method, headers, body, signal, redirect: "manual" });
+  if (response.status === 200) {
+    const text = await readBodyCapped(response);
+    return text === null ? null : { status: 200, location: null, body: text };
+  }
+  try {
+    await response.body?.cancel();
+  } catch {}
+  return { status: response.status, location: response.headers.get("location"), body: null };
+}
+
+function tlsEngineHop(
+  impersonate: EngineImpersonation,
+  target: URL,
+  headers: Record<string, string>,
+  method: string,
+  body: string | undefined,
+  caller: AbortSignal | undefined,
+  timeoutMs: number,
+): Promise<EngineHopResponse | null> {
+  return impersonate({
+    url: target,
+    timeoutMs,
+    signal: caller,
+    maxBytes: MAX_ENGINE_RESPONSE_BYTES,
+    maxPdfBytes: MAX_ENGINE_RESPONSE_BYTES,
+    method,
+    body,
+    extraHeaders: headers,
+  }).then((response) => {
+    if (response === null || response.truncated) return null;
+    return {
+      status: response.status,
+      location: response.headers.location ?? null,
+      body: response.status === 200 ? new TextDecoder("utf-8").decode(response.body) : null,
+    };
+  });
 }
 
 async function httpFetch(
@@ -601,32 +661,67 @@ async function httpFetch(
     : null;
   if (cookie) headers["Cookie"] = cookie;
   const timeoutMs = Math.min(MAX_SIGNAL_TIMEOUT_MS, Math.max(1, options.timeoutMs));
-  const signals: AbortSignal[] = [AbortSignal.timeout(timeoutMs)];
-  if (options.signal) signals.push(options.signal);
-  let response: Response;
-  try {
-    response = await fetch(url, {
-      method: options.method ?? "GET",
-      headers,
-      body: options.method === "POST" ? options.body : undefined,
-      signal: AbortSignal.any(signals),
-    });
-  } catch (err) {
-    throw mapFetchError(err);
-  }
-  if (response.status !== 200) {
-    try {
-      await response.body?.cancel();
-    } catch {
+  const deadline = Date.now() + timeoutMs;
+  const caller = options.signal;
+  const transport = options.ctx?.transport ?? DEFAULT_ENGINE_TRANSPORT;
+  const policy = options.ctx?.policy ?? null;
+  const impersonate = options.ctx?.impersonate ?? (transport === "off" ? null : impersonatedRequest);
+  let method = options.method ?? "GET";
+  let body = method === "POST" ? options.body : undefined;
+  let target = url;
+  for (let hop = 0; hop < MAX_ENGINE_HOPS; hop++) {
+    if (caller?.aborted) throw new SearchCancelled();
+    const remaining = deadline - Date.now();
+    if (remaining <= 0) throw new SearchTimeoutError();
+    const targetUrl = new URL(target);
+    const hopSignal = caller
+      ? AbortSignal.any([caller, AbortSignal.timeout(remaining)])
+      : AbortSignal.timeout(remaining);
+    let response: EngineHopResponse | null = null;
+    let failure: unknown = null;
+    for (const kind of engineTransportOrder(transport, targetUrl)) {
+      try {
+        let attempt: EngineHopResponse | null = null;
+        if (kind === "tls") {
+          if (impersonate !== null) {
+            attempt = await tlsEngineHop(impersonate, targetUrl, headers, method, body, caller, remaining);
+          }
+        } else {
+          attempt = await directEngineHop(targetUrl, headers, method, body, hopSignal);
+        }
+        if (attempt === null) continue;
+        if (response === null || (attempt.status < 400 && response.status >= 400)) response = attempt;
+        if (attempt.status < 400) break;
+      } catch (err) {
+        const mapped = classifyRequestError(err, caller, hopSignal);
+        if (mapped) throw mapped;
+        failure = err;
+      }
+    }
+    if (response === null) {
+      if (failure) throw failure;
       return null;
     }
-    return null;
-  }
-  try {
-    return await readBodyCapped(response);
-  } catch (err) {
-    throw mapFetchError(err);
+    if (response.status >= 300 && response.status < 400) {
+      if (!response.location) return null;
+      let next: string;
+      try {
+        next = new URL(response.location, target).toString();
+      } catch {
+        return null;
+      }
+      if (!engineTargetAllowed(next, policy)) return null;
+      if (response.status !== 307 && response.status !== 308) {
+        method = "GET";
+        body = undefined;
+      }
+      target = next;
+      continue;
+    }
+    if (response.status !== 200) return null;
+    return response.body ?? "";
   }
+  return null;
 }
 
 const DUCKDUCKGO: Engine = {
@@ -636,7 +731,7 @@ const DUCKDUCKGO: Engine = {
     const html = await httpPost(
       "https://html.duckduckgo.com/html/",
       { q: query, b: "", l: ctx.region },
-      { headers: { "User-Agent": randomUserAgent() }, timeoutMs, signal },
+      { headers: { "User-Agent": randomUserAgent() }, timeoutMs, signal, ctx },
     );
     if (!html) return null;
     const results = extractResults(html, "//div[contains(@class, 'body')]", {
@@ -648,126 +743,15 @@ const DUCKDUCKGO: Engine = {
   },
 };
 
-const BRAVE: Engine = {
-  name: "brave",
-  provider: "brave",
-  async search(query, ctx, timeoutMs, signal) {
-    const country = ctx.region.toLowerCase().split("-")[0];
-    const cookies: Record<string, string> = { [country]: country, useLocation: "0" };
-    if (ctx.safesearch !== "moderate") {
-      cookies["safesearch"] = ctx.safesearch === "on" ? "strict" : "off";
-    }
-    const html = await httpGet(
-      "https://search.brave.com/search",
-      { q: query, source: "web" },
-      { cookies, timeoutMs, signal },
-    );
-    if (!html) return null;
-    return extractResults(html, "//div[@data-type='web']", {
-      title:
-        ".//div[(contains(@class,'title') or contains(@class,'sitename-container')) and position()=last()]//text()",
-      href: ".//a[div[contains(@class, 'title')]]/@href",
-      body: ".//div[contains(@class, 'snippet')]//div[contains(@class, 'content')]//text()",
-    });
-  },
-};
-
-const GOOGLE: Engine = {
-  name: "google",
-  provider: "google",
-  async search(query, ctx, timeoutMs, signal) {
-    const [country, lang] = ctx.region.split("-");
-    const safesearchBase: Record<string, string> = { on: "2", moderate: "1", off: "0" };
-    const html = await httpGet(
-      "https://www.google.com/search",
-      {
-        q: query,
-        filter: safesearchBase[ctx.safesearch.toLowerCase()] ?? "1",
-        start: "0",
-        hl: `${lang}-${country.toUpperCase()}`,
-        lr: `lang_${lang}`,
-        cr: `country${country.toUpperCase()}`,
-      },
-      {
-        headers: { "User-Agent": googleUserAgent() },
-        cookies: { CONSENT: "YES+" },
-        timeoutMs,
-        signal,
-      },
-    );
-    if (!html) return null;
-    const results = extractResults(html, "//div[@data-hveid][.//h3]", {
-      title: ".//h3//text()",
-      href: ".//a[.//h3]/@href",
-      body: "./div/div[last()]//text()",
-    });
-    return results
-      .map((r) => {
-        if (r.href.startsWith("/url?q=")) {
-          r.href = r.href.split("?q=")[1].split("&")[0];
-        }
-        return r;
-      })
-      .filter((r) => r.title && r.href.startsWith("http"));
-  },
-};
-
-const MOJEEK: Engine = {
-  name: "mojeek",
-  provider: "mojeek",
-  async search(query, ctx, timeoutMs, signal) {
-    const [country, lang] = ctx.region.toLowerCase().split("-");
-    const params: Record<string, string> = { q: query };
-    if (ctx.safesearch === "on") params["safe"] = "1";
-    const html = await httpGet(
-      "https://www.mojeek.com/search",
-      params,
-      { cookies: { arc: country, lb: lang }, timeoutMs, signal },
-    );
-    if (!html) return null;
-    return extractResults(html, "//ul[contains(@class, 'results')]/li", {
-      title: ".//h2//text()",
-      href: ".//h2/a/@href",
-      body: ".//p[@class='s']//text()",
-    });
-  },
-};
-
-const YAHOO: Engine = {
-  name: "yahoo",
-  provider: "bing",
-  async search(query, _ctx, timeoutMs, signal) {
-    const ylt = tokenUrlSafe(18);
-    const ylu = tokenUrlSafe(35);
-    const html = await httpGet(
-      `https://search.yahoo.com/search;_ylt=${ylt};_ylu=${ylu}`,
-      { p: query },
-      { timeoutMs, signal },
-    );
-    if (!html) return null;
-    const results = extractResults(html, "//div[contains(@class, 'relsrch')]", {
-      title: ".//div[contains(@class, 'Title')]//h3//text()",
-      href: ".//div[contains(@class, 'Title')]//a/@href",
-      body: ".//div[contains(@class, 'Text')]//text()",
-    });
-    return results
-      .filter((r) => !r.href.startsWith("https://www.bing.com/aclick?"))
-      .map((r) => {
-        if (r.href.includes("/RU=")) r.href = yahooExtractUrl(r.href);
-        return r;
-      });
-  },
-};
-
 const YANDEX: Engine = {
   name: "yandex",
   provider: "yandex",
-  async search(query, _ctx, timeoutMs, signal) {
+  async search(query, ctx, timeoutMs, signal) {
     const searchid = 1000000 + Math.floor(Math.random() * 9000000);
     const html = await httpGet(
       "https://yandex.com/search/site/",
       { text: query, web: "1", searchid: String(searchid) },
-      { timeoutMs, signal },
+      { timeoutMs, signal, ctx },
     );
     if (!html) return null;
     return extractResults(html, "//li[contains(@class, 'serp-item')]", {
@@ -778,60 +762,36 @@ const YANDEX: Engine = {
   },
 };
 
-const WIKIPEDIA: Engine = {
-  name: "wikipedia",
-  provider: "wikipedia",
-  priority: 2,
+const START_PAGE: Engine = {
+  name: "startpage",
+  provider: "google",
   async search(query, ctx, timeoutMs, signal) {
-    const started = Date.now();
-    const lang = ctx.region.toLowerCase().split("-")[1] ?? "en";
-    const encoded = encodeURIComponent(query);
-    const opensearchUrl =
-      `https://${lang}.wikipedia.org/w/api.php?action=opensearch&profile=fuzzy&limit=1&search=${encoded}`;
-    const opensearch = await httpGet(opensearchUrl, {}, { timeoutMs, signal });
-    if (!opensearch) return null;
-    let data: unknown;
-    try {
-      data = JSON.parse(opensearch);
-    } catch {
-      return null;
-    }
-    const payload = data as [string, string[], string[], string[]];
-    if (!payload[1] || !payload[1].length) return [];
-    const title = payload[1][0];
-    const href = payload[3][0];
-    let body = "";
-    const extractUrl =
-      `https://${lang}.wikipedia.org/w/api.php?action=query&format=json&prop=extracts` +
-      `&titles=${encodeURIComponent(title)}&explaintext=0&exintro=0&redirects=1`;
-    const extract = await httpGet(extractUrl, {}, { timeoutMs: Math.max(1, timeoutMs - (Date.now() - started)), signal });
-    if (extract) {
-      try {
-        const pageData = JSON.parse(extract) as {
-          query: { pages: Record<string, { extract?: string }> };
-        };
-        const pages = Object.values(pageData.query.pages);
-        if (pages.length) body = pages[0].extract ?? "";
-      } catch {
-        body = "";
-      }
-    }
-    if (body.includes("may refer to:")) return [];
-    return [{ title: normalizeText(title), href: normalizeUrl(href), body: normalizeText(body) }];
+    const [country, lang] = ctx.region.toLowerCase().split("-");
+    const html = await httpGet(
+      "https://www.startpage.com/sp/search",
+      { query, qsr: `${lang}_${country.toUpperCase()}` },
+      { headers: { Referer: "https://www.startpage.com/" }, timeoutMs, signal, ctx },
+    );
+    if (!html) return null;
+    return extractResults(html, "//div[contains(@class, 'result')][./a]", {
+      title: ".//h2//text()",
+      href: "./a/@href",
+      body: ".//p//text()",
+    });
   },
 };
 
-export const TEXT_ENGINES: Engine[] = [DUCKDUCKGO, BRAVE, GOOGLE, MOJEEK, YAHOO, YANDEX, WIKIPEDIA];
+export const TEXT_ENGINES: Engine[] = [DUCKDUCKGO, YANDEX, START_PAGE];
 
 export class ResultsAggregator {
   private cache = new Map<string, SearchResult>();
-  private counter = new Map<string, number>();
+  private scores = new Map<string, number>();
 
   get size(): number {
     return this.cache.size;
   }
 
-  append(item: SearchResult): void {
+  append(item: SearchResult, weight = 1, rank = 1): void {
     if (typeof item.href !== "string" || !item.href.trim()) return;
     const key = canonicalizeHref(item.href);
     if (!key) return;
@@ -839,54 +799,66 @@ export class ResultsAggregator {
     if (!existing || item.body.length > existing.body.length) {
       this.cache.set(key, { ...item, href: key });
     }
-    this.counter.set(key, (this.counter.get(key) ?? 0) + 1);
+    this.scores.set(key, (this.scores.get(key) ?? 0) + weight / (RRF_RANK_CONSTANT + rank));
   }
 
-  extend(items: SearchResult[]): void {
-    for (const item of items) this.append(item);
+  extend(items: SearchResult[], weight = 1): void {
+    const seen = new Set<string>();
+    items.forEach((item, index) => {
+      const key = canonicalizeHref(item.href);
+      if (!key) return;
+      const first = !seen.has(key);
+      seen.add(key);
+      this.append(item, first ? weight : 0, index + 1);
+    });
   }
 
-  extractDicts(): SearchResult[] {
-    return [...this.counter.entries()]
-      .sort((a, b) => b[1] - a[1])
-      .map(([key]) => this.cache.get(key)!);
+  ranked(): SearchResult[] {
+    return [...this.cache.entries()]
+      .filter(([, doc]) => !isWikimediaCategory(doc))
+      .map(([key, doc]) => ({ doc, score: this.scores.get(key) ?? 0 }))
+      .sort((a, b) => b.score - a.score || a.doc.href.localeCompare(b.doc.href))
+      .map((entry) => entry.doc);
   }
 }
 
-function extractTokens(query: string): Set<string> {
-  return new Set(query.toLowerCase().split(/\W+/u).filter((t) => t.length >= 3));
-}
+const RRF_RANK_CONSTANT = 60;
+const DEFAULT_MAX_PER_HOST = 0;
+const MULTI_PART_SUFFIXES = new Set(["co.uk", "org.uk", "com.au", "co.jp", "co.nz", "com.br", "co.in"]);
 
-function hasAnyToken(text: string, tokens: Set<string>): boolean {
-  const lower = text.toLowerCase();
-  for (const token of tokens) {
-    if (lower.includes(token)) return true;
+export function registrableDomain(url: string): string {
+  let hostname = "";
+  try {
+    hostname = new URL(url).hostname.toLowerCase().replace(/^www\./, "");
+  } catch {
+    return "";
   }
-  return false;
+  const parts = hostname.split(".");
+  if (parts.length < 2) return hostname;
+  const lastTwo = parts.slice(-2).join(".");
+  return MULTI_PART_SUFFIXES.has(lastTwo) && parts.length >= 3 ? parts.slice(-3).join(".") : lastTwo;
 }
 
-export function rankResults(docs: SearchResult[], query: string): SearchResult[] {
-  const tokens = extractTokens(query);
-  const wiki: SearchResult[] = [];
-  const both: SearchResult[] = [];
-  const titleOnly: SearchResult[] = [];
-  const bodyOnly: SearchResult[] = [];
-  const neither: SearchResult[] = [];
+export function capByHost(docs: SearchResult[], limit: number, maxPerHost: number): SearchResult[] {
+  if (limit <= 0) return [];
+  const perHost = new Map<string, number>();
+  const capped: SearchResult[] = [];
+  const allowed = maxPerHost > 0 ? maxPerHost : Number.POSITIVE_INFINITY;
   for (const doc of docs) {
-    if (doc.title.includes("Category:") && doc.title.includes("Wikimedia")) continue;
-    if (doc.href.includes("wikipedia.org")) {
-      wiki.push(doc);
-      continue;
-    }
-    const hitTitle = hasAnyToken(doc.title, tokens);
-    const hitBody = hasAnyToken(doc.body, tokens);
-    if (hitTitle && hitBody) both.push(doc);
-    else if (hitTitle) titleOnly.push(doc);
-    else if (hitBody) bodyOnly.push(doc);
-    else neither.push(doc);
+    const domain = registrableDomain(doc.href);
+    const used = perHost.get(domain) ?? 0;
+    if (used >= allowed) continue;
+    perHost.set(domain, used + 1);
+    capped.push(doc);
+    if (capped.length >= limit) break;
   }
-  return [...wiki, ...both, ...titleOnly, ...bodyOnly, ...neither];
+  return capped;
 }
+
+function isWikimediaCategory(doc: SearchResult): boolean {
+  return doc.title.includes("Category:") && doc.title.includes("Wikimedia");
+}
+
 async function recordSweepStats(query: string, maxResults: number, started: number, timedOutProviders: string[], resultCount: number): Promise<void> {
   const flag = process.env.PI_UNSLOTH_WEBTOOLS_STATS?.trim();
   if (!flag) return;
@@ -922,15 +894,20 @@ async function recordSweepStats(query: string, maxResults: number, started: numb
   } catch {}
 }
 
-function shuffledEngines(): Engine[] {
-  const shuffled = [...TEXT_ENGINES];
+function selectedEngines(names: string[] | undefined): Engine[] {
+  if (!names?.length) return TEXT_ENGINES;
+  const wanted = new Set(names.map((name) => name.trim().toLowerCase()));
+  const chosen = TEXT_ENGINES.filter((engine) => wanted.has(engine.name));
+  return chosen.length ? chosen : TEXT_ENGINES;
+}
+
+function shuffledEngines(engines: Engine[] = TEXT_ENGINES): Engine[] {
+  const shuffled = [...engines];
   for (let i = shuffled.length - 1; i > 0; i--) {
     const j = Math.floor(Math.random() * (i + 1));
     [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
   }
-  const wikipedia = shuffled.find((e) => e.priority === 2);
-  const rest = shuffled.filter((e) => e.priority !== 2);
-  return wikipedia ? [wikipedia, ...rest] : shuffled;
+  return shuffled;
 }
 
 export async function autoTextSearch(
@@ -938,13 +915,23 @@ export async function autoTextSearch(
   maxResults: number,
   timeoutMs: number,
   signal?: AbortSignal,
+  options: SearchEngineOptions = {},
 ): Promise<SearchResult[]> {
   const started = Date.now();
-  const engines = shuffledEngines();
+  const engines = shuffledEngines(selectedEngines(options.engines));
   const deadline = started + timeoutMs;
   const seenProviders = new Set<string>();
   const aggregator = new ResultsAggregator();
-  const ctx: EngineContext = { region: "us-en", safesearch: "moderate" };
+  const engineWeights = options.engineWeights ?? {};
+  const maxPerHost = options.maxPerHost ?? DEFAULT_MAX_PER_HOST;
+  const enough = () => capByHost(aggregator.ranked(), maxResults, maxPerHost).length >= maxResults;
+  const ctx: EngineContext = {
+    region: "us-en",
+    safesearch: "moderate",
+    transport: options.transport ?? DEFAULT_ENGINE_TRANSPORT,
+    policy: options.policy ?? null,
+    impersonate: options.impersonate,
+  };
   const controller = new AbortController();
   let onAbort: (() => void) | undefined;
   if (signal) {
@@ -998,18 +985,18 @@ export async function autoTextSearch(
       }
     }
     if (results && results.length) {
-      aggregator.extend(results);
+      aggregator.extend(results, engineWeights[engine.name] ?? 1);
       seenProviders.add(engine.provider);
-      if (aggregator.size >= maxResults) controller.abort();
+      if (enough()) controller.abort();
     }
   };
   while (i < engines.length || pending.size > 0) {
-    if (aggregator.size >= maxResults || cancelled) {
+    if (enough() || cancelled) {
       controller.abort();
       break;
     }
     while (i < engines.length && pending.size < maxWorkers) {
-      if (aggregator.size >= maxResults || cancelled) {
+      if (enough() || cancelled) {
         controller.abort();
         break;
       }
@@ -1027,7 +1014,7 @@ export async function autoTextSearch(
       );
     }
     if (pending.size === 0) break;
-    if (aggregator.size >= maxResults || cancelled) {
+    if (enough() || cancelled) {
       controller.abort();
       break;
     }
@@ -1036,10 +1023,10 @@ export async function autoTextSearch(
   await Promise.allSettled(pending);
   if (onAbort && signal) signal.removeEventListener("abort", onAbort);
   if (cancelled) throw new SearchCancelled();
-  const results = rankResults(aggregator.extractDicts(), query);
+  const results = capByHost(aggregator.ranked(), maxResults, maxPerHost);
   if (results.length) {
     void recordSweepStats(query, maxResults, started, [...timedOutProviders], results.length);
-    return results.slice(0, maxResults);
+    return results;
   }
   if (timedOutProviders.size) {
     const sorted = [...timedOutProviders].sort();

package/index.ts

@@ -10,7 +10,13 @@ import {
   type FetchPageOutcome,
 } from "./web-fetch.ts";
 import { renderPageWithLightpanda as defaultRenderLocalPageText } from "./lightpanda.ts";
-import { loadDefaultFetchSettings, loadDefaultFetchTimeoutMs, loadLightpandaSettings } from "./settings.ts";
+import {
+  loadDefaultEngines,
+  loadDefaultEngineWeights,
+  loadDefaultFetchSettings,
+  loadDefaultMaxPerHost,
+  loadLightpandaSettings,
+} from "./settings.ts";
 
 function toolCallLine(theme: Theme, name: string, detail: string) {
   const line = theme.fg("toolTitle", theme.bold(name)) + (detail ? ` ${theme.fg("accent", detail)}` : "");
@@ -179,12 +185,20 @@ export function createWebTools(deps: WebToolsDeps = {}) {
         onUpdate?.({ content: [{ type: "text", text: "Searching the web..." }], details: {} });
         const timeoutParam = positiveNumber(params.timeoutMs);
         const searchCwd = (_ctx as ExtensionContext | undefined)?.cwd;
-        const searchTimeoutMs = timeoutParam ?? (await loadDefaultFetchTimeoutMs(searchCwd)) ?? SEARCH_TIMEOUT_MS;
+        const searchSettings = await loadDefaultFetchSettings(searchCwd);
+        const searchEngines = await loadDefaultEngines(searchCwd);
+        const searchEngineWeights = await loadDefaultEngineWeights(searchCwd);
+        const searchMaxPerHost = await loadDefaultMaxPerHost(searchCwd);
+        const searchTimeoutMs = timeoutParam ?? searchSettings.timeoutMs ?? SEARCH_TIMEOUT_MS;
         const text = await webSearch(params.query, {
           signal: signal ?? undefined,
           timeoutMs: searchTimeoutMs,
           maxResults: positiveNumber(params.maxResults),
           cwd: searchCwd,
+          transport: searchSettings.transport,
+          engines: searchEngines,
+          engineWeights: searchEngineWeights,
+          maxPerHost: searchMaxPerHost,
         });
         return { content: [{ type: "text", text }], details: {} };
       },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.9.1",
+  "version": "0.10.0",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools that began as a port of the Unsloth Studio codebase and now diverge from it (multi-engine search, opt-in SSRF guard, HTML-to-Markdown extraction)",
   "main": "index.ts",
@@ -60,6 +60,7 @@
     "test:smoke": "vitest run test/smoke.test.ts",
     "test:lightpanda": "vitest run test/lightpanda-live.test.ts",
     "camoufox:warmup": "node scripts/camoufox-warmup.ts",
+    "engine:eval": "node scripts/engine-eval.ts",
     "typecheck": "tsc --noEmit",
     "check:package": "node -e \"const fs=require('fs');const pkg=require('./package.json');const listed=new Set(pkg.files);const bad=[];for(const f of pkg.files){if(!fs.existsSync(f))bad.push('missing file: '+f)}for(const f of fs.readdirSync('.').filter(f=>f.endsWith('.ts'))){if(!listed.has(f))bad.push('unlisted source: '+f)}if(bad.length){console.error(bad.join('\\n'));process.exit(1)}\"",
     "prepublishOnly": "npm run typecheck && npm run lint && npm run test:unit && npm run check:package && npm run check:publint",

package/settings.ts

@@ -50,6 +50,26 @@ function pickBoolean(data: Record<string, unknown>, paths: string[][]): boolean
   return undefined;
 }
 
+function pickNumberRecord(data: Record<string, unknown>, paths: string[][]): Record<string, number> | undefined {
+  for (const path of paths) {
+    let cur: unknown = data;
+    for (const key of path) {
+      if (cur && typeof cur === "object" && !Array.isArray(cur)) cur = (cur as Record<string, unknown>)[key];
+      else {
+        cur = undefined;
+        break;
+      }
+    }
+    if (!cur || typeof cur !== "object" || Array.isArray(cur)) continue;
+    const record: Record<string, number> = {};
+    for (const [key, value] of Object.entries(cur as Record<string, unknown>)) {
+      if (typeof value === "number" && Number.isFinite(value) && value > 0) record[key] = value;
+    }
+    if (Object.keys(record).length) return record;
+  }
+  return undefined;
+}
+
 function pickStringArray(data: Record<string, unknown>, paths: string[][]): string[] | undefined {
   for (const path of paths) {
     let cur: unknown = data;
@@ -133,6 +153,21 @@ const LIGHTPANDA_COMMAND_PATHS: string[][] = [
   ["unslothWebTools", "lightpandaCommand"],
   ["webRender", "lightpandaCommand"],
 ];
+
+const ENGINE_WEIGHTS_PATHS: string[][] = [
+  ["unslothWebTools", "engineWeights"],
+  ["webSearch", "engineWeights"],
+];
+
+const MAX_PER_HOST_PATHS: string[][] = [
+  ["unslothWebTools", "maxPerHost"],
+  ["webSearch", "maxPerHost"],
+];
+
+const ENGINE_PATHS: string[][] = [
+  ["unslothWebTools", "engines"],
+  ["webSearch", "engines"],
+];
 function clampMaxResults(value: number): number {
   return Math.min(20, Math.max(1, value));
 }
@@ -163,6 +198,28 @@ export async function loadDefaultMaxResults(cwd?: string): Promise<number> {
   return result;
 }
 
+export async function loadDefaultEngines(cwd?: string): Promise<string[] | undefined> {
+  let result: string[] | undefined;
+  for (const data of await settingsEntries(cwd)) {
+    const candidate = pickStringArray(data, ENGINE_PATHS);
+    if (candidate !== undefined) result = candidate;
+  }
+  return result;
+}
+
+export async function loadDefaultEngineWeights(cwd?: string): Promise<Record<string, number> | undefined> {
+  let result: Record<string, number> | undefined;
+  for (const data of await settingsEntries(cwd)) {
+    const candidate = pickNumberRecord(data, ENGINE_WEIGHTS_PATHS);
+    if (candidate !== undefined) result = candidate;
+  }
+  return result;
+}
+
+export async function loadDefaultMaxPerHost(cwd?: string): Promise<number | undefined> {
+  return loadFetchSetting(cwd, MAX_PER_HOST_PATHS, (n) => n >= 0);
+}
+
 async function loadFetchSetting(cwd: string | undefined, paths: string[][], valid: (n: number) => boolean): Promise<number | undefined> {
   let result: number | undefined;
   for (const data of await settingsEntries(cwd)) {

package/tls-fetch.ts

@@ -49,15 +49,19 @@ interface ImpersonationModule {
 
 export type ImpersonationLoader = () => Promise<ImpersonationModule | null>;
 
+export type FetchTransport = "tls-first" | "direct-first" | "off";
+
 export interface TlsHopOptions {
   url: URL;
-  pinnedIp: string;
-  family: number;
+  pinnedIp?: string;
+  family?: number;
   timeoutMs: number;
   signal?: AbortSignal;
   maxBytes: number;
   maxPdfBytes?: number;
   profile?: string;
+  method?: string;
+  body?: string;
   extraHeaders?: Record<string, string>;
   loader?: ImpersonationLoader;
 }
@@ -85,7 +89,7 @@ async function loadImpersonationModule(): Promise<ImpersonationModule | null> {
 const transports = new Map<string, Promise<ImpersonationTransport>>();
 
 function transportKey(options: TlsHopOptions): string {
-  return `${options.profile ?? DEFAULT_PROFILE}|${options.url.hostname}|${options.pinnedIp}|${options.family}`;
+  return `${options.profile ?? DEFAULT_PROFILE}|${options.url.hostname}|${options.pinnedIp ?? ""}|${options.family ?? 0}`;
 }
 
 function evictTransports(): void {
@@ -107,12 +111,13 @@ async function transportFor(
   const key = transportKey(options);
   const existing = transports.get(key);
   if (existing) return existing;
+  const transportOptions: Record<string, unknown> = {
+    browser: options.profile ?? DEFAULT_PROFILE,
+    os: DEFAULT_OS,
+  };
+  if (options.pinnedIp) transportOptions.resolve = { [options.url.hostname]: options.pinnedIp };
   const created = module
-    .createTransport({
-      resolve: { [options.url.hostname]: options.pinnedIp },
-      browser: options.profile ?? DEFAULT_PROFILE,
-      os: DEFAULT_OS,
-    })
+    .createTransport(transportOptions)
     .catch((error: unknown) => {
       transports.delete(key);
       throw error;
@@ -179,12 +184,15 @@ export async function impersonatedRequest(options: TlsHopOptions): Promise<TlsHo
   const signal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;
   let response: ImpersonationResponse;
   try {
-    response = await module.fetch(options.url.toString(), {
+    const init: Record<string, unknown> = {
       transport,
+      method: options.method ?? "GET",
       redirect: "manual",
       signal,
       headers: hopHeaders(options.extraHeaders),
-    });
+    };
+    if (options.body !== undefined) init.body = options.body;
+    response = await module.fetch(options.url.toString(), init);
   } catch (error) {
     throw new Error(failureMessage(error, options.signal, signal));
   }

package/web-fetch.ts

@@ -23,7 +23,7 @@ import {
   stripIpv6Brackets,
   type WebsitePolicy,
 } from "./web-access.ts";
-import { impersonatedRequest, type TlsHopOptions, type TlsHopResponse } from "./tls-fetch.ts";
+import { impersonatedRequest, type FetchTransport, type TlsHopOptions, type TlsHopResponse } from "./tls-fetch.ts";
 import { collapseWhitespace, decodeHtmlEntities, feedHtml, htmlToMarkdown, visibleChars } from "./html-to-md.ts";
 import type { AttrDict } from "./html-to-md.ts";
 import { INVALID_CHARREFS } from "./entities.ts";
@@ -210,7 +210,7 @@ export interface ResolvedHost {
   alternates?: { ip: string; family: number }[];
 }
 
-export type FetchTransport = "tls-first" | "direct-first" | "off";
+export type { FetchTransport };
 
 type TransportKind = "direct" | "tls";
 export interface FetchSeams {

package/web-search.ts

@@ -6,8 +6,10 @@ import {
   EmptySweepError,
   SearchCancelled,
   SearchTimeoutError,
+  type SearchEngineOptions,
   type SearchResult,
 } from "./engines.ts";
+import type { FetchTransport } from "./tls-fetch.ts";
 
 export { EmptySweepError, SearchCancelled, SearchTimeoutError } from "./engines.ts";
 
@@ -24,15 +26,20 @@ export type SearchClient = (
   maxResults: number,
   signal?: AbortSignal,
   timeoutMs?: number,
+  options?: SearchEngineOptions,
 ) => Promise<SearchResult[]>;
 export async function ddgSearch(
   query: string,
   maxResults = MAX_RESULTS,
   signal?: AbortSignal,
   timeoutMs = SEARCH_TIMEOUT_MS,
+  options: SearchEngineOptions = {},
 ): Promise<SearchResult[]> {
   if (signal?.aborted) throw new SearchCancelled();
-  return autoTextSearch(query, maxResults, timeoutMs, signal);
+  return autoTextSearch(query, maxResults, timeoutMs, signal, {
+    ...options,
+    transport: options.transport ?? "tls-first",
+  });
 }
 
 const POLICY_OVERFETCH = 4;
@@ -44,6 +51,10 @@ export interface WebSearchOptions {
   websitePolicy?: WebsitePolicy | null;
   client?: SearchClient;
   cwd?: string;
+  transport?: FetchTransport;
+  engines?: string[];
+  engineWeights?: Record<string, number>;
+  maxPerHost?: number;
 }
 
 export { loadDefaultMaxResults };
@@ -71,7 +82,13 @@ export async function webSearch(
         (policy?.blockedDomains?.length ?? 0) > 0,
     );
     const wanted = restricted ? maxResults * POLICY_OVERFETCH : maxResults;
-    const results = await client(effectiveQuery, wanted, signal, timeoutMs);
+    const results = await client(effectiveQuery, wanted, signal, timeoutMs, {
+      transport: options.transport,
+      policy,
+      engines: options.engines,
+      engineWeights: options.engineWeights,
+      maxPerHost: options.maxPerHost,
+    });
     if (signal?.aborted) return "Search cancelled.";
     if (!results.length) return EMPTY_SEARCH_RESULTS[0];
     const allowed: SearchResult[] = [];