pi-unsloth-webtools 0.9.0 → 0.10.0

package/engines.ts

@@ -1,11 +1,12 @@
-import { randomBytes } from "node:crypto";
 import { appendFile, chmod, mkdir } from "node:fs/promises";
 import { dirname, isAbsolute, join } from "node:path";
 import { collapseWhitespace, decodeHtmlEntities, feedHtml } from "./html-to-md.ts";
 import type { AttrDict } from "./html-to-md.ts";
 import { randomUserAgent } from "./user-agents.ts";
 import { agentDir } from "./agent-dir.ts";
-import { MAX_SIGNAL_TIMEOUT_MS } from "./web-access.ts";
+import { impersonatedRequest, type FetchTransport, type TlsHopOptions, type TlsHopResponse } from "./tls-fetch.ts";
+import { checkUrlAccess, isPublicIp, MAX_SIGNAL_TIMEOUT_MS, type WebsitePolicy } from "./web-access.ts";
+import { socksProxyForUrl } from "./proxy.ts";
 export class EmptySweepError extends Error {
   constructor() {
     super("No results found");
@@ -247,6 +248,13 @@ function parsePredExpr(input: string): Pred {
       const name = word();
       return { op: "desc", tag: name };
     }
+    if (input.startsWith("./", pos)) {
+      pos += 2;
+      const name = word();
+      const { preds, next } = parsePredicateBlocks(input, pos);
+      pos = next;
+      return { op: "child", tag: name, preds };
+    }
     const name = word();
     const { preds, next } = parsePredicateBlocks(input, pos);
     pos = next;
@@ -471,42 +479,18 @@ export function extractResults(
   return results;
 }
 
-function googleUserAgent(): string {
-  const devices: [string, string, number, number][] = [
-    ["5.0", "SM-G900P Build/LRX21T", 39, 60],
-    ["6.0", "Nexus 5 Build/MRA58N", 39, 60],
-    ["8.0", "Pixel 2 Build/OPD3.170816.012", 39, 60],
-  ];
-  const [androidVer, device, chromeMin, chromeMax] = devices[Math.floor(Math.random() * devices.length)];
-  const chromeMajor = chromeMin + Math.floor(Math.random() * (chromeMax - chromeMin + 1));
-  const chromeBuild = 1000 + Math.floor(Math.random() * 9000);
-  const chromePatch = 1000 + Math.floor(Math.random() * 1000);
-  return (
-    `Mozilla/5.0 (Linux; Android ${androidVer}; ${device}) ` +
-    `AppleWebKit/537.36 (KHTML, like Gecko) ` +
-    `Chrome/${chromeMajor}.0.${chromeBuild}.${chromePatch} Mobile Safari/537.36`
-  );
-}
-
-function tokenUrlSafe(byteLength: number): string {
-  return randomBytes(byteLength).toString("base64url");
-}
-
-function unquotePlus(value: string): string {
-  try {
-    return decodeURIComponent(value.replace(/\+/g, "%20"));
-  } catch {
-    return value.replace(/\+/g, " ");
-  }
-}
+export type EngineImpersonation = (options: TlsHopOptions) => Promise<TlsHopResponse | null>;
 
-function yahooExtractUrl(raw: string): string {
-  const afterRu = raw.split("/RU=", 2)[1] ?? "";
-  const t = afterRu.split("/RK=", 1)[0].split("/RS=", 1)[0];
-  return unquotePlus(t);
+export interface SearchEngineOptions {
+  transport?: FetchTransport;
+  policy?: WebsitePolicy | null;
+  impersonate?: EngineImpersonation;
+  engines?: string[];
+  engineWeights?: Record<string, number>;
+  maxPerHost?: number;
 }
 
-interface EngineContext {
+export interface EngineContext extends SearchEngineOptions {
   region: string;
   safesearch: string;
 }
@@ -514,7 +498,6 @@ interface EngineContext {
 export interface Engine {
   name: string;
   provider: string;
-  priority?: number;
   search(
     query: string,
     ctx: EngineContext,
@@ -528,6 +511,7 @@ interface HttpRequestOptions {
   cookies?: Record<string, string>;
   timeoutMs: number;
   signal?: AbortSignal;
+  ctx?: EngineContext;
 }
 
 interface HttpOptions extends HttpRequestOptions {
@@ -555,6 +539,8 @@ async function httpPost(
 
 const MAX_ENGINE_RESPONSE_BYTES = 5 * 1024 * 1024;
 const ENGINE_RETRY_BACKOFF_MS = 250;
+const MAX_ENGINE_HOPS = 5;
+const DEFAULT_ENGINE_TRANSPORT: FetchTransport = "off";
 
 const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
 
@@ -578,10 +564,84 @@ async function readBodyCapped(response: Response): Promise<string | null> {
   return new TextDecoder("utf-8").decode(Buffer.concat(chunks));
 }
 
-function mapFetchError(err: unknown): never {
-  if (err instanceof DOMException && err.name === "TimeoutError") throw new SearchTimeoutError();
-  if (err instanceof DOMException && err.name === "AbortError") throw new SearchCancelled();
-  throw err;
+interface EngineHopResponse {
+  status: number;
+  location: string | null;
+  body: string | null;
+}
+
+type EngineTransportKind = "direct" | "tls";
+
+function engineTransportOrder(transport: FetchTransport, target: URL): EngineTransportKind[] {
+  if (transport === "off" || socksProxyForUrl(target) !== null) return ["direct"];
+  return transport === "direct-first" ? ["direct", "tls"] : ["tls", "direct"];
+}
+
+function engineTargetAllowed(target: string, policy: WebsitePolicy | null): boolean {
+  const [allowed, , hostname] = checkUrlAccess(target, policy);
+  if (!allowed) return false;
+  const isLiteral = hostname.includes(":") || /^\d+\.\d+\.\d+\.\d+$/.test(hostname);
+  return !isLiteral || isPublicIp(hostname);
+}
+
+function classifyRequestError(
+  err: unknown,
+  caller: AbortSignal | undefined,
+  hopSignal: AbortSignal,
+): Error | null {
+  if (caller?.aborted) return new SearchCancelled();
+  if (hopSignal.aborted) return new SearchTimeoutError();
+  if (err instanceof DOMException && err.name === "TimeoutError") return new SearchTimeoutError();
+  if (err instanceof DOMException && err.name === "AbortError") return new SearchTimeoutError();
+  if (err instanceof Error && err.message === "timed out") return new SearchTimeoutError();
+  if (err instanceof Error && err.message === "cancelled") return new SearchCancelled();
+  return null;
+}
+
+async function directEngineHop(
+  target: URL,
+  headers: Record<string, string>,
+  method: string,
+  body: string | undefined,
+  signal: AbortSignal,
+): Promise<EngineHopResponse | null> {
+  const response = await fetch(target.toString(), { method, headers, body, signal, redirect: "manual" });
+  if (response.status === 200) {
+    const text = await readBodyCapped(response);
+    return text === null ? null : { status: 200, location: null, body: text };
+  }
+  try {
+    await response.body?.cancel();
+  } catch {}
+  return { status: response.status, location: response.headers.get("location"), body: null };
+}
+
+function tlsEngineHop(
+  impersonate: EngineImpersonation,
+  target: URL,
+  headers: Record<string, string>,
+  method: string,
+  body: string | undefined,
+  caller: AbortSignal | undefined,
+  timeoutMs: number,
+): Promise<EngineHopResponse | null> {
+  return impersonate({
+    url: target,
+    timeoutMs,
+    signal: caller,
+    maxBytes: MAX_ENGINE_RESPONSE_BYTES,
+    maxPdfBytes: MAX_ENGINE_RESPONSE_BYTES,
+    method,
+    body,
+    extraHeaders: headers,
+  }).then((response) => {
+    if (response === null || response.truncated) return null;
+    return {
+      status: response.status,
+      location: response.headers.location ?? null,
+      body: response.status === 200 ? new TextDecoder("utf-8").decode(response.body) : null,
+    };
+  });
 }
 
 async function httpFetch(
@@ -601,32 +661,67 @@ async function httpFetch(
     : null;
   if (cookie) headers["Cookie"] = cookie;
   const timeoutMs = Math.min(MAX_SIGNAL_TIMEOUT_MS, Math.max(1, options.timeoutMs));
-  const signals: AbortSignal[] = [AbortSignal.timeout(timeoutMs)];
-  if (options.signal) signals.push(options.signal);
-  let response: Response;
-  try {
-    response = await fetch(url, {
-      method: options.method ?? "GET",
-      headers,
-      body: options.method === "POST" ? options.body : undefined,
-      signal: AbortSignal.any(signals),
-    });
-  } catch (err) {
-    throw mapFetchError(err);
-  }
-  if (response.status !== 200) {
-    try {
-      await response.body?.cancel();
-    } catch {
+  const deadline = Date.now() + timeoutMs;
+  const caller = options.signal;
+  const transport = options.ctx?.transport ?? DEFAULT_ENGINE_TRANSPORT;
+  const policy = options.ctx?.policy ?? null;
+  const impersonate = options.ctx?.impersonate ?? (transport === "off" ? null : impersonatedRequest);
+  let method = options.method ?? "GET";
+  let body = method === "POST" ? options.body : undefined;
+  let target = url;
+  for (let hop = 0; hop < MAX_ENGINE_HOPS; hop++) {
+    if (caller?.aborted) throw new SearchCancelled();
+    const remaining = deadline - Date.now();
+    if (remaining <= 0) throw new SearchTimeoutError();
+    const targetUrl = new URL(target);
+    const hopSignal = caller
+      ? AbortSignal.any([caller, AbortSignal.timeout(remaining)])
+      : AbortSignal.timeout(remaining);
+    let response: EngineHopResponse | null = null;
+    let failure: unknown = null;
+    for (const kind of engineTransportOrder(transport, targetUrl)) {
+      try {
+        let attempt: EngineHopResponse | null = null;
+        if (kind === "tls") {
+          if (impersonate !== null) {
+            attempt = await tlsEngineHop(impersonate, targetUrl, headers, method, body, caller, remaining);
+          }
+        } else {
+          attempt = await directEngineHop(targetUrl, headers, method, body, hopSignal);
+        }
+        if (attempt === null) continue;
+        if (response === null || (attempt.status < 400 && response.status >= 400)) response = attempt;
+        if (attempt.status < 400) break;
+      } catch (err) {
+        const mapped = classifyRequestError(err, caller, hopSignal);
+        if (mapped) throw mapped;
+        failure = err;
+      }
+    }
+    if (response === null) {
+      if (failure) throw failure;
       return null;
     }
-    return null;
-  }
-  try {
-    return await readBodyCapped(response);
-  } catch (err) {
-    throw mapFetchError(err);
+    if (response.status >= 300 && response.status < 400) {
+      if (!response.location) return null;
+      let next: string;
+      try {
+        next = new URL(response.location, target).toString();
+      } catch {
+        return null;
+      }
+      if (!engineTargetAllowed(next, policy)) return null;
+      if (response.status !== 307 && response.status !== 308) {
+        method = "GET";
+        body = undefined;
+      }
+      target = next;
+      continue;
+    }
+    if (response.status !== 200) return null;
+    return response.body ?? "";
   }
+  return null;
 }
 
 const DUCKDUCKGO: Engine = {
@@ -636,7 +731,7 @@ const DUCKDUCKGO: Engine = {
     const html = await httpPost(
       "https://html.duckduckgo.com/html/",
       { q: query, b: "", l: ctx.region },
-      { headers: { "User-Agent": randomUserAgent() }, timeoutMs, signal },
+      { headers: { "User-Agent": randomUserAgent() }, timeoutMs, signal, ctx },
     );
     if (!html) return null;
     const results = extractResults(html, "//div[contains(@class, 'body')]", {
@@ -648,126 +743,15 @@ const DUCKDUCKGO: Engine = {
   },
 };
 
-const BRAVE: Engine = {
-  name: "brave",
-  provider: "brave",
-  async search(query, ctx, timeoutMs, signal) {
-    const country = ctx.region.toLowerCase().split("-")[0];
-    const cookies: Record<string, string> = { [country]: country, useLocation: "0" };
-    if (ctx.safesearch !== "moderate") {
-      cookies["safesearch"] = ctx.safesearch === "on" ? "strict" : "off";
-    }
-    const html = await httpGet(
-      "https://search.brave.com/search",
-      { q: query, source: "web" },
-      { cookies, timeoutMs, signal },
-    );
-    if (!html) return null;
-    return extractResults(html, "//div[@data-type='web']", {
-      title:
-        ".//div[(contains(@class,'title') or contains(@class,'sitename-container')) and position()=last()]//text()",
-      href: ".//a[div[contains(@class, 'title')]]/@href",
-      body: ".//div[contains(@class, 'snippet')]//div[contains(@class, 'content')]//text()",
-    });
-  },
-};
-
-const GOOGLE: Engine = {
-  name: "google",
-  provider: "google",
-  async search(query, ctx, timeoutMs, signal) {
-    const [country, lang] = ctx.region.split("-");
-    const safesearchBase: Record<string, string> = { on: "2", moderate: "1", off: "0" };
-    const html = await httpGet(
-      "https://www.google.com/search",
-      {
-        q: query,
-        filter: safesearchBase[ctx.safesearch.toLowerCase()] ?? "1",
-        start: "0",
-        hl: `${lang}-${country.toUpperCase()}`,
-        lr: `lang_${lang}`,
-        cr: `country${country.toUpperCase()}`,
-      },
-      {
-        headers: { "User-Agent": googleUserAgent() },
-        cookies: { CONSENT: "YES+" },
-        timeoutMs,
-        signal,
-      },
-    );
-    if (!html) return null;
-    const results = extractResults(html, "//div[@data-hveid][.//h3]", {
-      title: ".//h3//text()",
-      href: ".//a[.//h3]/@href",
-      body: "./div/div[last()]//text()",
-    });
-    return results
-      .map((r) => {
-        if (r.href.startsWith("/url?q=")) {
-          r.href = r.href.split("?q=")[1].split("&")[0];
-        }
-        return r;
-      })
-      .filter((r) => r.title && r.href.startsWith("http"));
-  },
-};
-
-const MOJEEK: Engine = {
-  name: "mojeek",
-  provider: "mojeek",
-  async search(query, ctx, timeoutMs, signal) {
-    const [country, lang] = ctx.region.toLowerCase().split("-");
-    const params: Record<string, string> = { q: query };
-    if (ctx.safesearch === "on") params["safe"] = "1";
-    const html = await httpGet(
-      "https://www.mojeek.com/search",
-      params,
-      { cookies: { arc: country, lb: lang }, timeoutMs, signal },
-    );
-    if (!html) return null;
-    return extractResults(html, "//ul[contains(@class, 'results')]/li", {
-      title: ".//h2//text()",
-      href: ".//h2/a/@href",
-      body: ".//p[@class='s']//text()",
-    });
-  },
-};
-
-const YAHOO: Engine = {
-  name: "yahoo",
-  provider: "bing",
-  async search(query, _ctx, timeoutMs, signal) {
-    const ylt = tokenUrlSafe(18);
-    const ylu = tokenUrlSafe(35);
-    const html = await httpGet(
-      `https://search.yahoo.com/search;_ylt=${ylt};_ylu=${ylu}`,
-      { p: query },
-      { timeoutMs, signal },
-    );
-    if (!html) return null;
-    const results = extractResults(html, "//div[contains(@class, 'relsrch')]", {
-      title: ".//div[contains(@class, 'Title')]//h3//text()",
-      href: ".//div[contains(@class, 'Title')]//a/@href",
-      body: ".//div[contains(@class, 'Text')]//text()",
-    });
-    return results
-      .filter((r) => !r.href.startsWith("https://www.bing.com/aclick?"))
-      .map((r) => {
-        if (r.href.includes("/RU=")) r.href = yahooExtractUrl(r.href);
-        return r;
-      });
-  },
-};
-
 const YANDEX: Engine = {
   name: "yandex",
   provider: "yandex",
-  async search(query, _ctx, timeoutMs, signal) {
+  async search(query, ctx, timeoutMs, signal) {
     const searchid = 1000000 + Math.floor(Math.random() * 9000000);
     const html = await httpGet(
       "https://yandex.com/search/site/",
       { text: query, web: "1", searchid: String(searchid) },
-      { timeoutMs, signal },
+      { timeoutMs, signal, ctx },
     );
     if (!html) return null;
     return extractResults(html, "//li[contains(@class, 'serp-item')]", {
@@ -778,60 +762,36 @@ const YANDEX: Engine = {
   },
 };
 
-const WIKIPEDIA: Engine = {
-  name: "wikipedia",
-  provider: "wikipedia",
-  priority: 2,
+const START_PAGE: Engine = {
+  name: "startpage",
+  provider: "google",
   async search(query, ctx, timeoutMs, signal) {
-    const started = Date.now();
-    const lang = ctx.region.toLowerCase().split("-")[1] ?? "en";
-    const encoded = encodeURIComponent(query);
-    const opensearchUrl =
-      `https://${lang}.wikipedia.org/w/api.php?action=opensearch&profile=fuzzy&limit=1&search=${encoded}`;
-    const opensearch = await httpGet(opensearchUrl, {}, { timeoutMs, signal });
-    if (!opensearch) return null;
-    let data: unknown;
-    try {
-      data = JSON.parse(opensearch);
-    } catch {
-      return null;
-    }
-    const payload = data as [string, string[], string[], string[]];
-    if (!payload[1] || !payload[1].length) return [];
-    const title = payload[1][0];
-    const href = payload[3][0];
-    let body = "";
-    const extractUrl =
-      `https://${lang}.wikipedia.org/w/api.php?action=query&format=json&prop=extracts` +
-      `&titles=${encodeURIComponent(title)}&explaintext=0&exintro=0&redirects=1`;
-    const extract = await httpGet(extractUrl, {}, { timeoutMs: Math.max(1, timeoutMs - (Date.now() - started)), signal });
-    if (extract) {
-      try {
-        const pageData = JSON.parse(extract) as {
-          query: { pages: Record<string, { extract?: string }> };
-        };
-        const pages = Object.values(pageData.query.pages);
-        if (pages.length) body = pages[0].extract ?? "";
-      } catch {
-        body = "";
-      }
-    }
-    if (body.includes("may refer to:")) return [];
-    return [{ title: normalizeText(title), href: normalizeUrl(href), body: normalizeText(body) }];
+    const [country, lang] = ctx.region.toLowerCase().split("-");
+    const html = await httpGet(
+      "https://www.startpage.com/sp/search",
+      { query, qsr: `${lang}_${country.toUpperCase()}` },
+      { headers: { Referer: "https://www.startpage.com/" }, timeoutMs, signal, ctx },
+    );
+    if (!html) return null;
+    return extractResults(html, "//div[contains(@class, 'result')][./a]", {
+      title: ".//h2//text()",
+      href: "./a/@href",
+      body: ".//p//text()",
+    });
   },
 };
 
-export const TEXT_ENGINES: Engine[] = [DUCKDUCKGO, BRAVE, GOOGLE, MOJEEK, YAHOO, YANDEX, WIKIPEDIA];
+export const TEXT_ENGINES: Engine[] = [DUCKDUCKGO, YANDEX, START_PAGE];
 
 export class ResultsAggregator {
   private cache = new Map<string, SearchResult>();
-  private counter = new Map<string, number>();
+  private scores = new Map<string, number>();
 
   get size(): number {
     return this.cache.size;
   }
 
-  append(item: SearchResult): void {
+  append(item: SearchResult, weight = 1, rank = 1): void {
     if (typeof item.href !== "string" || !item.href.trim()) return;
     const key = canonicalizeHref(item.href);
     if (!key) return;
@@ -839,54 +799,66 @@ export class ResultsAggregator {
     if (!existing || item.body.length > existing.body.length) {
       this.cache.set(key, { ...item, href: key });
     }
-    this.counter.set(key, (this.counter.get(key) ?? 0) + 1);
+    this.scores.set(key, (this.scores.get(key) ?? 0) + weight / (RRF_RANK_CONSTANT + rank));
   }
 
-  extend(items: SearchResult[]): void {
-    for (const item of items) this.append(item);
+  extend(items: SearchResult[], weight = 1): void {
+    const seen = new Set<string>();
+    items.forEach((item, index) => {
+      const key = canonicalizeHref(item.href);
+      if (!key) return;
+      const first = !seen.has(key);
+      seen.add(key);
+      this.append(item, first ? weight : 0, index + 1);
+    });
   }
 
-  extractDicts(): SearchResult[] {
-    return [...this.counter.entries()]
-      .sort((a, b) => b[1] - a[1])
-      .map(([key]) => this.cache.get(key)!);
+  ranked(): SearchResult[] {
+    return [...this.cache.entries()]
+      .filter(([, doc]) => !isWikimediaCategory(doc))
+      .map(([key, doc]) => ({ doc, score: this.scores.get(key) ?? 0 }))
+      .sort((a, b) => b.score - a.score || a.doc.href.localeCompare(b.doc.href))
+      .map((entry) => entry.doc);
   }
 }
 
-function extractTokens(query: string): Set<string> {
-  return new Set(query.toLowerCase().split(/\W+/u).filter((t) => t.length >= 3));
-}
+const RRF_RANK_CONSTANT = 60;
+const DEFAULT_MAX_PER_HOST = 0;
+const MULTI_PART_SUFFIXES = new Set(["co.uk", "org.uk", "com.au", "co.jp", "co.nz", "com.br", "co.in"]);
 
-function hasAnyToken(text: string, tokens: Set<string>): boolean {
-  const lower = text.toLowerCase();
-  for (const token of tokens) {
-    if (lower.includes(token)) return true;
+export function registrableDomain(url: string): string {
+  let hostname = "";
+  try {
+    hostname = new URL(url).hostname.toLowerCase().replace(/^www\./, "");
+  } catch {
+    return "";
   }
-  return false;
+  const parts = hostname.split(".");
+  if (parts.length < 2) return hostname;
+  const lastTwo = parts.slice(-2).join(".");
+  return MULTI_PART_SUFFIXES.has(lastTwo) && parts.length >= 3 ? parts.slice(-3).join(".") : lastTwo;
 }
 
-export function rankResults(docs: SearchResult[], query: string): SearchResult[] {
-  const tokens = extractTokens(query);
-  const wiki: SearchResult[] = [];
-  const both: SearchResult[] = [];
-  const titleOnly: SearchResult[] = [];
-  const bodyOnly: SearchResult[] = [];
-  const neither: SearchResult[] = [];
+export function capByHost(docs: SearchResult[], limit: number, maxPerHost: number): SearchResult[] {
+  if (limit <= 0) return [];
+  const perHost = new Map<string, number>();
+  const capped: SearchResult[] = [];
+  const allowed = maxPerHost > 0 ? maxPerHost : Number.POSITIVE_INFINITY;
   for (const doc of docs) {
-    if (doc.title.includes("Category:") && doc.title.includes("Wikimedia")) continue;
-    if (doc.href.includes("wikipedia.org")) {
-      wiki.push(doc);
-      continue;
-    }
-    const hitTitle = hasAnyToken(doc.title, tokens);
-    const hitBody = hasAnyToken(doc.body, tokens);
-    if (hitTitle && hitBody) both.push(doc);
-    else if (hitTitle) titleOnly.push(doc);
-    else if (hitBody) bodyOnly.push(doc);
-    else neither.push(doc);
+    const domain = registrableDomain(doc.href);
+    const used = perHost.get(domain) ?? 0;
+    if (used >= allowed) continue;
+    perHost.set(domain, used + 1);
+    capped.push(doc);
+    if (capped.length >= limit) break;
   }
-  return [...wiki, ...both, ...titleOnly, ...bodyOnly, ...neither];
+  return capped;
 }
+
+function isWikimediaCategory(doc: SearchResult): boolean {
+  return doc.title.includes("Category:") && doc.title.includes("Wikimedia");
+}
+
 async function recordSweepStats(query: string, maxResults: number, started: number, timedOutProviders: string[], resultCount: number): Promise<void> {
   const flag = process.env.PI_UNSLOTH_WEBTOOLS_STATS?.trim();
   if (!flag) return;
@@ -922,15 +894,20 @@ async function recordSweepStats(query: string, maxResults: number, started: numb
   } catch {}
 }
 
-function shuffledEngines(): Engine[] {
-  const shuffled = [...TEXT_ENGINES];
+function selectedEngines(names: string[] | undefined): Engine[] {
+  if (!names?.length) return TEXT_ENGINES;
+  const wanted = new Set(names.map((name) => name.trim().toLowerCase()));
+  const chosen = TEXT_ENGINES.filter((engine) => wanted.has(engine.name));
+  return chosen.length ? chosen : TEXT_ENGINES;
+}
+
+function shuffledEngines(engines: Engine[] = TEXT_ENGINES): Engine[] {
+  const shuffled = [...engines];
   for (let i = shuffled.length - 1; i > 0; i--) {
     const j = Math.floor(Math.random() * (i + 1));
     [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
   }
-  const wikipedia = shuffled.find((e) => e.priority === 2);
-  const rest = shuffled.filter((e) => e.priority !== 2);
-  return wikipedia ? [wikipedia, ...rest] : shuffled;
+  return shuffled;
 }
 
 export async function autoTextSearch(
@@ -938,13 +915,23 @@ export async function autoTextSearch(
   maxResults: number,
   timeoutMs: number,
   signal?: AbortSignal,
+  options: SearchEngineOptions = {},
 ): Promise<SearchResult[]> {
   const started = Date.now();
-  const engines = shuffledEngines();
+  const engines = shuffledEngines(selectedEngines(options.engines));
   const deadline = started + timeoutMs;
   const seenProviders = new Set<string>();
   const aggregator = new ResultsAggregator();
-  const ctx: EngineContext = { region: "us-en", safesearch: "moderate" };
+  const engineWeights = options.engineWeights ?? {};
+  const maxPerHost = options.maxPerHost ?? DEFAULT_MAX_PER_HOST;
+  const enough = () => capByHost(aggregator.ranked(), maxResults, maxPerHost).length >= maxResults;
+  const ctx: EngineContext = {
+    region: "us-en",
+    safesearch: "moderate",
+    transport: options.transport ?? DEFAULT_ENGINE_TRANSPORT,
+    policy: options.policy ?? null,
+    impersonate: options.impersonate,
+  };
   const controller = new AbortController();
   let onAbort: (() => void) | undefined;
   if (signal) {
@@ -998,18 +985,18 @@ export async function autoTextSearch(
       }
     }
     if (results && results.length) {
-      aggregator.extend(results);
+      aggregator.extend(results, engineWeights[engine.name] ?? 1);
       seenProviders.add(engine.provider);
-      if (aggregator.size >= maxResults) controller.abort();
+      if (enough()) controller.abort();
     }
   };
   while (i < engines.length || pending.size > 0) {
-    if (aggregator.size >= maxResults || cancelled) {
+    if (enough() || cancelled) {
       controller.abort();
       break;
     }
     while (i < engines.length && pending.size < maxWorkers) {
-      if (aggregator.size >= maxResults || cancelled) {
+      if (enough() || cancelled) {
         controller.abort();
         break;
       }
@@ -1027,7 +1014,7 @@ export async function autoTextSearch(
       );
     }
     if (pending.size === 0) break;
-    if (aggregator.size >= maxResults || cancelled) {
+    if (enough() || cancelled) {
       controller.abort();
       break;
     }
@@ -1036,10 +1023,10 @@ export async function autoTextSearch(
   await Promise.allSettled(pending);
   if (onAbort && signal) signal.removeEventListener("abort", onAbort);
   if (cancelled) throw new SearchCancelled();
-  const results = rankResults(aggregator.extractDicts(), query);
+  const results = capByHost(aggregator.ranked(), maxResults, maxPerHost);
   if (results.length) {
     void recordSweepStats(query, maxResults, started, [...timedOutProviders], results.length);
-    return results.slice(0, maxResults);
+    return results;
   }
   if (timedOutProviders.size) {
     const sorted = [...timedOutProviders].sort();