search-web-api 1.0.13 → 1.0.15

package/package.json

@@ -2,7 +2,7 @@
   "name": "search-web-api",
   "description": "70+ search engines across 10 categories and Scrape Extract API",
   "type": "module",
-  "version": "1.0.13",
+  "version": "1.0.15",
   "author": "vtempest",
   "license": "MIT",
   "repository": {
@@ -10,6 +10,16 @@
     "url": "git+https://github.com/OpenSourceAGI/qwksearch-research-agent.git",
     "directory": "packages/search-web-api"
   },
+  "exports": {
+    ".": {
+      "import": "./src/search.ts",
+      "require": "./src/search.ts"
+    },
+    "./*": {
+      "import": "./src/*",
+      "require": "./src/*"
+    }
+  },
   "scripts": {
     "dev": "bun --watch demo/index.ts",
     "start": "bun demo/index.ts",
@@ -18,9 +28,11 @@
   "dependencies": {
     "@huggingface/transformers": "^3.8.1",
     "@scalar/hono-api-reference": "^0.9.26",
+    "chrono-node": "^2.9.0",
     "grab-url": "^1.0.8",
     "hono": "^4.6.12",
-    "linkedom": "^0.18.0"
+    "linkedom": "^0.18.0",
+    "tldts": "^7.0.25"
   },
   "devDependencies": {
     "@types/node": "^25.9.3",
@@ -29,4 +41,4 @@
     "typescript": "^5.9.3",
     "vitest": "^4.0.18"
   }
-}
+}

package/src/search/__tests__/public-searxng.test.ts

@@ -0,0 +1,529 @@
+/**
+ * @fileoverview Unit tests for SearXNG search functionality
+ */
+import { searchWeb, searchSearxng } from "../public-searxng";
+import grab from "grab-url";
+
+// Mock grab-url
+jest.mock("grab-url");
+const mockGrab = grab as jest.MockedFunction<typeof grab>;
+
+describe("searchWeb", () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe("Private SearXNG instance (JSON)", () => {
+    it("should search with private SearXNG instance", async () => {
+      const mockResults = {
+        results: [
+          {
+            title: "Test Result 1",
+            url: "https://example.com/1",
+            content: "Test snippet 1",
+            score: 0.95,
+            metadata: "2024-01-01 | Example News",
+          },
+          {
+            title: "Test Result 2",
+            url: "https://example.com/2",
+            content: "Test snippet 2",
+            score: 0.85,
+          },
+        ],
+        suggestions: ["suggestion1", "suggestion2"],
+        infoboxes: [],
+      };
+
+      mockGrab.mockResolvedValueOnce(mockResults);
+
+      const result = await searchWeb("test query", {
+        privateSearxng: "https://search.example.com",
+      });
+
+      expect(mockGrab).toHaveBeenCalledWith(
+        "https://search.example.com/search",
+        expect.objectContaining({
+          q: expect.stringContaining("test"),
+          format: "json",
+        })
+      );
+
+      expect(result).toHaveProperty("results");
+      expect(result).toHaveProperty("suggestions");
+      if (!Array.isArray(result)) {
+        expect(result.results).toHaveLength(2);
+        expect(result.results[0].title).toBe("Test Result 1");
+        expect(result.results[0].score).toBe(0.95);
+      }
+    });
+
+    it("should parse metadata for date and source", async () => {
+      const mockResults = {
+        results: [
+          {
+            title: "Article with Metadata",
+            url: "https://example.com/article",
+            content: "Content",
+            score: 0.9,
+            metadata: "Jan 15, 2024 | TechNews",
+          },
+        ],
+        suggestions: [],
+        infoboxes: [],
+      };
+
+      mockGrab.mockResolvedValueOnce(mockResults);
+
+      const result = await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+      });
+
+      if (!Array.isArray(result) && result.results.length > 0) {
+        const firstResult = result.results[0];
+        expect(firstResult.date).toBeDefined();
+        expect(firstResult.source).toBeDefined();
+      }
+    });
+
+    it("should clean HTML entities from titles", async () => {
+      const mockResults = {
+        results: [
+          {
+            title: "Test &amp; Title with &lt;HTML&gt; Entities",
+            url: "https://example.com",
+            content: "Content",
+            score: 0.9,
+          },
+        ],
+        suggestions: [],
+        infoboxes: [],
+      };
+
+      mockGrab.mockResolvedValueOnce(mockResults);
+
+      const result = await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+      });
+
+      if (!Array.isArray(result) && result.results.length > 0) {
+        expect(result.results[0].title).not.toContain("&amp;");
+        expect(result.results[0].title).toContain("&");
+        expect(result.results[0].title).not.toContain("&lt;");
+      }
+    });
+
+    it("should handle breadcrumbed titles", async () => {
+      const mockResults = {
+        results: [
+          {
+            title: "Site Name | Very Long Article Title Here",
+            url: "https://example.com",
+            content: "Content",
+            score: 0.9,
+          },
+        ],
+        suggestions: [],
+        infoboxes: [],
+      };
+
+      mockGrab.mockResolvedValueOnce(mockResults);
+
+      const result = await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+      });
+
+      if (!Array.isArray(result) && result.results.length > 0) {
+        // Should extract the longest part
+        expect(result.results[0].title).toContain("Very Long Article Title Here");
+      }
+    });
+
+    it("should add favicon URLs", async () => {
+      const mockResults = {
+        results: [
+          {
+            title: "Test",
+            url: "https://example.com/page",
+            content: "Content",
+            score: 0.9,
+          },
+        ],
+        suggestions: [],
+        infoboxes: [],
+      };
+
+      mockGrab.mockResolvedValueOnce(mockResults);
+
+      const result = await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+      });
+
+      if (!Array.isArray(result) && result.results.length > 0) {
+        expect(result.results[0].favicon).toContain("googleusercontent.com");
+        expect(result.results[0].favicon).toContain("example.com");
+      }
+    });
+
+    it("should handle string response by parsing JSON", async () => {
+      const mockResultsString = JSON.stringify({
+        results: [
+          {
+            title: "Test",
+            url: "https://example.com",
+            content: "Content",
+            score: 0.9,
+          },
+        ],
+        suggestions: [],
+        infoboxes: [],
+      });
+
+      mockGrab.mockResolvedValueOnce(mockResultsString);
+
+      const result = await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+      });
+
+      if (!Array.isArray(result)) {
+        expect(result.results).toHaveLength(1);
+      }
+    });
+
+    it("should handle invalid JSON gracefully", async () => {
+      mockGrab.mockResolvedValueOnce("Not valid JSON");
+
+      const result = await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+      });
+
+      if (!Array.isArray(result)) {
+        expect(result.results).toEqual([]);
+        expect(result.suggestions).toEqual([]);
+      }
+    });
+
+    it("should validate URL paths", async () => {
+      const mockResults = {
+        results: [
+          {
+            title: "Domain Only",
+            url: "https://example.com/", // Just domain
+            content: "Content",
+            score: 0.9,
+          },
+          {
+            title: "Full Path",
+            url: "https://example.com/article/page",
+            content: "Content",
+            score: 0.8,
+          },
+        ],
+        suggestions: [],
+        infoboxes: [],
+      };
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+      mockGrab.mockResolvedValueOnce(mockResults);
+
+      const result = await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+      });
+
+      expect(consoleSpy).toHaveBeenCalledWith(
+        expect.stringContaining("domain-only"),
+        expect.any(String)
+      );
+      consoleSpy.mockRestore();
+    });
+  });
+
+  describe("Public SearXNG instance (HTML scraping)", () => {
+    it("should scrape results from HTML", async () => {
+      const mockHtml = `
+        <article class="result">
+          <h3><a href="https://example.com/1">First Result</a></h3>
+          <p class="content">First snippet</p>
+        </article>
+        <article class="result">
+          <h3><a href="https://example.com/2">Second Result</a></h3>
+          <p class="content">Second snippet</p>
+        </article>
+      `;
+
+      mockGrab.mockResolvedValueOnce(mockHtml);
+
+      const result = await searchWeb("test", {
+        privateSearxng: false,
+      });
+
+      expect(Array.isArray(result)).toBe(true);
+      if (Array.isArray(result)) {
+        expect(result).toHaveLength(2);
+        expect(result[0].title).toBe("First Result");
+        expect(result[0].url).toBe("https://example.com/1");
+        expect(result[0].snippet).toBe("First snippet");
+      }
+    });
+
+    it("should handle HTML entities in scraped content", async () => {
+      const mockHtml = `
+        <article class="result">
+          <h3><a href="https://example.com?param=value&amp;other=test">Test &amp; Title</a></h3>
+          <p class="content">Content with &lt;entities&gt;</p>
+        </article>
+      `;
+
+      mockGrab.mockResolvedValueOnce(mockHtml);
+
+      const result = await searchWeb("test", {
+        privateSearxng: false,
+      });
+
+      if (Array.isArray(result) && result.length > 0) {
+        expect(result[0].title).toContain("&");
+        expect(result[0].title).not.toContain("&amp;");
+        expect(result[0].url).toContain("&");
+        expect(result[0].snippet).toContain("<entities>");
+      }
+    });
+
+    it("should add favicons to scraped results", async () => {
+      const mockHtml = `
+        <article class="result">
+          <h3><a href="https://example.com/page">Test</a></h3>
+          <p class="content">Content</p>
+        </article>
+      `;
+
+      mockGrab.mockResolvedValueOnce(mockHtml);
+
+      const result = await searchWeb("test", {
+        privateSearxng: false,
+      });
+
+      if (Array.isArray(result) && result.length > 0) {
+        expect(result[0].favicon).toContain("google.com/s2/favicons");
+        expect(result[0].domain).toBe("example.com");
+      }
+    });
+
+    it("should validate URLs in scraped results", async () => {
+      const mockHtml = `
+        <article class="result">
+          <h3><a href="https://example.com/">Domain Only</a></h3>
+          <p class="content">Content</p>
+        </article>
+      `;
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+      mockGrab.mockResolvedValueOnce(mockHtml);
+
+      await searchWeb("test", { privateSearxng: false });
+
+      expect(consoleSpy).toHaveBeenCalledWith(
+        expect.stringContaining("domain-only"),
+        expect.any(String)
+      );
+      consoleSpy.mockRestore();
+    });
+  });
+
+  describe("Retry logic", () => {
+    it("should retry on fetch failure", async () => {
+      mockGrab
+        .mockRejectedValueOnce(new Error("Network error"))
+        .mockResolvedValueOnce({
+          results: [
+            { title: "Retry Success", url: "https://example.com", content: "Content", score: 0.9 },
+          ],
+          suggestions: [],
+          infoboxes: [],
+        });
+
+      const result = await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+        maxRetries: 3,
+      });
+
+      expect(mockGrab).toHaveBeenCalledTimes(2);
+      if (!Array.isArray(result)) {
+        expect(result.results[0].title).toBe("Retry Success");
+      }
+    });
+
+    it("should retry on empty results", async () => {
+      mockGrab
+        .mockResolvedValueOnce("") // Empty HTML
+        .mockResolvedValueOnce(`
+          <article class="result">
+            <h3><a href="https://example.com">Retry Result</a></h3>
+            <p class="content">Found on retry</p>
+          </article>
+        `);
+
+      const result = await searchWeb("test", {
+        privateSearxng: false,
+        maxRetries: 2,
+      });
+
+      expect(mockGrab).toHaveBeenCalledTimes(2);
+      if (Array.isArray(result)) {
+        expect(result[0].title).toBe("Retry Result");
+      }
+    });
+
+    it("should return empty array after exhausting retries", async () => {
+      mockGrab.mockRejectedValue(new Error("Network error"));
+
+      const result = await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+        maxRetries: 2,
+      });
+
+      expect(mockGrab).toHaveBeenCalledTimes(3); // Initial + 2 retries
+      expect(result).toEqual([]);
+    });
+  });
+
+  describe("Search parameters", () => {
+    it("should apply category filter", async () => {
+      mockGrab.mockResolvedValueOnce({
+        results: [],
+        suggestions: [],
+        infoboxes: [],
+      });
+
+      await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+        category: "news",
+      });
+
+      expect(mockGrab).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          category_news: 1,
+        })
+      );
+    });
+
+    it("should apply recency filter", async () => {
+      mockGrab.mockResolvedValueOnce({
+        results: [],
+        suggestions: [],
+        infoboxes: [],
+      });
+
+      await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+        recency: "week",
+      });
+
+      expect(mockGrab).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          time_range: "week",
+        })
+      );
+    });
+
+    it("should handle pagination", async () => {
+      mockGrab.mockResolvedValueOnce({
+        results: [],
+        suggestions: [],
+        infoboxes: [],
+      });
+
+      await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+        page: 3,
+      });
+
+      expect(mockGrab).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          pageno: 3,
+        })
+      );
+    });
+
+    it("should apply safesearch", async () => {
+      mockGrab.mockResolvedValueOnce({
+        results: [],
+        suggestions: [],
+        infoboxes: [],
+      });
+
+      await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+        safesearch: true,
+      });
+
+      expect(mockGrab).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          safesearch: "1",
+        })
+      );
+    });
+
+    it("should use custom language", async () => {
+      mockGrab.mockResolvedValueOnce({
+        results: [],
+        suggestions: [],
+        infoboxes: [],
+      });
+
+      await searchWeb("test", {
+        privateSearxng: "https://search.example.com",
+        lang: "es",
+      });
+
+      expect(mockGrab).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          language: "es",
+        })
+      );
+    });
+  });
+});
+
+describe("searchSearxng", () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it("should adapt options to searchWeb", async () => {
+    mockGrab.mockResolvedValueOnce({
+      results: [{ title: "Test", url: "https://example.com", content: "Content", score: 0.9 }],
+      suggestions: ["test1"],
+      infoboxes: [],
+    });
+
+    const result = await searchSearxng("test query", {
+      categories: ["news"],
+      pageno: 2,
+      language: "fr",
+    });
+
+    expect(result).toHaveProperty("results");
+    expect(result).toHaveProperty("suggestions");
+    expect(result.results).toHaveLength(1);
+    expect(result.suggestions).toHaveLength(1);
+  });
+
+  it("should handle array results from searchWeb", async () => {
+    mockGrab.mockResolvedValueOnce(`
+      <article class="result">
+        <h3><a href="https://example.com">Test</a></h3>
+        <p class="content">Content</p>
+      </article>
+    `);
+
+    const result = await searchSearxng("test", { categories: ["general"] });
+
+    expect(result.results).toHaveLength(1);
+    expect(result.suggestions).toEqual([]);
+  });
+});

package/src/search/public-searxng.ts

@@ -0,0 +1,433 @@
+/**
+ * @module search-web-api/public-searxng
+ * @description SearXNG metasearch via public or private instances.
+ */
+import { getDomainWithoutSuffix } from "tldts";
+import { parseDate } from "chrono-node";
+import grab from "grab-url";
+
+/**
+ * Search Web via SearXNG metasearch of all major search engines.
+ */
+export async function searchWeb(
+  query: string,
+  options: SearchOptions = {},
+): Promise<SearxngSearchResult[] | SearchResponse> {
+  const {
+    category = "general",
+    recency,
+    privateSearxng = null,
+    maxRetries = 3,
+    page = 1,
+    safesearch = false,
+    lang = "en-US",
+    proxy = null,
+  } = options;
+
+  const CATEGORY_LIST = [
+    "general",
+    "news",
+    "videos",
+    "images",
+    "science",
+    "it",
+    "files",
+    "social+media",
+  ];
+  const RECENCY_ALLOWED_LIST = ["day", "week", "month", "year"];
+
+  const SEARX_DOMAINS = [
+    "baresearch.org",
+    "copp.gg",
+    "darmarit.org",
+    "etsi.me",
+    "fairsuch.net",
+    "nogoo.me",
+    "northboot.xyz",
+    "nyc1.sx.ggtyler.dev",
+    "ooglester.com",
+    "opnxng.com",
+    "paulgo.io",
+    "priv.au",
+    "s.trung.fun",
+    "search.blitzw.in",
+    "search.charliewhiskey.net",
+    "search.citw.lgbt",
+    "search.darkness.services",
+    "search.datura.network",
+    "search.dotone.nl",
+    "search.gcomm.ch",
+    "search.hbubli.cc",
+    "search.im-in.space",
+    "search.incogniweb.net",
+    "search.inetol.net",
+    "search.leptons.xyz",
+    "search.nadeko.net",
+    "search.ngn.tf",
+    "search.ononoki.org",
+    "search.privacyredirect.com",
+    "search.sapti.me",
+    "search.rowie.at",
+    "search.projectsegfau.lt",
+    "search.tommy-tran.com",
+    "searx.aleteoryx.me",
+    "searx.ankha.ac",
+    "searx.be",
+    "searx.colbster937.dev",
+    "searx.daetalytica.io",
+    "searx.dresden.network",
+    "searx.foss.family",
+    "searx.hu",
+    "searx.juancord.xyz",
+    "searx.lunar.icu",
+    "searx.mxchange.org",
+    "searx.namejeff.xyz",
+    "searx.oakleycord.dev",
+    "searx.ro",
+    "searx.sev.monster",
+    "searx.thefloatinglab.world",
+    "searx.tiekoetter.com",
+    "searx.tuxcloud.net",
+    "searx.work",
+    "searx.zhenyapav.com",
+    "searxng.hweeren.com",
+    "searxng.online",
+    "searxng.shreven.org",
+    "searxng.site",
+    "skyrimhater.com",
+    "sx.ca.zorby.top",
+    "sx.catgirl.cloud",
+    "sx.thatxtreme.dev",
+    "sx.zorby.top",
+    "xo.wtf",
+  ];
+
+  //select a random domain if none is provided
+  const searchDomain =
+    privateSearxng ||
+    "https://" +
+      SEARX_DOMAINS[Math.floor(Math.random() * SEARX_DOMAINS.length)];
+
+  const categoryName = category === "tech" ? "it" : category;
+
+  let url = `${searchDomain}/search`;
+
+  if (privateSearxng) url += "&format=json";
+
+  //on cloudflare to avoid "Too many redirects" change SSL mode to Full
+  if (proxy && !privateSearxng) url = proxy + url;
+
+  let resultHTML: any;
+  try {
+    const params: Record<string, any> = {
+      q: encodeURIComponent(query),
+      ["category_" + categoryName]: 1,
+      language: lang,
+      safesearch: safesearch ? "1" : "0",
+      pageno: page,
+      headers: {
+        "accept-language": lang + ",en;q=0.9",
+      },
+    };
+    if (privateSearxng) params.format = "json";
+    if (recency && RECENCY_ALLOWED_LIST.includes(recency)) params.time_range = recency;
+
+    resultHTML = await grab(searchDomain + "/search", params);
+  } catch (error: any) {
+    const errorMsg = error instanceof Error ? error.message : String(error);
+    console.warn(`[searchWeb] Failed to fetch from SearXNG domain "${searchDomain}": ${errorMsg}`);
+    if (maxRetries > 0) {
+      console.log(`[searchWeb] Retrying with another instance... (${maxRetries} retries left)`);
+      return await searchWeb(query, {
+        ...options,
+        maxRetries: maxRetries - 1,
+      });
+    }
+    console.error(`[searchWeb] All retries exhausted. Returning empty results.`);
+    return [];
+  }
+
+  if (privateSearxng) {
+    let parsedData: any;
+
+    // Check if resultHTML is already an object (grab-url auto-parsed JSON)
+    if (typeof resultHTML === "object" && resultHTML !== null) {
+      parsedData = resultHTML;
+    } else if (typeof resultHTML === "string") {
+      // It's a string, try to parse it
+      if (!resultHTML.startsWith("{")) {
+        console.warn(
+          "Private SearXNG instance did not return valid JSON, falling back or returning empty",
+        );
+        return { results: [], suggestions: [], infoboxes: [] };
+      }
+
+      try {
+        parsedData = JSON.parse(resultHTML);
+      } catch (e) {
+        console.error("Failed to parse JSON from private instance", e);
+        return { results: [], suggestions: [], infoboxes: [] };
+      }
+    } else {
+      console.error("Unexpected resultHTML type:", typeof resultHTML);
+      return { results: [], suggestions: [], infoboxes: [] };
+    }
+
+    let { results, suggestions, infoboxes } = parsedData;
+
+    results = results.map((result: any) => {
+      let title = result.title.replace(/<\/?[^>]+(>|$)/g, "");
+
+      const TITLE_SPLITTERS_RE = /( [|\-\/:\u00bb] )|( - )|(\|)/;
+
+      if (TITLE_SPLITTERS_RE.test(title)) {
+        const splitTitle = title.split(TITLE_SPLITTERS_RE);
+        // Handle breadcrumbed titles
+        if (splitTitle.length >= 2) {
+          const longestPart = splitTitle.reduce(
+            (acc: string, part: string) =>
+              part?.length > acc?.length ? part : acc,
+            "",
+          );
+          if (longestPart.length > 10) {
+            title = longestPart;
+          }
+        }
+      }
+
+      title = convertURLSafeHTMLToHTML(title);
+      let urlPtr = result.url.replace(/&amp;/g, "&");
+
+      // Validate URL has path component, not just domain
+      try {
+        const parsedUrl = new URL(urlPtr);
+        // If URL is just domain (path is just "/"), log warning
+        if (parsedUrl.pathname === "/" || parsedUrl.pathname === "") {
+          console.warn(`[searchWeb] Result URL is domain-only: ${urlPtr}. Full result:`, JSON.stringify(result).slice(0, 200));
+        }
+      } catch (e) {
+        console.error(`[searchWeb] Invalid URL in result: ${urlPtr}`);
+      }
+
+      const snippet = result.content?.replace(/<\/?[^>]+(>|$)/g, "");
+      const thumbnail = result.thumbnail;
+      const score = Math.round(result.score * 100) / 100;
+
+      const domain = result.url
+        ?.replace(/(http:\/\/|https:\/\/|www.)/gi, "")
+        .split("/")[0];
+
+      let date: string | undefined = undefined;
+      let source: string | undefined = undefined;
+
+      if (typeof result.metadata === "string") {
+        const parts = result.metadata.split("|").map((s: string) => s.trim());
+        if (parts.length > 1) {
+          // Basic check
+          const dateObj = parseDate(result.metadata);
+          date = dateObj ? dateObj.toISOString().split("T")[0] : undefined;
+          const sourcePart = parts[1]; // assuming second part might be source
+          source = sourcePart || null;
+        }
+      }
+
+      if (!source && domain) {
+        source =
+          getDomainWithoutSuffix(domain)?.replace(/\b\w/g, (l) =>
+            l.toUpperCase(),
+          ) || undefined;
+        if (source && source.length < 5) source = source.toUpperCase();
+      }
+
+      const favicon = `https://s2.googleusercontent.com/s2/favicons?domain_url=${result.url}`;
+      // const favicon =
+      //   "https://www.google.com/s2/favicons?domain=" +
+      //   result.url.match(
+      //     /^(?:https?:\/\/)?(?:www\.)?([^/:?\s]+)(?:[/:?]|$)/i,
+      //   )?.[0] +
+      //   "&sz=16";
+
+      return {
+        title,
+        url: urlPtr,
+        snippet,
+        score,
+        ...(date ? { date } : {}),
+        ...(source ? { source } : {}),
+        domain,
+        favicon,
+        // Compatibility fields
+        content: snippet,
+        thumbnail,
+        ...(result.img_src ? { img_src: result.img_src } : {}),
+        ...(result.iframe_src ? { iframe_src: result.iframe_src } : {}),
+      };
+    });
+    return { results, suggestions: suggestions || [], infoboxes };
+  }
+
+  // Public instance scraping (HTML parsing)
+  let results: SearxngSearchResult[] = [];
+  const resultRegex = /<article class="result[^>]*>[\s\S]*?<\/article>/g;
+  const titleUrlRegex = /<h3><a href="([^"]*)"[^>]*>(.*?)<\/a><\/h3>/;
+  const snippetRegex = /<p class="content">\s*(.*?)\s*<\/p>/;
+
+  // Unused in current logic but kept from original code for potential future use or completeness
+  // const enginesRegex = /<span>(bing|duckduckgo|yahoo|google)<\/span>/g;
+  // const linksRegex = /<a href="([^"]*)" class="(cache_link|proxyfied_link)"[^>]*>(cached|proxied)<\/a>/g;
+
+  let match;
+  while ((match = resultRegex.exec(resultHTML)) !== null) {
+    const resultHtml = match[0];
+    const titleUrlMatch = titleUrlRegex.exec(resultHtml);
+    const snippetMatch = snippetRegex.exec(resultHtml);
+
+    if (titleUrlMatch && titleUrlMatch[1] && titleUrlMatch[2]) {
+      // const urlFound = convertURLSafeHTMLToHTML(titleUrlMatch[1]); // Not used in original, seemingly
+      let title = titleUrlMatch[2].replace(/<\/?[^>]+(>|$)/g, "");
+      let snippet = snippetMatch
+        ? snippetMatch[1].replace(/<\/?[^>]+(>|$)/g, "")
+        : "";
+
+      title = convertURLSafeHTMLToHTML(title);
+      snippet = convertURLSafeHTMLToHTML(snippet);
+      const urlClean = convertURLSafeHTMLToHTML(titleUrlMatch[1]);
+
+      // Validate URL has path component, not just domain
+      try {
+        const parsedUrl = new URL(urlClean);
+        if (parsedUrl.pathname === "/" || parsedUrl.pathname === "") {
+          console.warn(`[searchWeb] Public scrape URL is domain-only: ${urlClean}`);
+        }
+      } catch (e) {
+        console.error(`[searchWeb] Invalid URL in public scrape: ${urlClean}`);
+      }
+
+      results.push({
+        title,
+        url: urlClean,
+        snippet,
+        content: snippet, // Compatibility
+      });
+    }
+  }
+
+  if (results.length === 0 && maxRetries > 0) {
+    return (await searchWeb(query, {
+      ...options,
+      maxRetries: maxRetries - 1,
+      useProxy: true,
+    })) as SearxngSearchResult[];
+  }
+
+  results = results.map((result) => {
+    const match = result.url.match(
+      /^(?:https?:\/\/)?(?:www\.)?([^/:?\s]+)(?:[/:?]|$)/i,
+    );
+    const domainStr = match ? match[0] : "";
+
+    const favicon = "https://www.google.com/s2/favicons?domain=" + domainStr;
+
+    const domain = result.url
+      ?.replace(/(http:\/\/|https:\/\/|www.)/gi, "")
+      .split("/")[0];
+
+    return {
+      ...result,
+      domain,
+      favicon,
+      thumbnail: favicon, // Compatibility
+    };
+  });
+
+  return results;
+}
+
+// Wrapper to match existing `searchSearxng` signature if needed elsewhere,
+// OR the user might want this to be the primary `searchWeb` and we just export `searchSearxng` that calls it.
+// The user's request showed `searchWeb` being imported.
+// But the application likely calls `searchSearxng`. Let's reimplement `searchSearxng` to use `searchWeb`.
+
+interface SearxngSearchOptions {
+  categories?: string[];
+  engines?: string[];
+  language?: string;
+  pageno?: number;
+}
+
+export const searchSearxng = async (
+  query: string,
+  opts?: SearxngSearchOptions,
+): Promise<{ results: SearxngSearchResult[]; suggestions: string[] }> => {
+  // Adapter to call the new searchWeb
+  const category = opts?.categories?.[0] || "general"; // simplistic mapping
+  const page = opts?.pageno || 1;
+  const lang = opts?.language || "en-US";
+
+  const result = await searchWeb(query, {
+    category,
+    page,
+    lang,
+    // privateSearxng: true // or false? The user code said "use custom or false to use the public instances"
+    // Let's rely on the default behavior or what `searchWeb` does.
+    // However, `searchWeb` logic branches on `privateSearxng` significantly.
+    // If we want JSON, we probably want `privateSearxng` set to a domain if we have one, or handle the array return.
+
+    // IMPORTANT: The user code's `GET` handler passes `privateSearxng: publicInstances ? false : searxngDomain`.
+    // where `searxngDomain` was imported from `customize-site`.
+    // Since we don't have that file, we used empty string defaults.
+    // If `searxngDomain` is falsy, `privateSearxng` becomes falsy (or we should be careful).
+  });
+
+  if (Array.isArray(result)) {
+    return { results: result, suggestions: [] };
+  } else {
+    return { results: result.results, suggestions: result.suggestions || [] };
+  }
+};
+
+// Helper function to decode HTML entities
+function convertURLSafeHTMLToHTML(html: string): string {
+  if (!html) return "";
+  return html
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'");
+}
+
+interface SearchOptions {
+  category?: string | number;
+  recency?: string;
+  privateSearxng?: string | boolean | null;
+  maxRetries?: number;
+  page?: number;
+  safesearch?: boolean;
+  lang?: string;
+  proxy?: string | null;
+  useProxy?: boolean;
+}
+
+export interface SearxngSearchResult {
+  title: string;
+  url: string;
+  snippet?: string;
+  domain?: string;
+  favicon?: string;
+  score?: number;
+  source?: string;
+  date?: string;
+  img_src?: string; // Added for compatibility with existing interfaces
+  thumbnail_src?: string; // Added for compatibility
+  thumbnail?: string; // Added for compatibility
+  content?: string; // Added for compatibility
+  author?: string; // Added for compatibility
+  iframe_src?: string; // Added for compatibility
+}
+
+export interface SearchResponse {
+  results: SearxngSearchResult[];
+  suggestions: string[];
+  infoboxes?: any[];
+}

package/src/search/tavily.ts

@@ -0,0 +1,99 @@
+/**
+ * @module search-web-api/tavily
+ * @description Tavily search API integration.
+ */
+
+interface TavilySearchOptions {
+  searchDepth?: "basic" | "advanced";
+  maxResults?: number;
+  includeDomains?: string[];
+  excludeDomains?: string[];
+}
+
+interface TavilySearchResult {
+  title: string;
+  url: string;
+  content: string;
+  score: number;
+  raw_content?: string;
+}
+
+interface TavilyResponse {
+  results: TavilySearchResult[];
+  query: string;
+}
+
+export const searchTavily = async (
+  query: string,
+  opts?: TavilySearchOptions,
+): Promise<{ results: TavilySearchResult[]; suggestions: string[] }> => {
+  const tavilyApiKey =
+    (typeof process !== "undefined" ? process.env.TAVILY_API_KEY : "") || "";
+
+  if (!tavilyApiKey) {
+    throw new Error(
+      "Tavily API key not configured. Set TAVILY_API_KEY environment variable.",
+    );
+  }
+
+  let sanitizedQuery = query.trim();
+  if (sanitizedQuery.length > 400) {
+    console.warn(`[Tavily] Query too long (${sanitizedQuery.length} chars), truncating to 400 chars`);
+    sanitizedQuery = sanitizedQuery.slice(0, 400);
+  }
+
+  try {
+    const response = await fetch("https://api.tavily.com/search", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        api_key: tavilyApiKey,
+        query: sanitizedQuery,
+        search_depth: opts?.searchDepth || "basic",
+        max_results: opts?.maxResults || 10,
+        include_domains: opts?.includeDomains || [],
+        exclude_domains: opts?.excludeDomains || [],
+        include_answer: false,
+        include_raw_content: false,
+      }),
+    });
+
+    if (!response.ok) {
+      const errorBody = await response.text();
+      throw new Error(`HTTP ${response.status}: ${errorBody}`);
+    }
+
+    const data = (await response.json()) as TavilyResponse;
+    const results = data.results || [];
+
+    return {
+      results: results.map((r) => ({
+        title: r.title,
+        url: r.url,
+        content: r.content,
+        score: r.score,
+        raw_content: r.raw_content,
+      })),
+      suggestions: [],
+    };
+  } catch (error: any) {
+    console.error("Tavily search error:", error);
+
+    if (error.message?.includes("400")) {
+      console.error("Tavily 400 error - Query length:", sanitizedQuery.length);
+      console.error("Tavily 400 error - Query preview:", sanitizedQuery.slice(0, 200));
+    }
+
+    throw new Error(
+      `Tavily search failed: ${error.message}`,
+    );
+  }
+};
+
+export const getTavilyApiKey = () =>
+  (typeof process !== "undefined" ? process.env.TAVILY_API_KEY : "") || "";
+
+export const isTavilyConfigured = () => {
+  const apiKey = getTavilyApiKey();
+  return apiKey && apiKey.length > 0;
+};