search-web-api 1.0.124 → 1.0.126

package/README.md

@@ -26,6 +26,7 @@
     <a href="https://codespaces.new/vtempest/qwksearch-research-agent">
     <img src="https://github.com/codespaces/badge.svg" width="150" height="20" />
     </a>
+    <a href="https://codecov.io/gh/OpenSourceAGI/qwksearch-research-agent"><img src="https://codecov.io/gh/OpenSourceAGI/qwksearch-research-agent/graph/badge.svg?component=package-search-web-api" alt="Coverage" /></a>
 </p>
 
     

package/package.json

@@ -2,7 +2,7 @@
   "name": "search-web-api",
   "description": "70+ search engines across 10 categories and Scrape Extract API",
   "type": "module",
-  "version": "1.0.124",
+  "version": "1.0.126",
   "author": "vtempest",
   "license": "rights.institute/PROSPER",
   "repository": {

package/src/category-registry.ts

@@ -3,4 +3,4 @@
  * config map (with per-category scoring weights), and the global
  * `categoryRegistry` singleton from the canonical registry module.
  */
-export * from "./registry/search-engine-category-registry.js";
+export * from "./registry/search-engine-category-registry";

package/src/constants.ts

@@ -3,4 +3,4 @@
  * config module: CATEGORY_LIST, RECENCY_ALLOWED_LIST, SEARX_DOMAINS, and the
  * per-category engine directory used for documentation and UI rendering.
  */
-export * from "./config/search-engine-constants.js";
+export * from "./config/search-engine-constants";

package/src/engine-descriptions.ts

@@ -4,4 +4,4 @@
  * canonical registry module. Descriptions are sourced from SearXNG's
  * engine_descriptions.json and cover all 75+ supported engines.
  */
-export * from "./registry/search-engine-descriptions.js";
+export * from "./registry/search-engine-descriptions";

package/src/engine-status.ts

@@ -4,4 +4,4 @@
  * The tracker records per-engine success/failure counts, response times,
  * and health status, and gates which engines are eligible to run.
  */
-export * from "./registry/search-engine-status-tracker.js";
+export * from "./registry/search-engine-status-tracker";

package/src/engine.ts

@@ -3,4 +3,4 @@
  * and the `extractResponseData` helper from the canonical types module. All
  * engine source adapters import from `types/search-engine-interface` directly.
  */
-export * from "./types/search-engine-interface.js";
+export * from "./types/search-engine-interface";

package/src/registry/search-engine-category-registry.ts

@@ -5,8 +5,8 @@
  * system, it supports multi-category searches and weighted result combination.
  */
 
-import { Engine } from "../types/search-engine-interface.js";
-import { CategoryConfig } from "../types/search-result-types.js";
+import { Engine } from "../types/search-engine-interface";
+import { CategoryConfig } from "../types/search-result-types";
 
 export const CATEGORIES: { [key: string]: CategoryConfig } = {
   general: {

package/src/registry/search-engine-status-tracker.ts

@@ -6,7 +6,7 @@
  * their success rate exceeds twice their failure count.
  */
 
-import { EngineStatus, EngineLog } from "../types/search-result-types.js";
+import { EngineStatus, EngineLog } from "../types/search-result-types";
 
 export class EngineStatusTracker {
   private statusMap: Map<string, EngineStatus> = new Map();

package/src/result-container.ts

@@ -5,4 +5,4 @@
  * using position-weighted per-engine and per-category multipliers, and groups
  * the final list by result category.
  */
-export * from "./search/search-result-container.js";
+export * from "./search/search-result-container";

package/src/search/__tests__/public-searxng.test.ts

@@ -2,7 +2,7 @@
  * @fileoverview Unit tests for SearXNG search functionality
  */
 import { beforeEach, describe, expect, it, vi, type MockedFunction } from "vitest";
-import { searchWeb, searchSearxng } from "../public-searxng";
+import { searchWeb, searchSearxng, normalizeGrabResponse } from "../public-searxng";
 import grab from "grab-url";
 
 // Mock grab-url
@@ -547,3 +547,178 @@ describe("searchSearxng", () => {
     expect(result.suggestions).toEqual([]);
   });
 });
+
+describe("grab-url response shapes", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.spyOn(console, "warn").mockImplementation(() => {});
+    vi.spyOn(console, "error").mockImplementation(() => {});
+    vi.spyOn(console, "log").mockImplementation(() => {});
+  });
+
+  // grab-url resolves with `{ error }` instead of rejecting, so these bodies
+  // used to reach `parsedData.results.map` and throw a TypeError, which the
+  // API route reported as a 500.
+  it("returns an empty response when the private instance errors out", async () => {
+    mockGrab.mockResolvedValue({ error: "HTTP error: 429 Too Many Requests" } as any);
+
+    const result = await searchWeb("dd", {
+      privateSearxng: "https://search.example.com",
+      page: 2,
+      maxRetries: 6,
+    });
+
+    expect(result).toEqual({ results: [], suggestions: [], infoboxes: [] });
+    // No point hammering the same private host - the caller falls back.
+    expect(mockGrab).toHaveBeenCalledTimes(1);
+  });
+
+  it("returns an empty response when the private instance serves HTML", async () => {
+    mockGrab.mockResolvedValue({
+      data: "<html><body>Too many requests</body></html>",
+    } as any);
+
+    const result = await searchWeb("dd", {
+      privateSearxng: "https://search.example.com",
+      page: 2,
+    });
+
+    expect(result).toEqual({ results: [], suggestions: [], infoboxes: [] });
+  });
+
+  it("returns an empty response when the private instance JSON has no results", async () => {
+    mockGrab.mockResolvedValue({ isLoading: false, suggestions: [] } as any);
+
+    const result = await searchWeb("dd", {
+      privateSearxng: "https://search.example.com",
+    });
+
+    expect(result).toEqual({ results: [], suggestions: [], infoboxes: [] });
+  });
+
+  it("skips malformed results instead of throwing", async () => {
+    mockGrab.mockResolvedValue({
+      results: [
+        { url: "https://example.com/no-title" },
+        { title: "No URL at all" },
+        null,
+        {
+          title: "Good",
+          url: "https://example.com/good",
+          content: "Snippet",
+          score: "not-a-number",
+          metadata: "not a date | Source",
+        },
+      ],
+      suggestions: ["one"],
+    } as any);
+
+    const result = await searchWeb("dd", {
+      privateSearxng: "https://search.example.com",
+    });
+
+    expect(Array.isArray(result)).toBe(false);
+    if (!Array.isArray(result)) {
+      expect(result.results).toHaveLength(2);
+      expect(result.results[0].title).toBe("");
+      expect(result.results[1].title).toBe("Good");
+      expect(result.results[1].score).toBeUndefined();
+      expect(result.results[1].date).toBeUndefined();
+      expect(result.suggestions).toEqual(["one"]);
+    }
+  });
+
+  it("scrapes HTML that grab-url returned on the `data` field", async () => {
+    mockGrab.mockResolvedValue({
+      data: `
+        <article class="result">
+          <h3><a href="https://example.com/1">Wrapped Result</a></h3>
+          <p class="content">Wrapped snippet</p>
+        </article>
+      `,
+    } as any);
+
+    const result = await searchWeb("test", { privateSearxng: false });
+
+    expect(Array.isArray(result)).toBe(true);
+    if (Array.isArray(result)) {
+      expect(result).toHaveLength(1);
+      expect(result[0].title).toBe("Wrapped Result");
+      expect(result[0].url).toBe("https://example.com/1");
+    }
+  });
+
+  it("tries another public instance when one errors out", async () => {
+    mockGrab
+      .mockResolvedValueOnce({ error: "HTTP error: 502 Bad Gateway" } as any)
+      .mockResolvedValueOnce(`
+        <article class="result">
+          <h3><a href="https://example.com/2">Second Instance</a></h3>
+          <p class="content">Snippet</p>
+        </article>
+      ` as any);
+
+    const result = await searchWeb("test", { privateSearxng: false, maxRetries: 3 });
+
+    expect(mockGrab).toHaveBeenCalledTimes(2);
+    if (Array.isArray(result)) {
+      expect(result[0].title).toBe("Second Instance");
+    }
+  });
+
+  it("gives up with an empty array when every public instance errors out", async () => {
+    mockGrab.mockResolvedValue({ error: "HTTP error: 502 Bad Gateway" } as any);
+
+    const result = await searchWeb("test", { privateSearxng: false, maxRetries: 2 });
+
+    expect(mockGrab).toHaveBeenCalledTimes(3); // initial + 2 retries
+    expect(result).toEqual([]);
+  });
+
+  it("passes the query unencoded and disables grab-url HTML parsing", async () => {
+    mockGrab.mockResolvedValueOnce({ results: [], suggestions: [] } as any);
+
+    await searchWeb("olympic games 2028", {
+      privateSearxng: "https://search.example.com",
+    });
+
+    // grab-url encodes GET params itself; pre-encoding turned spaces into
+    // "%2520" and searched for the literal escape sequence.
+    expect(mockGrab).toHaveBeenCalledWith(
+      "https://search.example.com/search",
+      expect.objectContaining({ q: "olympic games 2028", dom: false }),
+    );
+  });
+});
+
+describe("normalizeGrabResponse", () => {
+  it("reports missing and non-object responses as errors", () => {
+    expect(normalizeGrabResponse(null).error).toBeTruthy();
+    expect(normalizeGrabResponse(undefined).error).toBeTruthy();
+    expect(normalizeGrabResponse(42).error).toBeTruthy();
+  });
+
+  it("passes strings through as text", () => {
+    expect(normalizeGrabResponse("<html></html>")).toEqual({ text: "<html></html>" });
+  });
+
+  it("surfaces the error field grab-url sets on failure", () => {
+    expect(normalizeGrabResponse({ error: "HTTP error: 500" })).toEqual({
+      error: "HTTP error: 500",
+    });
+  });
+
+  it("treats a root-level results array as JSON", () => {
+    const raw = { results: [{ url: "https://example.com" }], isLoading: false };
+    expect(normalizeGrabResponse(raw)).toEqual({ json: raw });
+  });
+
+  it("unwraps text and JSON bodies carried on `data`", () => {
+    expect(normalizeGrabResponse({ data: "<html></html>" })).toEqual({
+      text: "<html></html>",
+    });
+
+    const nested = { results: [{ url: "https://example.com" }] };
+    expect(normalizeGrabResponse({ data: nested })).toEqual({ json: nested });
+  });
+});

package/src/search/public-searxng.ts

@@ -112,27 +112,33 @@ export async function searchWeb(
 
   let url = `${searchDomain}/search`;
 
-  if (privateSearxng) url += "&format=json";
-
   //on cloudflare to avoid "Too many redirects" change SSL mode to Full
   if (proxy && !privateSearxng) url = proxy + url;
 
-  let resultHTML: any;
+  let rawResponse: any;
   try {
     const params: Record<string, any> = {
-      q: encodeURIComponent(query),
+      // grab-url encodes GET params itself, so the query is passed raw here -
+      // pre-encoding it would double-encode spaces and symbols.
+      q: query,
       ["category_" + categoryName]: 1,
       language: lang,
       safesearch: safesearch ? "1" : "0",
       pageno: page,
+      // grab-url parses HTML responses into a DOM by default; the public
+      // instance path scrapes raw markup, so keep the body as text.
+      dom: false,
       headers: {
         "accept-language": lang + ",en;q=0.9",
+        accept: privateSearxng
+          ? "application/json, text/html;q=0.9"
+          : "text/html, application/xhtml+xml",
       },
     };
     if (privateSearxng) params.format = "json";
     if (recency && RECENCY_ALLOWED_LIST.includes(recency)) params.time_range = recency;
 
-    resultHTML = await grab(searchDomain + "/search", params);
+    rawResponse = await grab(url, params);
   } catch (error: any) {
     const errorMsg = error instanceof Error ? error.message : String(error);
     console.warn(`[searchWeb] Failed to fetch from SearXNG domain "${searchDomain}": ${errorMsg}`);
@@ -147,36 +153,54 @@ export async function searchWeb(
     return [];
   }
 
-  if (privateSearxng) {
-    let parsedData: any;
-
-    // Check if resultHTML is already an object (grab-url auto-parsed JSON)
-    if (typeof resultHTML === "object" && resultHTML !== null) {
-      parsedData = resultHTML;
-    } else if (typeof resultHTML === "string") {
-      // It's a string, try to parse it
-      if (!resultHTML.startsWith("{")) {
-        console.warn(
-          "Private SearXNG instance did not return valid JSON, falling back or returning empty",
-        );
-        return { results: [], suggestions: [], infoboxes: [] };
-      }
+  // grab-url resolves instead of rejecting when a request fails: network and
+  // HTTP errors come back as `{ error }` and non-JSON bodies as `{ data }`.
+  // Normalizing here keeps the parsing below from reading `.results` off an
+  // error object, which used to throw and surface as a 500 from the API route.
+  const response = normalizeGrabResponse(rawResponse);
 
-      try {
-        parsedData = JSON.parse(resultHTML);
-      } catch (e) {
-        console.error("Failed to parse JSON from private instance", e);
-        return { results: [], suggestions: [], infoboxes: [] };
-      }
-    } else {
-      console.error("Unexpected resultHTML type:", typeof resultHTML);
+  if (response.error) {
+    console.warn(
+      `[searchWeb] SearXNG domain "${searchDomain}" returned an error: ${response.error}`,
+    );
+
+    // Public instances are picked at random, so another one is worth a try.
+    // A private instance would just fail the same way; the caller falls back
+    // to the public instances instead.
+    if (!privateSearxng && maxRetries > 0) {
+      return (await searchWeb(query, {
+        ...options,
+        maxRetries: maxRetries - 1,
+      })) as SearxngSearchResult[];
+    }
+
+    return privateSearxng ? { results: [], suggestions: [], infoboxes: [] } : [];
+  }
+
+  if (privateSearxng) {
+    const parsedData = response.json ?? parseJSONText(response.text);
+
+    // A private instance can answer 200 with an HTML error/captcha page, or
+    // with JSON that has no `results` (rate limited, format=json disabled).
+    // Return an empty response so the caller can fall back to public
+    // instances instead of blowing up on `undefined.map`.
+    if (!parsedData || !Array.isArray(parsedData.results)) {
+      console.warn(
+        `[searchWeb] Private SearXNG instance "${searchDomain}" did not return JSON results (page ${page}).`,
+      );
       return { results: [], suggestions: [], infoboxes: [] };
     }
 
-    let { results, suggestions, infoboxes } = parsedData;
+    const { suggestions, infoboxes } = parsedData;
 
-    results = results.map((result: any) => {
-      let title = result.title.replace(/<\/?[^>]+(>|$)/g, "");
+    // Some engines return entries without a usable link; skip those rather
+    // than throwing while normalizing them.
+    const usableResults = parsedData.results.filter(
+      (result: any) => result && typeof result.url === "string" && result.url,
+    );
+
+    const results = usableResults.map((result: any) => {
+      let title = String(result.title ?? "").replace(/<\/?[^>]+(>|$)/g, "");
 
       const TITLE_SPLITTERS_RE = /( [|\-\/:\u00bb] )|( - )|(\|)/;
 
@@ -211,7 +235,9 @@ export async function searchWeb(
 
       const snippet = result.content?.replace(/<\/?[^>]+(>|$)/g, "");
       const thumbnail = result.thumbnail;
-      const score = Math.round(result.score * 100) / 100;
+      const score = Number.isFinite(result.score)
+        ? Math.round(result.score * 100) / 100
+        : undefined;
 
       const domain = result.url
         ?.replace(/(http:\/\/|https:\/\/|www.)/gi, "")
@@ -225,7 +251,10 @@ export async function searchWeb(
         if (parts.length > 1) {
           // Basic check
           const dateObj = parseDate(result.metadata);
-          date = dateObj ? dateObj.toISOString().split("T")[0] : undefined;
+          date =
+            dateObj && !Number.isNaN(dateObj.getTime())
+              ? dateObj.toISOString().split("T")[0]
+              : undefined;
           const sourcePart = parts[1]; // assuming second part might be source
           source = sourcePart || null;
         }
@@ -263,10 +292,16 @@ export async function searchWeb(
         ...(result.iframe_src ? { iframe_src: result.iframe_src } : {}),
       };
     });
-    return { results, suggestions: suggestions || [], infoboxes };
+
+    return {
+      results,
+      suggestions: suggestions || [],
+      infoboxes: infoboxes || [],
+    };
   }
 
   // Public instance scraping (HTML parsing)
+  const resultHTML = response.text ?? "";
   let results: SearxngSearchResult[] = [];
   const resultRegex = /<article class="result[^>]*>[\s\S]*?<\/article>/g;
   const titleUrlRegex = /<h3><a href="([^"]*)"[^>]*>(.*?)<\/a><\/h3>/;
@@ -386,6 +421,74 @@ export const searchSearxng = async (
   }
 };
 
+/**
+ * Normalized view of whatever `grab` handed back.
+ * Exactly one of `json`, `text` or `error` is meaningful.
+ */
+export interface NormalizedGrabResponse {
+  /** Parsed JSON body, when the response was JSON with search results. */
+  json?: any;
+  /** Raw body text, when the response was HTML or another text format. */
+  text?: string;
+  /** Message describing why the request did not produce a usable body. */
+  error?: string;
+}
+
+/**
+ * `grab` resolves rather than rejects on failure: network and HTTP errors come
+ * back as `{ error }`, JSON bodies are merged onto the root of the response
+ * object, and text/binary bodies are placed on `.data`. This flattens those
+ * shapes (plus the plain string a raw fetch would return) into one union so
+ * callers never read `.results` off an error object.
+ *
+ * @param raw The value returned by `grab`.
+ * @returns The body as JSON or text, or the error that prevented both.
+ */
+export function normalizeGrabResponse(raw: any): NormalizedGrabResponse {
+  if (raw === null || raw === undefined) return { error: "empty response" };
+  if (typeof raw === "string") return { text: raw };
+  if (typeof raw !== "object") return { error: `unexpected response type "${typeof raw}"` };
+
+  if (typeof raw.error === "string" && raw.error) return { error: raw.error };
+  if (Array.isArray(raw.results)) return { json: raw };
+
+  const { data } = raw;
+  if (typeof data === "string") return { text: data };
+  if (data && typeof data === "object") {
+    if (typeof data.error === "string" && data.error) return { error: data.error };
+    if (Array.isArray(data.results)) return { json: data };
+  }
+
+  // An object with neither results nor a body: hand it back as JSON and let
+  // the caller decide whether it is usable.
+  return { json: raw };
+}
+
+/**
+ * Parse a JSON object out of a response body, tolerating HTML error pages.
+ *
+ * @param text The raw response body, if there was one.
+ * @returns The parsed object, or null when the body was not a JSON object.
+ */
+function parseJSONText(text: string | undefined): any {
+  if (!text) return null;
+
+  const trimmed = text.trim();
+  if (!trimmed.startsWith("{")) {
+    console.warn(
+      "Private SearXNG instance did not return valid JSON, falling back or returning empty",
+    );
+    return null;
+  }
+
+  try {
+    return JSON.parse(trimmed);
+  } catch (e) {
+    console.error("Failed to parse JSON from private instance", e);
+    return null;
+  }
+}
+
 // Helper function to decode HTML entities
 function convertURLSafeHTMLToHTML(html: string): string {
   if (!html) return "";