search-web-api 1.0.13

package/src/sources/general/duckduckgo.ts

@@ -0,0 +1,49 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const duckduckgo: EngineFunction = async (
+  query: string,
+  page?: number
+) => {
+  const response = await fetch("https://duckduckgo.com/html", {
+    method: "POST",
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+      "Content-Type": "application/x-www-form-urlencoded",
+      "Accept-Language": "en-US,en;q=0.9",
+    },
+    body: `q=${encodeURIComponent(query)}&b=&kl=us-en`,
+  });
+  if (!response.ok) return [];
+  const htmlString = await response.text();
+  const { document } = parseHTML(htmlString);
+
+  const snippets = document.querySelectorAll(".result");
+  console.log(`DDG: Found ${snippets.length} results`);
+
+  const data = Array.from(snippets)
+    .map((element) => {
+      const link = element.querySelector(".result__title a");
+      const url = link?.getAttribute("href") || "";
+      const title = link?.textContent?.trim() || "";
+      const content =
+        element.querySelector(".result__snippet")?.textContent?.trim() ||
+        "";
+
+      return {
+        url,
+        title,
+        content,
+        engine: "duckduckgo",
+      };
+    })
+    .filter((r) => r.url && r.title);
+
+  console.log(`DDG: Extracted ${data.length} valid results`);
+  if (data.length === 0 && htmlString.length > 0) {
+    console.log("DDG: HTML sample:", htmlString.substring(0, 1000));
+  }
+
+  return data;
+};

package/src/sources/general/google.ts

@@ -0,0 +1,68 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const google: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    start: String(((page || 1) - 1) * 10),
+    gbv: "1",
+  });
+  const response = await fetch(`https://www.google.com/search?${params}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+      "Accept-Language": "en-US,en;q=0.5",
+      Cookie: "CONSENT=YES+; SOCS=CAESBQgYEgAgAA==",
+    },
+  });
+  if (!response.ok) return [];
+  const htmlString = await response.text();
+
+  if (!htmlString || typeof htmlString !== "string") {
+    return [];
+  }
+
+  const { document } = parseHTML(htmlString);
+
+  // Google's classes change frequently. Let's try to be more robust.
+  const snippets = document.querySelectorAll(".Gx5Zad.fP1Qef.xpd.EtOod.pkphOe");
+  console.log(`Google: Found ${snippets.length} snippets with main selector`);
+
+  const data = Array.from(snippets)
+    .map((element) => {
+      const link = element.querySelector("a");
+      const url = link?.getAttribute("href");
+
+      let realUrl = url;
+      if (url && url.startsWith("/url?q=")) {
+        realUrl = url.split("/url?q=")[1].split("&")[0];
+        realUrl = decodeURIComponent(realUrl);
+      }
+
+      const title =
+        element.querySelector(".BNeawe.vvjwJb.AP7Wnd")?.textContent?.trim() ||
+        "";
+      const content =
+        element.querySelector(".BNeawe.s3v9rd.AP7Wnd")?.textContent?.trim() ||
+        "";
+
+      return {
+        url: realUrl,
+        title,
+        content,
+        engine: "google",
+      };
+    })
+    .filter((r) => r.url && r.title);
+
+  console.log(`Google: Extracted ${data.length} valid results`);
+  if (data.length === 0 && htmlString.length > 0) {
+    console.log("Google: HTML sample:", htmlString.substring(0, 1000));
+  }
+
+  return data;
+};

package/src/sources/general/mojeek.ts

@@ -0,0 +1,56 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const mojeek: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    s: String(10 * ((page || 1) - 1)),
+    safe: "0",
+  });
+  const response = await fetch(`https://www.mojeek.com/search?${params}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+      "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+      "Accept-Language": "en-US,en;q=0.9",
+      "Referer": "https://www.mojeek.com/",
+    },
+  });
+  if (!response.ok) return [];
+  const htmlString = await response.text();
+
+  if (!htmlString || typeof htmlString !== "string") {
+    return [];
+  }
+
+  const { document } = parseHTML(htmlString);
+
+  const snippets = document.querySelectorAll("ul.results-standard li");
+  console.log(`Mojeek: Found ${snippets.length} snippets`);
+
+  const data = Array.from(snippets)
+    .map((element) => {
+      const mainLink = element.querySelector("a.ob");
+      const url = mainLink?.getAttribute("href") || "";
+      const title = element.querySelector("h2 a")?.textContent?.trim() || "";
+      const content = element.querySelector("p.s")?.textContent?.trim() || "";
+
+      return {
+        url,
+        title,
+        content,
+        engine: "mojeek",
+      };
+    })
+    .filter((r) => r.url && r.title);
+
+  console.log(`Mojeek: Extracted ${data.length} valid results`);
+  if (data.length === 0 && htmlString.length > 0) {
+    console.log("Mojeek: HTML sample:", htmlString.substring(0, 1000));
+  }
+
+  return data;
+};

package/src/sources/general/qwant.ts

@@ -0,0 +1,37 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const qwant: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    count: "10",
+    offset: String(((page || 1) - 1) * 10),
+    locale: "en_US",
+  });
+  const response = await fetch(`https://api.qwant.com/v3/search/web?${params}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+    },
+  });
+  if (!response.ok) return [];
+  const jsonData = await response.json();
+
+  if (
+    !jsonData ||
+    !jsonData.data ||
+    !jsonData.data.result ||
+    !jsonData.data.result.items
+  ) {
+    return [];
+  }
+
+  return jsonData.data.result.items.map((item: any) => ({
+    url: item.url,
+    title: item.title,
+    content: item.desc,
+    engine: "qwant",
+  }));
+};

package/src/sources/general/startpage.ts

@@ -0,0 +1,44 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const startpage: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const formData = new URLSearchParams();
+  formData.append("query", query);
+  formData.append("page", String(page || 1));
+
+  const response = await fetch("https://www.startpage.com/sp/search", {
+    method: "POST",
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+      "Content-Type": "application/x-www-form-urlencoded",
+    },
+    body: formData.toString(),
+  });
+  if (!response.ok) return [];
+  const htmlString = await response.text();
+  const { document } = parseHTML(htmlString);
+
+  return Array.from(
+    document.querySelectorAll(".w-gl__result")
+  )
+    .map((element) => {
+      const link = element.querySelector(".w-gl__result-title");
+      const url = link?.getAttribute("href") || "";
+      const title = link?.querySelector("h3")?.textContent?.trim() || "";
+      const content =
+        element.querySelector(".w-gl__description")?.textContent?.trim() ||
+        "";
+
+      return {
+        url,
+        title,
+        content,
+        engine: "startpage",
+      };
+    })
+    .filter((r) => r.url && r.title);
+};

package/src/sources/general/yahoo.ts

@@ -0,0 +1,41 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const yahoo: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    p: query,
+    b: String(((page || 1) - 1) * 7 + 1),
+  });
+  const response = await fetch(`https://search.yahoo.com/search?${params}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+      Accept:
+        "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+      "Accept-Language": "en-US,en;q=0.9",
+    },
+  });
+  if (!response.ok) return [];
+  const htmlString = await response.text();
+  const { document } = parseHTML(htmlString);
+
+  return Array.from(document.querySelectorAll(".algo-sr"))
+    .map((element) => {
+      const link = element.querySelector("a");
+      const url = link?.getAttribute("href") || "";
+      const title = link?.textContent?.trim() || "";
+      const content =
+        element.querySelector(".compText")?.textContent?.trim() || "";
+
+      return {
+        url,
+        title,
+        content,
+        engine: "yahoo",
+      };
+    })
+    .filter((r) => r.url && r.title);
+};

package/src/sources/general/yandex.ts

@@ -0,0 +1,56 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const yandex: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    tmpl_version: "releases",
+    text: query,
+    web: "1",
+    frame: "1",
+    searchid: "3131712",
+    lang: "en",
+    ...((page || 1) > 1 ? { p: String((page || 1) - 1) } : {}),
+  });
+  const response = await fetch(`https://yandex.com/search/site/?${params}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+      Accept:
+        "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+      "Accept-Language": "en-US,en;q=0.5",
+      Cookie: "yp=1716337604.sp.family%3A0#1685406411.szm.1:1920x1080:1920x999",
+    },
+  });
+  if (!response.ok) return [];
+  const htmlString = await response.text();
+  const { document } = parseHTML(htmlString);
+
+  return Array.from(
+    document.querySelectorAll('li.serp-item, li[class*="serp-item"]')
+  )
+    .map((element) => {
+      const link = element.querySelector(
+        'a.b-serp-item__title-link, a[class*="title-link"]'
+      );
+      const url = link?.getAttribute("href") || "";
+      const title =
+        link?.querySelector("span, h3")?.textContent?.trim() || "";
+      const content =
+        element
+          .querySelector(
+            'div.b-serp-item__text, div[class*="text"]'
+          )
+          ?.textContent?.trim() || "";
+
+      return {
+        url,
+        title,
+        content,
+        engine: "yandex",
+      };
+    })
+    .filter((r) => r.url && r.title);
+};

package/src/sources/images/bing_images.ts

@@ -0,0 +1,68 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const bing_images: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    async: "1",
+    first: String(((page || 1) - 1) * 35 + 1),
+    count: "35",
+  });
+  const response = await fetch(
+    `https://www.bing.com/images/async?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        Accept:
+          "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const html = await response.text();
+  const { document } = parseHTML(html);
+
+  return Array.from(
+    document.querySelectorAll(
+      'ul.dgControl_list li, li[class*="dgControl"]'
+    )
+  )
+    .map((element) => {
+      const metadataStr = element
+        .querySelector("a.iusc")
+        ?.getAttribute("m");
+      if (!metadataStr) return null;
+
+      try {
+        const metadata = JSON.parse(metadataStr);
+        const title =
+          element.querySelector("div.infnmpt a")?.textContent?.trim() ||
+          metadata.t ||
+          "";
+        const imgFormat =
+          element
+            .querySelector("div.imgpt div span")
+            ?.textContent?.trim() || "";
+        const source =
+          element
+            .querySelector("div.imgpt div.lnkw a")
+            ?.textContent?.trim() || "";
+
+        return {
+          url: metadata.purl,
+          title,
+          content: metadata.desc || source || "",
+          thumbnail: metadata.turl,
+          engine: "bing_images",
+        };
+      } catch (e) {
+        return null;
+      }
+    })
+    .filter((r) => r !== null);
+};

package/src/sources/images/deviantart.ts

@@ -0,0 +1,71 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const deviantart: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({ q: query });
+  const response = await fetch(
+    `https://www.deviantart.com/search?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const html = await response.text();
+
+  if (!html || typeof html !== "string") {
+    return [];
+  }
+
+  const { document } = parseHTML(html);
+
+  return Array.from(
+    document.querySelectorAll("div.V_S0t_ > div > div > a")
+  )
+    .map((element) => {
+      const premiumText = element.parentElement
+        ?.querySelector("div div div")
+        ?.textContent;
+      if (premiumText && premiumText.includes("Watch the artist to view")) {
+        return null;
+      }
+
+      const url = element.getAttribute("href");
+      const title = element.getAttribute("aria-label");
+      const thumbnail =
+        element.querySelector("div img")?.getAttribute("src") || undefined;
+
+      let imgSrc = element.querySelector("div img")?.getAttribute("srcset");
+      if (imgSrc) {
+        imgSrc = imgSrc.split(" ")[0];
+        try {
+          const imgUrl = new URL(imgSrc);
+          const pathParts = imgUrl.pathname.split("/v1");
+          if (pathParts.length > 0) {
+            imgUrl.pathname = pathParts[0];
+            imgSrc = imgUrl.toString();
+          }
+        } catch (e) {
+          // If URL parsing fails, keep original
+        }
+      }
+
+      if (!url || !title) {
+        return null;
+      }
+
+      return {
+        url,
+        title,
+        content: "",
+        engine: "deviantart",
+        thumbnail: imgSrc || thumbnail,
+      };
+    })
+    .filter((r) => r !== null);
+};

package/src/sources/images/flickr.ts

@@ -0,0 +1,132 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const flickr: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    text: query,
+    page: String(page || 1),
+  });
+  const response = await fetch(
+    `https://www.flickr.com/search?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        Accept:
+          "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const html = await response.text();
+
+  if (!html || typeof html !== "string") {
+    return [];
+  }
+
+  try {
+    const modelExportMatch = html.match(/modelExport:\s*({[\s\S]*?}),\s*[\n\r]/);
+    if (!modelExportMatch) {
+      console.log("Flickr: modelExport not found in HTML");
+      if (html.length > 0) {
+        console.log("Flickr: HTML sample:", html.substring(0, 1000));
+      }
+      return [];
+    }
+
+    const modelExport = JSON.parse(modelExportMatch[1]);
+    console.log("Flickr: Parsed modelExport successfully");
+
+    if (!modelExport.legend || !modelExport.legend[0]) {
+      return [];
+    }
+
+    const legend = modelExport.legend;
+
+    const data = legend
+      .map((index: any) => {
+        if (index.length !== 8) {
+          return null;
+        }
+
+        try {
+          const photo =
+            modelExport.main[index[0]][parseInt(index[1])][index[2]][
+              index[3]
+            ][index[4]][index[5]][parseInt(index[6])][index[7]];
+
+          const title = photo.title || "";
+          const description = photo.description || "";
+          const author = photo.realname || photo.username || "";
+
+          const imageSizes = [
+            "o",
+            "k",
+            "h",
+            "b",
+            "c",
+            "z",
+            "m",
+            "n",
+            "t",
+            "q",
+            "s",
+          ];
+          let sizeData = null;
+
+          for (const size of imageSizes) {
+            if (photo.sizes?.data?.[size]?.data) {
+              sizeData = photo.sizes.data[size].data;
+              break;
+            }
+          }
+
+          if (!sizeData) {
+            return null;
+          }
+
+          const imgSrc = sizeData.url;
+          const resolution = `${sizeData.width} x ${sizeData.height}`;
+
+          let thumbnail = imgSrc;
+          if (photo.sizes?.data?.n?.data?.url) {
+            thumbnail = photo.sizes.data.n.data.url;
+          } else if (photo.sizes?.data?.z?.data?.url) {
+            thumbnail = photo.sizes.data.z.data.url;
+          }
+
+          const url = photo.ownerNsid
+            ? `https://www.flickr.com/photos/${photo.ownerNsid}/${photo.id}`
+            : imgSrc;
+
+          const content = [
+            description,
+            author ? `Author: ${author}` : "",
+            `Resolution: ${resolution}`,
+          ]
+            .filter(Boolean)
+            .join("\n");
+
+          return {
+            url,
+            title,
+            content,
+            thumbnail,
+            engine: "flickr",
+          };
+        } catch (e) {
+          return null;
+        }
+      })
+      .filter((r: any) => r !== null);
+
+    console.log(`Flickr: Extracted ${data.length} valid results`);
+    return data;
+  } catch (e) {
+    console.error("Error parsing Flickr response:", e);
+    return [];
+  }
+};