search-web-api 1.0.13

package/src/sources/academic/core.ts

@@ -0,0 +1,65 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const core: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const apiKey = process.env.CORE_API_KEY;
+  if (!apiKey) return [];
+
+  const limit = 10;
+  const offset = ((page || 1) - 1) * limit;
+
+  const response = await fetch(
+    "https://api.core.ac.uk/v3/search/works",
+    {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`,
+        "User-Agent":
+          "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+      },
+      body: JSON.stringify({ q: query, limit, offset, scroll: false }),
+    }
+  );
+
+  if (!response.ok) return [];
+
+  const data = await response.json();
+
+  if (!data || !Array.isArray(data.results)) return [];
+
+  return data.results
+    .map((r: any) => {
+      const title = r.title || "";
+      const doi = r.doi || "";
+      const url =
+        r.downloadUrl ||
+        (doi ? `https://doi.org/${doi}` : "") ||
+        (r.id ? `https://core.ac.uk/works/${r.id}` : "");
+      const authors = (r.authors || [])
+        .slice(0, 3)
+        .map((a: any) => a.name)
+        .filter(Boolean);
+      const journal = r.journals?.[0]?.title || "";
+      const abstract = (r.abstract || "").slice(0, 500);
+
+      const content = [
+        abstract,
+        authors.length
+          ? `Authors: ${authors.join(", ")}${
+              (r.authors || []).length > 3 ? " et al." : ""
+            }`
+          : "",
+        r.yearPublished ? `Year: ${r.yearPublished}` : "",
+        journal ? `Journal: ${journal}` : "",
+        doi ? `DOI: ${doi}` : "",
+      ]
+        .filter(Boolean)
+        .join("\n");
+
+      return { url, title, content, engine: "core" };
+    })
+    .filter((r: any) => r.url && r.title);
+};

package/src/sources/academic/crossref.ts

@@ -0,0 +1,75 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const crossref: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    query: query,
+    offset: String(20 * ((page || 1) - 1)),
+  });
+
+  const response = await fetch(
+    `https://api.crossref.org/works?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+      },
+    }
+  );
+
+  if (!response.ok) return [];
+
+  const data = await response.json();
+
+  if (!data || !data.message || !data.message.items) return [];
+
+  return data.message.items
+    .filter((record: any) => record.type !== "component")
+    .map((record: any) => {
+      let title = "";
+      let journal = "";
+
+      if (record.type === "book-chapter") {
+        title = record["container-title"]?.[0] || "";
+        if (
+          record.title?.[0] &&
+          record.title[0].toLowerCase().trim() !==
+            title.toLowerCase().trim()
+        ) {
+          title += ` (${record.title[0]})`;
+        }
+      } else {
+        title =
+          record.title?.[0] || record["container-title"]?.[0] || "";
+        journal =
+          record["container-title"]?.[0] && record.title?.[0]
+            ? record["container-title"][0]
+            : "";
+      }
+
+      const authors = (record.author || [])
+        .map((a: any) => `${a.given || ""} ${a.family || ""}`.trim())
+        .filter((a: string) => a)
+        .join(", ");
+
+      const content = [
+        record.abstract || "",
+        journal ? `Journal: ${journal}` : "",
+        authors ? `Authors: ${authors}` : "",
+        record.publisher ? `Publisher: ${record.publisher}` : "",
+        record.DOI ? `DOI: ${record.DOI}` : "",
+      ]
+        .filter(Boolean)
+        .join("\n");
+
+      return {
+        url: record.URL || `https://doi.org/${record.DOI}` || "",
+        title,
+        content,
+        engine: "crossref",
+      };
+    })
+    .filter((r: any) => r.url && r.title);
+};

package/src/sources/academic/doaj.ts

@@ -0,0 +1,63 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const doaj: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    pageSize: "10",
+    page: String(page || 1),
+    sort: "created_date:desc",
+  });
+
+  const response = await fetch(
+    `https://doaj.org/api/v2/search/articles/${encodeURIComponent(query)}?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+        Accept: "application/json",
+      },
+    }
+  );
+
+  if (!response.ok) return [];
+
+  const data = await response.json();
+
+  if (!data || !Array.isArray(data.results)) return [];
+
+  return data.results
+    .map((r: any) => {
+      const bib = r.bibjson || {};
+      const title = bib.title || "";
+      const doi =
+        bib.identifier?.find((i: any) => i.type === "doi")?.id || "";
+      const link =
+        bib.link?.find((l: any) => l.type === "fulltext")?.url || "";
+      const url = link || (doi ? `https://doi.org/${doi}` : "");
+      const authors = (bib.author || [])
+        .slice(0, 3)
+        .map((a: any) => a.name)
+        .filter(Boolean);
+      const journal = bib.journal?.title || "";
+      const abstract = (bib.abstract || "").slice(0, 500);
+
+      const content = [
+        abstract,
+        authors.length
+          ? `Authors: ${authors.join(", ")}${
+              (bib.author || []).length > 3 ? " et al." : ""
+            }`
+          : "",
+        bib.year ? `Year: ${bib.year}` : "",
+        journal ? `Journal: ${journal}` : "",
+        doi ? `DOI: ${doi}` : "",
+      ]
+        .filter(Boolean)
+        .join("\n");
+
+      return { url, title, content, engine: "doaj" };
+    })
+    .filter((r: any) => r.url && r.title);
+};

package/src/sources/academic/google_scholar.ts

@@ -0,0 +1,53 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const google_scholar: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    start: String(((page || 1) - 1) * 10),
+    hl: "en",
+  });
+
+  const response = await fetch(
+    `https://scholar.google.com/scholar?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+        Accept:
+          "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+        Cookie: "CONSENT=YES+",
+      },
+    }
+  );
+
+  if (!response.ok) return [];
+
+  const text = await response.text();
+  const { document } = parseHTML(text);
+
+  return Array.from(document.querySelectorAll(".gs_r.gs_or.gs_scl"))
+    .map((element) => {
+      const titleLink = element.querySelector(".gs_rt a");
+      const url = titleLink?.getAttribute("href") || "";
+      const title = titleLink?.textContent?.trim() || "";
+      const content =
+        element.querySelector(".gs_rs")?.textContent?.trim() || "";
+      const publicationInfo =
+        element.querySelector(".gs_a")?.textContent?.trim() || "";
+
+      return {
+        url,
+        title,
+        content: publicationInfo
+          ? `${publicationInfo} - ${content}`
+          : content,
+        engine: "google_scholar",
+      };
+    })
+    .filter((r) => r.url && r.title);
+};

package/src/sources/academic/openalex.ts

@@ -0,0 +1,75 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+function reconstructAbstract(
+  invertedIndex: Record<string, number[]>
+): string {
+  const words: string[] = [];
+  for (const [word, positions] of Object.entries(invertedIndex)) {
+    for (const pos of positions) words[pos] = word;
+  }
+  return words.filter(Boolean).join(" ");
+}
+
+export const openalex: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    search: query,
+    "per-page": "10",
+    page: String(page || 1),
+    sort: "cited_by_count:desc",
+    mailto: process.env.OPENALEX_MAILTO || "research@qwksearch.com",
+  });
+
+  const response = await fetch(
+    `https://api.openalex.org/works?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+        Accept: "application/json",
+      },
+    }
+  );
+
+  if (!response.ok) return [];
+
+  const data = await response.json();
+
+  if (!data || !Array.isArray(data.results)) return [];
+
+  return data.results
+    .map((w: any) => {
+      const title = w.title || "";
+      const doi = w.doi || "";
+      const url =
+        w.primary_location?.landing_page_url || doi || w.id || "";
+      const authors = (w.authorships || [])
+        .slice(0, 3)
+        .map((a: any) => a.author?.display_name)
+        .filter(Boolean);
+      const abstract = w.abstract_inverted_index
+        ? reconstructAbstract(w.abstract_inverted_index).slice(0, 500)
+        : "";
+      const journal = w.host_venue?.display_name || "";
+
+      const content = [
+        abstract,
+        authors.length
+          ? `Authors: ${authors.join(", ")}${
+              (w.authorships || []).length > 3 ? " et al." : ""
+            }`
+          : "",
+        w.publication_year ? `Year: ${w.publication_year}` : "",
+        journal ? `Journal: ${journal}` : "",
+        w.cited_by_count ? `Citations: ${w.cited_by_count}` : "",
+        doi ? `DOI: ${doi}` : "",
+      ]
+        .filter(Boolean)
+        .join("\n");
+
+      return { url, title, content, engine: "openalex" };
+    })
+    .filter((r: any) => r.url && r.title);
+};

package/src/sources/academic/pubmed.ts

@@ -0,0 +1,96 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const pubmed: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const pageno = page || 1;
+  const number_of_results = 10;
+  const retstart = (pageno - 1) * number_of_results;
+
+  const esearchParams = new URLSearchParams({
+    db: "pubmed",
+    term: query,
+    retstart: String(retstart),
+    retmax: String(number_of_results),
+  });
+
+  const esearchResponse = await fetch(
+    `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?${esearchParams}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+      },
+    }
+  );
+
+  if (!esearchResponse.ok) return [];
+
+  const esearchText = await esearchResponse.text();
+  const { document: esearchDoc } = parseHTML(esearchText);
+  const pmids: string[] = [];
+  esearchDoc.querySelectorAll("Id").forEach((elem) => {
+    const pmid = elem.textContent;
+    if (pmid) pmids.push(pmid);
+  });
+
+  if (pmids.length === 0) return [];
+
+  const efetchResponse = await fetch(
+    `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&retmode=xml&id=${pmids.join(",")}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+      },
+    }
+  );
+
+  if (!efetchResponse.ok) return [];
+
+  const efetchText = await efetchResponse.text();
+  const { document } = parseHTML(efetchText);
+
+  return Array.from(document.querySelectorAll("PubmedArticle"))
+    .map((article) => {
+      const title =
+        article.querySelector("ArticleTitle")?.textContent || "";
+      const pmid = article.querySelector("PMID")?.textContent || "";
+      const url = `https://www.ncbi.nlm.nih.gov/pubmed/${pmid}`;
+
+      const abstractElements = article.querySelectorAll("AbstractText");
+      const abstract = Array.from(abstractElements)
+        .map((el) => el.textContent)
+        .join(" ");
+
+      const journal =
+        article.querySelector("Journal Title")?.textContent || "";
+      const doi =
+        article.querySelector('ELocationID[EIdType="doi"]')
+          ?.textContent || "";
+
+      const authors: string[] = [];
+      article.querySelectorAll("AuthorList Author").forEach((author) => {
+        const firstName =
+          author.querySelector("ForeName")?.textContent || "";
+        const lastName =
+          author.querySelector("LastName")?.textContent || "";
+        const authorName = `${firstName} ${lastName}`.trim();
+        if (authorName) authors.push(authorName);
+      });
+
+      const content = [
+        abstract,
+        journal ? `Journal: ${journal}` : "",
+        authors.length ? `Authors: ${authors.join(", ")}` : "",
+        doi ? `DOI: ${doi}` : "",
+      ]
+        .filter(Boolean)
+        .join("\n");
+
+      return { url, title, content, engine: "pubmed" };
+    })
+    .filter((r) => r.url && r.title);
+};

package/src/sources/academic/semantic_scholar.ts

@@ -0,0 +1,77 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const semantic_scholar: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const response = await fetch(
+    "https://www.semanticscholar.org/api/1/search",
+    {
+      method: "POST",
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        queryString: query,
+        page: page || 1,
+        pageSize: 10,
+        sort: "relevance",
+        getQuerySuggestions: false,
+        authors: [],
+        coAuthors: [],
+        venues: [],
+        performTitleMatch: true,
+      }),
+    }
+  );
+
+  if (!response.ok) return [];
+
+  const data = await response.json();
+
+  if (!data.results) return [];
+
+  return data.results.map((result: any) => {
+    let url = result.primaryPaperLink?.url;
+    if (!url && result.links && result.links.length > 0) {
+      url = result.links[0];
+    }
+    if (
+      !url &&
+      result.alternatePaperLinks &&
+      result.alternatePaperLinks.length > 0
+    ) {
+      url = result.alternatePaperLinks[0].url;
+    }
+    if (!url) {
+      url = `https://www.semanticscholar.org/paper/${result.id}`;
+    }
+
+    const title = result.title?.text || "";
+    const abstract = result.abstract?.text || "";
+
+    const metadata: string[] = [];
+    if (result.authors && result.authors.length > 0) {
+      const authorNames = result.authors
+        .map((a: any) => a.name)
+        .slice(0, 3);
+      metadata.push(
+        `Authors: ${authorNames.join(", ")}${
+          result.authors.length > 3 ? " et al." : ""
+        }`
+      );
+    }
+    if (result.year) metadata.push(`Year: ${result.year}`);
+    if (result.venue) metadata.push(`Venue: ${result.venue}`);
+    if (result.citationCount) metadata.push(`Citations: ${result.citationCount}`);
+
+    const content =
+      metadata.length > 0
+        ? `${metadata.join(" | ")}\n\n${abstract}`
+        : abstract;
+
+    return { url, title, content, engine: "semantic_scholar" };
+  });
+};

package/src/sources/academic/wikidata.ts

@@ -0,0 +1,44 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const wikidata: EngineFunction = async (
+  query: string,
+  _page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    action: "wbsearchentities",
+    search: query,
+    language: "en",
+    format: "json",
+    limit: "20",
+  });
+
+  const response = await fetch(
+    `https://www.wikidata.org/w/api.php?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+      },
+    }
+  );
+
+  if (!response.ok) return [];
+
+  const data = await response.json();
+
+  if (!data || !data.search) return [];
+
+  return data.search.map((item: any) => {
+    const id = item.id;
+    const title = item.label || id;
+    const description = item.description || "No description available";
+    const url = item.url || `https://www.wikidata.org/wiki/${id}`;
+
+    return {
+      url,
+      title,
+      content: description,
+      engine: "wikidata",
+    };
+  });
+};

package/src/sources/general/baidu.ts

@@ -0,0 +1,59 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const baidu: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    wd: query,
+    rn: "10",
+    pn: String(((page || 1) - 1) * 10),
+  });
+  const response = await fetch(`https://www.baidu.com/s?${params}`, {
+    headers: {
+      "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+      Accept:
+        "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+      "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+      Cookie: "BAIDUID=1234567890:FG=1",
+    },
+  });
+  if (!response.ok) return [];
+  const htmlString = await response.text();
+  const { document } = parseHTML(htmlString);
+
+  return Array.from(
+    document.querySelectorAll(
+      "#content_left > div.result, div.c-container"
+    )
+  )
+    .map((element) => {
+      const titleLink = element.querySelector("h3 a, h3.c-title a");
+      const url = titleLink?.getAttribute("href") || "";
+      const title = titleLink?.textContent?.trim() || "";
+
+      if (!url || !title) {
+        return null;
+      }
+
+      const content =
+        element
+          .querySelector(
+            "div.c-abstract, div.c-span9, span.content-right_2s-H4"
+          )
+          ?.textContent?.trim() || "";
+      const timestamp =
+        element
+          .querySelector("span.c-color-gray2, span.newTimeFactor_2s-H4")
+          ?.textContent?.trim() || "";
+
+      return {
+        url,
+        title,
+        content: timestamp ? `${timestamp}\n${content}` : content,
+        engine: "baidu",
+      };
+    })
+    .filter((r) => r !== null && r.url && r.title);
+};

package/src/sources/general/bing.ts

@@ -0,0 +1,30 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const bing: EngineFunction = async (q: string, page?: number) => {
+  const params = new URLSearchParams({
+    q,
+    first: String(((page || 1) - 1) * 10 + 1),
+  });
+  const response = await fetch(`https://www.bing.com/search?${params}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+      "Accept-Language": "en-US,en;q=0.9",
+      Cookie: "CONSENT=YES+",
+    },
+  });
+  if (!response.ok) return [];
+  const text = await response.text();
+  const { document } = parseHTML(text);
+  return Array.from(document.querySelectorAll("li.b_algo"))
+    .map((el) => {
+      return {
+        url: el.querySelector("h2 a")?.getAttribute("href") || "",
+        title: el.querySelector("h2 a")?.textContent?.trim() || "",
+        content: el.querySelector(".b_caption p")?.textContent?.trim() || "",
+        engine: "bing",
+      };
+    })
+    .filter((r) => r.url && r.title);
+};

package/src/sources/general/brave.ts

@@ -0,0 +1,54 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const brave: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    p: String(page || 1),
+  });
+  const response = await fetch(`https://search.brave.com/search?${params}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+      Accept:
+        "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+      "Accept-Language": "en-US,en;q=0.9",
+    },
+  });
+  if (!response.ok) return [];
+  const htmlString = await response.text();
+  const { document } = parseHTML(htmlString);
+
+  const snippets = document.querySelectorAll(".snippet");
+  console.log(`Brave: Found ${snippets.length} snippets`);
+
+  const data = Array.from(snippets)
+    .map((element) => {
+      const link = element.querySelector("a");
+      const url = link?.getAttribute("href") || "";
+      const title =
+        element.querySelector(".search-snippet-title")?.textContent?.trim() || "";
+      const content =
+        element
+          .querySelector(".content")
+          ?.textContent?.trim() || "";
+
+      return {
+        url,
+        title,
+        content,
+        engine: "brave",
+      };
+    })
+    .filter((r) => r.url && r.title);
+
+  console.log(`Brave: Extracted ${data.length} valid results`);
+  if (data.length === 0 && htmlString.length > 0) {
+    console.log("Brave: HTML sample:", htmlString.substring(0, 1000));
+  }
+
+  return data;
+};