search-web-api 1.0.13

package/src/sources/social/mastodon.ts

@@ -0,0 +1,46 @@
+import { EngineFunction, EngineResult } from "../../types/search-engine-interface.js";
+
+export const mastodon: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    resolve: "false",
+    type: "accounts",
+    limit: "40",
+  });
+  const response = await fetch(
+    `https://mastodon.social/api/v2/search?${params}`
+  );
+  if (!response.ok) return [];
+  const data = await response.json();
+  const results: EngineResult[] = [];
+  const accounts = data.accounts || [];
+
+  for (const account of accounts) {
+    const url = account.uri || account.url;
+    const username = account.username || "";
+    const displayName = account.display_name || username;
+    const followersCount = account.followers_count || 0;
+    const note = account.note || "";
+
+    // Strip HTML tags from note
+    const cleanNote = note.replace(/<[^>]*>/g, "").trim();
+
+    const title = `${displayName} (@${username})`;
+    const content = `Followers: ${followersCount}\n${cleanNote}`;
+
+    const thumbnail = account.avatar || account.avatar_static;
+
+    results.push({
+      url,
+      title,
+      content,
+      thumbnail,
+      engine: "mastodon",
+    });
+  }
+
+  return results;
+};

package/src/sources/social/medium.ts

@@ -0,0 +1,52 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction, EngineResult } from "../../types/search-engine-interface.js";
+
+export const medium: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({ q: query });
+  const response = await fetch(`https://medium.com/search?${params}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    },
+  });
+  if (!response.ok) return [];
+  const html = await response.text();
+  const { document } = parseHTML(html);
+  const results: EngineResult[] = [];
+
+  // Medium uses dynamic content, so we'll parse what we can from the initial HTML
+  document.querySelectorAll("article").forEach((el) => {
+    const element = el;
+    const titleLink = element.querySelector("h2 a, h3 a");
+    const title = titleLink?.textContent?.trim() || "";
+    const href = titleLink?.getAttribute("href");
+    const url = href
+      ? href.startsWith("http")
+        ? href
+        : `https://medium.com${href}`
+      : "";
+    const content = element.querySelector("p")?.textContent?.trim() || "";
+    const author =
+      element
+        .querySelector('a[rel="author"]')
+        ?.textContent?.trim() || "";
+    const readTime =
+      element
+        .querySelector('[aria-label*="read"]')
+        ?.textContent?.trim() || "";
+
+    if (url && title) {
+      results.push({
+        url,
+        title,
+        content: `${content} | By ${author || "Unknown"} | ${readTime || ""}`,
+        engine: "medium",
+      });
+    }
+  });
+
+  return results;
+};

package/src/sources/social/reddit.ts

@@ -0,0 +1,48 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction, EngineResult } from "../../types/search-engine-interface.js";
+
+export const reddit: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    sort: "relevance",
+    t: "all",
+  });
+  const response = await fetch(
+    `https://old.reddit.com/search?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const html = await response.text();
+  const { document } = parseHTML(html);
+  const results: EngineResult[] = [];
+
+  document.querySelectorAll(".search-result").forEach((el) => {
+    const titleLink = el.querySelector("a.search-title");
+    const title = titleLink?.textContent?.trim() || "";
+    const link = titleLink?.getAttribute("href");
+    const content =
+      el.querySelector(".search-result-body")?.textContent?.trim() ||
+      "";
+
+    if (title && link) {
+      results.push({
+        title,
+        url: link.startsWith("http")
+          ? link
+          : `https://old.reddit.com${link}`,
+        content: content || "",
+        engine: "reddit",
+      });
+    }
+  });
+
+  return results;
+};

package/src/sources/social/soundcloud.ts

@@ -0,0 +1,64 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction, EngineResult } from "../../types/search-engine-interface.js";
+
+export const soundcloud: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({ q: query });
+  const response = await fetch(`https://soundcloud.com/search?${params}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    },
+  });
+  if (!response.ok) return [];
+  const html = await response.text();
+  const { document } = parseHTML(html);
+  const results: EngineResult[] = [];
+
+  // SoundCloud uses dynamic content, so we'll try to extract from initial HTML
+  document
+    .querySelectorAll("article, .searchList__item")
+    .forEach((el) => {
+      const element = el;
+      const titleLink = element.querySelector(
+        'a[itemprop="url"], h2 a'
+      );
+      const title =
+        element
+          .querySelector('[itemprop="name"], h2')
+          ?.textContent?.trim() || "";
+      const artist =
+        element
+          .querySelector('[itemprop="byArtist"], .soundTitle__username')
+          ?.textContent?.trim() || "";
+      const href = titleLink?.getAttribute("href");
+      const url = href
+        ? href.startsWith("http")
+          ? href
+          : `https://soundcloud.com${href}`
+        : "";
+      const plays =
+        element
+          .querySelector(".sc-ministats-plays, .soundStats__plays")
+          ?.textContent?.trim() || "";
+      const duration =
+        element
+          .querySelector(
+            '.soundTitle__tagContent time, [itemprop="duration"]'
+          )
+          ?.textContent?.trim() || "";
+
+      if (url && title) {
+        results.push({
+          url,
+          title: `${title}${artist ? " - " + artist : ""}`,
+          content: `${plays ? "Plays: " + plays : ""} ${duration ? "| Duration: " + duration : ""}`,
+          engine: "soundcloud",
+        });
+      }
+    });
+
+  return results;
+};

package/src/sources/social/twitter.ts

@@ -0,0 +1,56 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction, EngineResult } from "../../types/search-engine-interface.js";
+
+export const twitter: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    f: "tweets",
+    q: query,
+  });
+  const response = await fetch(`https://nitter.net/search?${params}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    },
+  });
+  if (!response.ok) return [];
+  const html = await response.text();
+  const { document } = parseHTML(html);
+  const results: EngineResult[] = [];
+
+  document.querySelectorAll(".timeline-item").forEach((el) => {
+    const element = el;
+    const tweetLink = element.querySelector(".tweet-link");
+    const username =
+      element.querySelector(".username")?.textContent?.trim() || "";
+    const fullname =
+      element.querySelector(".fullname")?.textContent?.trim() || "";
+    const content =
+      element.querySelector(".tweet-content")?.textContent?.trim() ||
+      "";
+    const timestamp =
+      element
+        .querySelector(".tweet-date a")
+        ?.textContent?.trim() || "";
+    const stats =
+      element.querySelector(".tweet-stats")?.textContent?.trim() || "";
+
+    const href = tweetLink?.getAttribute("href");
+    const url = href
+      ? `https://twitter.com${href.replace("/i/web", "")}`
+      : "";
+
+    if (url && content) {
+      results.push({
+        url,
+        title: `${fullname} (@${username})`,
+        content: `${content} | ${timestamp} | ${stats}`,
+        engine: "twitter",
+      });
+    }
+  });
+
+  return results;
+};

package/src/sources/specialized/annas_archive.ts

@@ -0,0 +1,97 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction, EngineResult } from "../../types/search-engine-interface.js";
+
+const baseDomains = [
+  "annas-archive.gl",
+  "annas-archive.gd",
+  "annas-archive.pk",
+];
+
+export const annas_archive: EngineFunction = async (
+  query: string,
+  page: number | undefined,
+  baseDomain: number = 0,
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    page: String(page || 1),
+  });
+
+  let response: Response;
+  try {
+    response = await fetch(
+      `https://${baseDomains[baseDomain]}/search?${params}`,
+      {
+        headers: {
+          "User-Agent":
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+        },
+        signal: AbortSignal.timeout(10000),
+      }
+    );
+  } catch (error) {
+    if (baseDomain + 1 < baseDomains.length) {
+      return await annas_archive(query, page, baseDomain + 1);
+    }
+    throw error;
+  }
+
+  if (!response.ok) {
+    if (baseDomain + 1 < baseDomains.length) {
+      return await annas_archive(query, page, baseDomain + 1);
+    }
+    return [];
+  }
+
+  const html = await response.text();
+  const results: EngineResult[] = [];
+
+  if (!html || typeof html !== "string") {
+    return results;
+  }
+
+  const { document } = parseHTML(html);
+
+  document
+    .querySelectorAll("main div.js-aarecord-list-outer > div")
+    .forEach((element) => {
+      const elElem = element;
+
+      const href = elElem.querySelector("a")?.getAttribute("href");
+      if (!href) return;
+
+      const url = "https://" + baseDomains[baseDomain] + href;
+      const title =
+        elElem.querySelector('a[href^="/md5"]')?.textContent?.trim() ||
+        "";
+      const author =
+        elElem.querySelector('a[href^="/search"]')?.textContent?.trim() ||
+        "";
+      const publisher =
+        elElem
+          .querySelectorAll('a[href^="/search"]')[1]
+          ?.textContent?.trim() || "";
+      const description =
+        elElem.querySelector("div.relative")?.textContent?.trim() || "";
+      const thumbnail =
+        elElem.querySelector("img")?.getAttribute("src") || undefined;
+
+      const content = [
+        description,
+        author ? `Author: ${author}` : "",
+        publisher ? `Publisher: ${publisher}` : "",
+      ]
+        .filter(Boolean)
+        .join("\n");
+
+      results.push({
+        url,
+        title,
+        content,
+        engine: "annas_archive",
+        thumbnail,
+      });
+    });
+
+  return results;
+};

package/src/sources/specialized/archive.ts

@@ -0,0 +1,48 @@
+import { EngineFunction, EngineResult } from "../../types/search-engine-interface.js";
+
+export const archive: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    "fl[]": "identifier,title,description,mediatype,downloads",
+    "sort[]": "",
+    rows: "50",
+    page: String(page || 1),
+    output: "json",
+  });
+  const response = await fetch(
+    `https://archive.org/advancedsearch.php?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const data = await response.json();
+  const results: EngineResult[] = [];
+
+  if (data && data.response && data.response.docs) {
+    data.response.docs.forEach((doc: any) => {
+      const identifier = doc.identifier;
+      const title = doc.title || identifier;
+      const description = doc.description || "No description available";
+      const mediatype = doc.mediatype || "unknown";
+      const downloads = doc.downloads || 0;
+      const url = `https://archive.org/details/${identifier}`;
+
+      results.push({
+        url,
+        title,
+        content: `${description} | Type: ${mediatype} | Downloads: ${downloads}`,
+        thumbnail: `https://archive.org/services/img/${identifier}`,
+        engine: "archive",
+      });
+    });
+  }
+
+  return results;
+};

package/src/sources/specialized/genius.ts

@@ -0,0 +1,43 @@
+import { EngineFunction, EngineResult } from "../../types/search-engine-interface.js";
+
+export const genius: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    per_page: "5",
+    q: query,
+  });
+  const response = await fetch(
+    `https://genius.com/api/search/multi?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const data = await response.json();
+  const results: EngineResult[] = [];
+
+  if (data && data.response && data.response.sections) {
+    data.response.sections.forEach((section: any) => {
+      if (section.type === "song" || section.type === "lyric") {
+        section.hits.forEach((hit: any) => {
+          if (hit.result) {
+            results.push({
+              url: hit.result.url,
+              title: hit.result.full_title,
+              content: `Artist: ${hit.result.artist_names}`,
+              thumbnail: hit.result.song_art_image_thumbnail_url,
+              engine: "genius",
+            });
+          }
+        });
+      }
+    });
+  }
+
+  return results;
+};

package/src/sources/specialized/goodreads.ts

@@ -0,0 +1,62 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction, EngineResult } from "../../types/search-engine-interface.js";
+
+export const goodreads: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    page: String(page || 1),
+  });
+  const response = await fetch(
+    `https://www.goodreads.com/search?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const html = await response.text();
+  const results: EngineResult[] = [];
+
+  if (!html || typeof html !== "string") {
+    return results;
+  }
+
+  const { document } = parseHTML(html);
+
+  document.querySelectorAll("table tr").forEach((element) => {
+    const rowElem = element;
+
+    const $link = rowElem.querySelector("a.bookTitle");
+    const href = $link?.getAttribute("href");
+    const title = $link?.textContent?.trim() || "";
+
+    if (!href || !title) return;
+
+    const thumbnail =
+      rowElem.querySelector("img.bookCover")?.getAttribute("src") ||
+      undefined;
+    const author =
+      rowElem.querySelector("a.authorName")?.textContent?.trim() || "";
+    const info =
+      rowElem.querySelector("span.uitext")?.textContent?.trim() || "";
+
+    const content = [info, author ? `Author: ${author}` : ""]
+      .filter(Boolean)
+      .join(" | ");
+
+    results.push({
+      url: `https://www.goodreads.com${href}`,
+      title,
+      content,
+      engine: "goodreads",
+      thumbnail,
+    });
+  });
+
+  return results;
+};

package/src/sources/specialized/imdb.ts

@@ -0,0 +1,55 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction, EngineResult } from "../../types/search-engine-interface.js";
+
+export const imdb: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    s: "all",
+  });
+  const response = await fetch(
+    `https://www.imdb.com/find/?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+        "Accept-Language": "en-US,en;q=0.9",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const html = await response.text();
+  const { document } = parseHTML(html);
+  const results: EngineResult[] = [];
+
+  document
+    .querySelectorAll(".ipc-metadata-list-summary-item")
+    .forEach((el) => {
+      const element = el;
+      const link = element.querySelector(
+        "a.ipc-metadata-list-summary-item__t"
+      );
+      const url = `https://www.imdb.com${link?.getAttribute("href")}`;
+      const title = link?.textContent?.trim() || "";
+      const content =
+        element
+          .querySelector(".ipc-metadata-list-summary-item__li")
+          ?.textContent?.trim() || "";
+      const thumbnail =
+        element.querySelector("img")?.getAttribute("src") || "";
+
+      if (url && title) {
+        results.push({
+          url,
+          title,
+          content,
+          thumbnail,
+          engine: "imdb",
+        });
+      }
+    });
+
+  return results;
+};

package/src/sources/specialized/openlibrary.ts

@@ -0,0 +1,59 @@
+import { EngineFunction, EngineResult } from "../../types/search-engine-interface.js";
+
+export const openlibrary: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    page: String(page || 1),
+    limit: "10",
+    fields: "*",
+  });
+  const response = await fetch(
+    `https://openlibrary.org/search.json?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+      },
+      signal: AbortSignal.timeout(10000),
+    }
+  );
+  if (!response.ok) return [];
+  const data = await response.json();
+  const results: EngineResult[] = [];
+
+  if (!data || !data.docs) {
+    return results;
+  }
+
+  for (const item of data.docs) {
+    const thumbnail = item.lending_identifier_s
+      ? `https://archive.org/services/img/${item.lending_identifier_s}`
+      : "";
+
+    const authors = item.author_name ? item.author_name.join(", ") : "";
+    const publishYear = item.first_publish_year || "";
+    const isbn = item.isbn ? item.isbn.slice(0, 3).join(", ") : "";
+
+    const content = [
+      item.first_sentence ? item.first_sentence.join(" / ") : "",
+      authors ? `Authors: ${authors}` : "",
+      publishYear ? `First published: ${publishYear}` : "",
+      isbn ? `ISBN: ${isbn}` : "",
+    ]
+      .filter(Boolean)
+      .join("\n");
+
+    results.push({
+      url: `https://openlibrary.org${item.key}`,
+      title: item.title,
+      content,
+      engine: "openlibrary",
+      thumbnail,
+    });
+  }
+
+  return results;
+};

package/src/sources/specialized/wikipedia.ts

@@ -0,0 +1,37 @@
+import { EngineFunction, EngineResult } from "../../types/search-engine-interface.js";
+
+export const wikipedia: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    action: "query",
+    list: "search",
+    srsearch: query,
+    format: "json",
+    sroffset: String(((page || 1) - 1) * 10),
+  });
+  const response = await fetch(
+    `https://en.wikipedia.org/w/api.php?${params}`,
+    {
+      headers: {
+        "User-Agent": "HonoxSearX/1.0 (mailto:admin@example.com)",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const data = await response.json();
+
+  const results: EngineResult[] = (
+    data.query?.search || []
+  ).map((item: any) => ({
+    url: `https://en.wikipedia.org/wiki/${encodeURIComponent(
+      item.title.replace(/ /g, "_")
+    )}`,
+    title: item.title,
+    content: item.snippet.replace(/<[^>]+>/g, ""),
+    engine: "wikipedia",
+  }));
+
+  return results;
+};