search-web-api 1.0.13

package/src/sources/it/stackoverflow.ts

@@ -0,0 +1,39 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const stackoverflow: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    order: "desc",
+    sort: "relevance",
+    q: query,
+    site: "stackoverflow",
+    page: String(page || 1),
+    pagesize: "10",
+  });
+  const response = await fetch(
+    `https://api.stackexchange.com/2.3/search?${params}`,
+    {
+      headers: {
+        "User-Agent": "HonoxSearX/1.0",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const data = await response.json();
+  const results: any[] = [];
+
+  if (data && data.items) {
+    data.items.forEach((item: any) => {
+      results.push({
+        url: item.link,
+        title: item.title,
+        content: item.tags ? `Tags: ${item.tags.join(", ")}` : "",
+        engine: "stackoverflow",
+      });
+    });
+  }
+
+  return results;
+};

package/src/sources/maps/apple_maps.ts

@@ -0,0 +1,105 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+// Token management for Apple Maps API
+let tokenCache = {
+  value: "",
+  lastUpdated: 0,
+};
+
+async function obtainToken(): Promise<string> {
+  const now = Date.now();
+
+  // Token expires every 30 minutes (1800 seconds)
+  if (tokenCache.value && now - tokenCache.lastUpdated < 1800000) {
+    return tokenCache.value;
+  }
+
+  try {
+    // Get DuckDuckGo's MapKit token
+    const tokenResponse = await fetch(
+      "https://duckduckgo.com/local.js?get_mk_token=1"
+    );
+    const tokenText = await tokenResponse.text();
+
+    // Exchange for actual Apple Maps token
+    const actualTokenResponse = await fetch(
+      "https://cdn.apple-mapkit.com/ma/bootstrap?apiVersion=2&mkjsVersion=5.72.53&poi=1",
+      {
+        headers: {
+          Authorization: `Bearer ${tokenText}`,
+        },
+      }
+    );
+    const actualTokenData = await actualTokenResponse.json();
+
+    tokenCache.value = actualTokenData.authInfo.access_token;
+    tokenCache.lastUpdated = now;
+    return tokenCache.value;
+  } catch (err) {
+    return "";
+  }
+}
+
+export const apple_maps: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const token = await obtainToken();
+
+  if (!token) {
+    throw new Error("Failed to obtain Apple Maps API token");
+  }
+
+  const params = new URLSearchParams({
+    q: query,
+    lang: "en",
+    mkjsVersion: "5.72.53",
+  });
+  const response = await fetch(
+    `https://api.apple-mapkit.com/v1/search?${params}`,
+    {
+      headers: {
+        Authorization: `Bearer ${token}`,
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const json = await response.json();
+
+  if (!json || !json.results) {
+    return [];
+  }
+
+  return json.results.map((result: any) => {
+    const lat = result.center?.lat;
+    const lng = result.center?.lng;
+
+    const addressParts = [
+      result.subThoroughfare,
+      result.thoroughfare,
+      result.locality,
+      result.postCode,
+      result.country,
+    ].filter(Boolean);
+
+    const content = [
+      addressParts.join(", "),
+      result.poiCategory ? `Type: ${result.poiCategory}` : "",
+      lat && lng ? `Coordinates: ${lat}, ${lng}` : "",
+      result.telephone ? `Phone: ${result.telephone}` : "",
+    ]
+      .filter(Boolean)
+      .join("\n");
+
+    return {
+      url: result.placecardUrl || result.urls?.[0] || "",
+      title: result.name,
+      content,
+      engine: "apple_maps",
+      latitude: lat,
+      longitude: lng,
+    };
+  });
+};

package/src/sources/maps/openstreetmap.ts

@@ -0,0 +1,34 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const openstreetmap: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    format: "json",
+    addressdetails: "1",
+    limit: "10",
+  });
+  const response = await fetch(
+    `https://nominatim.openstreetmap.org/search?${params}`,
+    {
+      headers: {
+        "User-Agent": "HonoxSearX/1.0",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const json = await response.json();
+
+  if (!Array.isArray(json)) {
+    return [];
+  }
+
+  return json.map((item: any) => ({
+    url: `https://www.openstreetmap.org/${item.osm_type}/${item.osm_id}`,
+    title: item.display_name,
+    content: `Type: ${item.type}, Class: ${item.class}`,
+    engine: "openstreetmap",
+  }));
+};

package/src/sources/maps/photon.ts

@@ -0,0 +1,77 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const photon: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    limit: "10",
+    lang: "en",
+  });
+  const response = await fetch(
+    `https://photon.komoot.io/api/?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (X1; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const json = await response.json();
+  const features = json?.features || [];
+
+  return features
+    .map((feature: any) => {
+      const properties = feature.properties;
+
+      if (!properties) {
+        return null;
+      }
+
+      const name = properties.name || "";
+      const osmType = properties.osm_type;
+
+      // Map OSM type to string
+      let osmTypeStr = "";
+      if (osmType === "N") osmTypeStr = "node";
+      else if (osmType === "W") osmTypeStr = "way";
+      else if (osmType === "R") osmTypeStr = "relation";
+      else return null;
+
+      const osmId = properties.osm_id;
+      const url = `https://openstreetmap.org/${osmTypeStr}/${osmId}`;
+
+      // Build address
+      const addressParts: string[] = [];
+      if (properties.street) addressParts.push(properties.street);
+      if (properties.housenumber)
+        addressParts.push(properties.housenumber);
+      if (properties.city) addressParts.push(properties.city);
+      if (properties.postcode) addressParts.push(properties.postcode);
+      if (properties.country) addressParts.push(properties.country);
+
+      // Get coordinates
+      const geometry = feature.geometry;
+      const coordinates = geometry?.coordinates || [];
+      const lon = coordinates[0];
+      const lat = coordinates[1];
+
+      const content = [
+        addressParts.join(", "),
+        properties.type ? `Type: ${properties.type}` : "",
+        coordinates.length === 2 ? `Coordinates: ${lat}, ${lon}` : "",
+      ]
+        .filter(Boolean)
+        .join("\n");
+
+      return {
+        url,
+        title: name,
+        content,
+        engine: "photon",
+      };
+    })
+    .filter(Boolean);
+};

package/src/sources/news/bing_news.ts

@@ -0,0 +1,95 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const bing_news: EngineFunction = async (
+  q: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q,
+    InfiniteScroll: "1",
+    first: String(((page || 1) - 1) * 10 + 1),
+    SFX: String((page || 1) - 1),
+    form: "PTFTNR",
+    setlang: "en",
+    cc: "US",
+  });
+  const response = await fetch(
+    `https://www.bing.com/news/infinitescrollajax?${params}`,
+    {
+      headers: {
+        Accept:
+          "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const html = await response.text();
+  const { document } = parseHTML(html);
+  const results: any[] = [];
+
+  // Parse news items from Bing News
+  document
+    .querySelectorAll('div.newsitem, div[class*="newsitem"]')
+    .forEach((el) => {
+      const element = el;
+
+      const link = element.querySelector('a.title, a[class*="title"]');
+      if (!link) return;
+
+      const url = link.getAttribute("href");
+      const title = link.textContent?.trim() || "";
+      const content =
+        element
+          .querySelector('div.snippet, div[class*="snippet"]')
+          ?.textContent?.trim() || "";
+
+      if (!url || !title) {
+        return; // continue to next iteration
+      }
+
+      // Extract metadata (source, time, author)
+      const metadata: string[] = [];
+      const source = element.querySelector(
+        'div.source, div[class*="source"]'
+      );
+
+      if (source) {
+        const ariaLabel = source
+          .querySelector("span[aria-label]")
+          ?.getAttribute("aria-label");
+        const author = link.getAttribute("data-author");
+
+        if (ariaLabel) {
+          metadata.push(ariaLabel);
+        }
+        if (author) {
+          metadata.push(author);
+        }
+      }
+
+      // Extract thumbnail
+      let thumbnail =
+        element
+          .querySelector('a.imagelink img, a[class*="imagelink"] img')
+          ?.getAttribute("src") || undefined;
+      if (thumbnail && !thumbnail.startsWith("https://www.bing.com")) {
+        thumbnail = "https://www.bing.com/" + thumbnail;
+      }
+
+      const metadataStr = metadata.length > 0 ? metadata.join(" | ") : "";
+      const fullContent = metadataStr
+        ? `${metadataStr}\n${content}`
+        : content;
+
+      results.push({
+        url,
+        title,
+        content: fullContent,
+        engine: "bing_news",
+      });
+    });
+
+  return results;
+};

package/src/sources/news/google_news.ts

@@ -0,0 +1,80 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const google_news: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    hl: "en",
+    gl: "US",
+    ceid: "US:en",
+  });
+  const response = await fetch(
+    `https://news.google.com/search?${params}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+        Cookie: "CONSENT=YES+; SOCS=CAESBQgYEgAgAA==",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const htmlString = await response.text();
+  const results: any[] = [];
+
+  if (!htmlString || typeof htmlString !== "string") {
+    return results;
+  }
+
+  const { document } = parseHTML(htmlString);
+
+  document.querySelectorAll("div.xrnccd").forEach((element) => {
+    const elElem = element;
+
+    // Extract the article link
+    const href = elElem.querySelector("article a")?.getAttribute("href");
+    if (!href) return;
+
+    // Decode the Google News internal link
+    try {
+      const hrefParts = href.split("/");
+      const encodedUrl = hrefParts[hrefParts.length - 1].split("?")[0];
+      const decodedBytes = Buffer.from(encodedUrl + "====", "base64url");
+      const decodedStr = decodedBytes.toString("utf-8");
+      const httpIndex = decodedStr.indexOf("http");
+      if (httpIndex === -1) return;
+      const actualUrl = decodedStr.slice(httpIndex).split("\xd2")[0];
+
+      const title =
+        elElem.querySelector("article h3")?.textContent?.trim() || "";
+      const pubDate =
+        elElem.querySelector("article time")?.textContent?.trim() || "";
+      const pubOrigin =
+        elElem
+          .querySelector("article a[data-n-tid]")
+          ?.textContent?.trim() || "";
+      const thumbnail =
+        elElem.previousElementSibling
+          ?.querySelector("figure img")
+          ?.getAttribute("src") || undefined;
+
+      const content = [pubOrigin, pubDate].filter(Boolean).join(" / ");
+
+      results.push({
+        url: actualUrl,
+        title,
+        content,
+        engine: "google_news",
+        thumbnail,
+      });
+    } catch (err) {
+      // Skip malformed URLs
+      return;
+    }
+  });
+
+  return results;
+};

package/src/sources/news/hackernews.ts

@@ -0,0 +1,94 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const hackernews: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const pageno = page || 1;
+  const resultsPerPage = 30;
+
+  let searchType = "search";
+  let queryParams: Record<string, any> = {
+    page: pageno - 1,
+  };
+
+  if (!query || query.trim() === "") {
+    // If search query is empty, show results from HN's front page
+    searchType = "search_by_date";
+    queryParams.tags = "front_page";
+  } else {
+    queryParams = {
+      query,
+      page: pageno - 1,
+      hitsPerPage: resultsPerPage,
+      minWordSizefor1Typo: 4,
+      minWordSizefor2Typos: 8,
+      advancedSyntax: true,
+      ignorePlurals: false,
+      minProximity: 7,
+      numericFilters: "[]",
+      tagFilters: '["story",[]]',
+      typoTolerance: true,
+      queryType: "prefixLast",
+      restrictSearchableAttributes:
+        '["title","comment_text","url","story_text","author"]',
+      getRankingInfo: true,
+    };
+  }
+
+  const searchParams = new URLSearchParams();
+  Object.entries(queryParams).forEach(([key, value]) => {
+    searchParams.append(key, String(value));
+  });
+
+  const response = await fetch(
+    `https://hn.algolia.com/api/v1/${searchType}?${searchParams.toString()}`,
+    {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (X1; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const jsonData = await response.json();
+  const results: any[] = [];
+
+  if (!jsonData?.hits) {
+    return results;
+  }
+
+  for (const hit of jsonData.hits) {
+    const objectId = hit.objectID;
+    const points = hit.points || 0;
+    const numComments = hit.num_comments || 0;
+
+    // Extract content from various possible fields
+    let content = hit.url || hit.comment_text || hit.story_text || "";
+
+    // Strip HTML tags from content if present
+    if (content) {
+      content = content.replace(/<[^>]*>/g, "").trim();
+    }
+
+    // Build metadata string
+    let metadata = "";
+    if (points !== 0 || numComments !== 0) {
+      metadata = `points: ${points} | comments: ${numComments}`;
+    }
+
+    // Combine content and metadata
+    const fullContent = metadata ? `${metadata}\n${content}` : content;
+
+    results.push({
+      title: hit.title || `author: ${hit.author}`,
+      url: `https://news.ycombinator.com/item?id=${objectId}`,
+      content: fullContent,
+      points,
+      numComments,
+      engine: "hackernews",
+    });
+  }
+
+  return results;
+};

package/src/sources/news/yahoo_news.ts

@@ -0,0 +1,77 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const yahoo_news: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const url = `https://news.search.yahoo.com/search?${new URLSearchParams({
+    p: query,
+  }).toString()}&b=${((page || 1) - 1) * 10 + 1}`;
+
+  const response = await fetch(url, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (X1; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+      Accept:
+        "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+      "Accept-Language": "en-US,en;q=0.5",
+    },
+  });
+  if (!response.ok) return [];
+  const html = await response.text();
+  const { document } = parseHTML(html);
+  const results: any[] = [];
+
+  // Parse results from Yahoo News search
+  document
+    .querySelectorAll('ol.searchCenterMiddle li, ol[class*="reg"] li')
+    .forEach((el) => {
+      const element = el;
+
+      const link = element.querySelector("h4 a, .fz-16 a, .title a");
+      let url = link?.getAttribute("href");
+
+      if (!url) {
+        return; // continue to next iteration
+      }
+
+      // Parse Yahoo's redirect URL to get the real URL
+      // Yahoo URLs are like: https://r.search.yahoo.com/_ylt=...;_ylu=.../RU=.../RK=.../RS=...
+      if (url.includes("r.search.yahoo.com") || url.includes("/RU=")) {
+        try {
+          const urlObj = new URL(url);
+          const ruMatch = url.match(/\/RU=([^/]+)/);
+          if (ruMatch) {
+            url = decodeURIComponent(ruMatch[1]);
+          } else if (urlObj.searchParams.has("url")) {
+            url = urlObj.searchParams.get("url") || url;
+          }
+        } catch (e) {
+          // If parsing fails, use URL as-is
+        }
+      }
+
+      const title = link?.textContent?.trim() || "";
+      const content =
+        element
+          .querySelector('p, .compText, div[class*="desc"]')
+          ?.textContent?.trim() || "";
+      const thumbnail =
+        element.querySelector("img")?.getAttribute("data-src") ||
+        element.querySelector("img")?.getAttribute("src") ||
+        "";
+
+      if (url && title) {
+        results.push({
+          url,
+          title,
+          content,
+          thumbnail,
+          engine: "yahoo_news",
+        });
+      }
+    });
+
+  return results;
+};

package/src/sources/shopping/ebay.ts

@@ -0,0 +1,96 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+/**
+ * eBay Shopping Search Engine
+ *
+ * Searches eBay marketplace for products, including:
+ * - Product listings
+ * - Prices and shipping info
+ * - Images and thumbnails
+ * - Source country information
+ */
+export const ebay: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    _nkw: query,
+    _sacat: String(page || 1),
+  });
+  const response = await fetch(
+    `https://www.ebay.com/sch/i.html?${params}`,
+    {
+      headers: {
+        Accept:
+          "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+      },
+    }
+  );
+  if (!response.ok) return [];
+  const html = await response.text();
+  const { document } = parseHTML(html);
+  const results: any[] = [];
+
+  // XPath equivalent: //li[contains(@class, "s-item")]
+  const resultels = document.querySelectorAll(
+    'li.s-item, li[class*="s-item"]'
+  );
+
+  for (const el of Array.from(resultels)) {
+    try {
+      // Extract data from DOM
+      const linkel = el.querySelector("a.s-item__link");
+      const url = linkel?.getAttribute("href");
+
+      const titleel = el.querySelector("h3.s-item__title");
+      const title = titleel?.textContent?.trim();
+
+      const contentel = el.querySelector('div[span="SECONDARY_INFO"]');
+      const content = contentel?.textContent?.trim();
+
+      const priceel = el.querySelector(
+        "div.s-item__detail span.s-item__price, span.s-item__price"
+      );
+      const price = priceel?.textContent?.trim();
+
+      const shippingel = el.querySelector(
+        'span.s-item__shipping, span[class*="s-item__shipping"]'
+      );
+      const shipping = shippingel?.textContent?.trim();
+
+      const locationel = el.querySelector(
+        'span.s-item__location, span[class*="s-item__location"]'
+      );
+      const source_country = locationel?.textContent?.trim();
+
+      const thumbnailel = el.querySelector("img.s-item__image-img");
+      const thumbnail = thumbnailel?.getAttribute("src");
+
+      // Skip if no title
+      if (!title || title === "") {
+        continue;
+      }
+
+      const result: any = {
+        url: url || "",
+        title,
+        content: content || "",
+        price,
+        shipping,
+        thumbnail,
+        source_country,
+        template: "products",
+        engine: "ebay",
+      };
+
+      results.push(result);
+    } catch (error) {
+      // Skip malformed results
+      continue;
+    }
+  }
+
+  return results;
+};