search-web-api 1.0.13

package/src/sources/videos/invidious.ts

@@ -0,0 +1,86 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+// Public Invidious instances - user can configure their own
+const DEFAULT_INSTANCES = [
+  "https://invidious.nerdvpn.de",
+  "https://invidious.private.coffee",
+  "https://inv.nadeko.net",
+];
+
+export const invidious: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const baseUrl = DEFAULT_INSTANCES[0];
+  const params = new URLSearchParams({
+    q: query,
+    page: String(page || 1),
+  });
+
+  const response = await fetch(`${baseUrl}/api/v1/search?${params}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    },
+  });
+  if (!response.ok) return [];
+  const json = await response.json();
+  const results = [] as any[];
+
+  if (!json || !Array.isArray(json)) {
+    return results;
+  }
+
+  for (const result of json) {
+    if (result.type !== "video") {
+      continue;
+    }
+
+    const videoId = result.videoId;
+    if (!videoId) continue;
+
+    const url = `${baseUrl}/watch?v=${videoId}`;
+
+    // Find thumbnail
+    const thumbs = result.videoThumbnails || [];
+    let thumbnail = "";
+    const sdThumb = thumbs.find((t: any) => t.quality === "sddefault");
+    if (sdThumb) {
+      thumbnail = sdThumb.url;
+      // Handle partial URLs
+      if (thumbnail && !thumbnail.startsWith("http")) {
+        thumbnail = baseUrl + thumbnail;
+      }
+    }
+
+    // Format duration
+    const lengthSeconds = result.lengthSeconds || 0;
+    const hours = Math.floor(lengthSeconds / 3600);
+    const minutes = Math.floor((lengthSeconds % 3600) / 60);
+    const seconds = lengthSeconds % 60;
+    const length =
+      hours > 0
+        ? `${hours}:${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`
+        : `${minutes}:${seconds.toString().padStart(2, "0")}`;
+
+    const content = [
+      result.description || "",
+      result.author ? `By: ${result.author}` : "",
+      result.viewCount ? `Views: ${result.viewCount.toLocaleString()}` : "",
+      length ? `Duration: ${length}` : "",
+    ]
+      .filter(Boolean)
+      .join(" | ");
+
+    results.push({
+      url,
+      title: result.title || "",
+      content,
+      engine: "invidious",
+      thumbnail,
+      iframe_src: `${baseUrl}/embed/${videoId}`,
+    });
+  }
+
+  return results;
+};

package/src/sources/videos/peertube.ts

@@ -0,0 +1,76 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const peertube: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const baseUrl = "https://peer.tube";
+  const start = ((page || 1) - 1) * 10;
+  const params = new URLSearchParams({
+    search: query,
+    searchTarget: "search-index",
+    resultType: "videos",
+    start: String(start),
+    count: "10",
+    sort: "-match",
+    nsfw: "false",
+  });
+
+  const response = await fetch(`${baseUrl}/api/v1/search/videos?${params}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    },
+  });
+  if (!response.ok) return [];
+  const json = await response.json();
+  const results = [] as any[];
+
+  if (!json || !json.data) {
+    return results;
+  }
+
+  for (const result of json.data) {
+    const channel = result.channel || {};
+    const account = result.account || {};
+
+    // Format duration
+    const duration = result.duration || 0;
+    const hours = Math.floor(duration / 3600);
+    const minutes = Math.floor((duration % 3600) / 60);
+    const seconds = duration % 60;
+    const length =
+      hours > 0
+        ? `${hours}:${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`
+        : `${minutes}:${seconds.toString().padStart(2, "0")}`;
+
+    const metadata = [
+      channel.displayName,
+      `${channel.name}@${channel.host}`,
+      result.tags ? result.tags.join(", ") : "",
+    ]
+      .filter(Boolean)
+      .join(" | ");
+
+    const content = [
+      result.description || "",
+      `Views: ${result.views?.toLocaleString() || 0}`,
+      length ? `Duration: ${length}` : "",
+      metadata,
+    ]
+      .filter(Boolean)
+      .join("\n");
+
+    results.push({
+      url: result.url,
+      title: result.name,
+      content,
+      engine: "peertube",
+      thumbnail: result.thumbnailUrl || result.previewUrl,
+      iframe_src: result.embedUrl,
+      author: account.displayName,
+    });
+  }
+
+  return results;
+};

package/src/sources/videos/vimeo.ts

@@ -0,0 +1,67 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const vimeo: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    q: query,
+    page: String(page || 1),
+  });
+  const response = await fetch(`https://vimeo.com/search?${params}`, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (X1; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+      "Accept-Language": "en-US,en;q=0.5",
+    },
+  });
+  if (!response.ok) return [];
+  const html = await response.text();
+  const results = [] as any[];
+
+  if (!html || typeof html !== "string") {
+    return results;
+  }
+
+  try {
+    // Extract JSON data from the page
+    const dataMatch = html.match(/var data = ({.*?});/s);
+    if (!dataMatch) {
+      return results;
+    }
+
+    const parsedData = JSON.parse(dataMatch[1]);
+    const filteredData = parsedData?.filtered?.data || [];
+
+    for (const resultItem of filteredData) {
+      const type = resultItem.type;
+      const result = resultItem[type];
+
+      if (!result) continue;
+
+      const videoId = result.uri?.split("/").pop();
+      if (!videoId) continue;
+
+      const url = `https://vimeo.com/${videoId}`;
+      const title = result.name || "";
+      const thumbnail =
+        result.pictures?.sizes?.slice(-1)[0]?.link || "";
+      const publishedDate = result.created_time || "";
+
+      results.push({
+        url,
+        title,
+        content: publishedDate
+          ? `Published: ${publishedDate.split("T")[0]}`
+          : "",
+        thumbnail,
+        engine: "vimeo",
+      });
+    }
+  } catch (e) {
+    console.error("Error parsing Vimeo response:", e);
+  }
+
+  return results;
+};

package/src/sources/videos/youtube.ts

@@ -0,0 +1,78 @@
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+// Multiple Invidious instances for fallback
+const INVIDIOUS_INSTANCES = [
+  "https://inv.nadeko.net",
+  "https://invidious.privacyredirect.com",
+  "https://yewtu.be",
+  "https://invidious.nerdvpn.de",
+  "https://inv.riverside.rocks",
+];
+
+async function tryInvidiousInstances(
+  query: string,
+  page: number | undefined
+): Promise<any> {
+  for (const instance of INVIDIOUS_INSTANCES) {
+    try {
+      const params = new URLSearchParams({
+        q: query,
+        page: String(page || 1),
+        type: "video",
+      });
+      const response = await fetch(`${instance}/api/v1/search?${params}`, {
+        headers: {
+          "User-Agent": "HonoxSearX/1.0",
+        },
+        signal: AbortSignal.timeout(5000),
+      });
+
+      if (!response.ok) continue;
+      const json = await response.json();
+      const results = [] as any[];
+
+      if (Array.isArray(json)) {
+        json.forEach((item: any) => {
+          if (item.type === "video") {
+            const duration = item.lengthSeconds
+              ? `${Math.floor(item.lengthSeconds / 60)}:${String(item.lengthSeconds % 60).padStart(2, "0")}`
+              : "";
+            const views = item.viewCount
+              ? item.viewCount.toLocaleString()
+              : "";
+            const published = item.publishedText || "";
+
+            results.push({
+              url: `https://www.youtube.com/watch?v=${item.videoId}`,
+              title: item.title,
+              content: `${item.description || ""} | ${duration} | ${views} views | ${published}`.trim(),
+              thumbnail:
+                item.videoThumbnails && item.videoThumbnails.length > 0
+                  ? item.videoThumbnails[0].url
+                  : undefined,
+              engine: "youtube",
+            });
+          }
+        });
+      }
+
+      // If successful, return the result
+      if (results.length > 0) {
+        return results;
+      }
+    } catch (error) {
+      // Try next instance
+      continue;
+    }
+  }
+
+  // All instances failed, return empty array
+  return [];
+}
+
+export const youtube: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  return await tryInvidiousInstances(query, page);
+};

package/src/suggest-next-words/autocomplete-ai.ts

@@ -0,0 +1,38 @@
+import { pipeline, type TextGenerationSingle } from "@huggingface/transformers";
+
+/**
+ * Generates the next words for a given prompt using a text generation model.
+ *
+ * The DistilGPT2 model is a distilled, smaller version of OpenAI's GPT-2, featuring
+ * around 82 million parameters (~80MB), and available in Hugging Face Transformers
+ * for efficient next word (next-token) prediction tasks in English text. It is
+ * well-suited for sequence generation and can be used as a next word prediction
+ * model with code similar to GPT2, but offers faster and lighter inference compared
+ * GPT-2 (124M parameter) model, preserving most capabilities but running
+ * significantly faster and requiring less memory.
+ * @see https://huggingface.co/Xenova/distilgpt2
+ * @param {string} prompt - The input prompt to complete.
+ * @param {Object} [options] - Generation options.
+ * @param {number} [options.maxTokens=16] - Maximum number of new tokens to generate.
+ * @param {string} [options.model='Xenova/distilgpt2'] - The model identifier to use.
+ * @returns {Promise<string>} - The generated text completions.
+ */
+export async function predictNextWordsWithSmallLocalModel(
+  prompt: string,
+  { model = "Xenova/distilgpt2", maxTokens = 16, modelParams = {} } = {}
+): Promise<string> {
+  if (!prompt) throw new Error("Prompt is required");
+  return (
+    await (
+      await pipeline("text-generation", model, {
+        dtype: "fp16",
+        device: "cpu",
+        ...modelParams,
+      })
+    )(prompt, { max_new_tokens: maxTokens })
+  )
+    .map((res) => (res as TextGenerationSingle).generated_text)
+    .join(" ")
+    .replace(prompt, "")
+    .trim();
+}