search-web-api 1.0.13

package/src/types/search-engine-interface.ts

@@ -0,0 +1,27 @@
+/**
+ * Search Engine Interface — Re-exports the core engine types from the canonical
+ * types module and provides the `extractResponseData` utility for normalising
+ * raw grab-url responses into plain data values. All engine source files
+ * import `EngineFunction` and `EngineResult` from here.
+ */
+
+import { EngineResult, Engine, EngineFunction } from "./search-result-types.js";
+
+export { EngineResult, Engine, EngineFunction };
+
+/**
+ * Extract the actual data from a grab-url response.
+ * grab-url returns `{ data: actualContent, error?: string }` or just the data directly.
+ */
+export function extractResponseData(response: any): any {
+  if (!response) return "";
+
+  if (typeof response === "string") return response;
+
+  // Error envelope — return empty string to allow graceful handling
+  if (typeof response === "object" && "error" in response) return "";
+
+  if ("data" in response) return response.data;
+
+  return response;
+}

package/src/types/search-result-types.ts

@@ -0,0 +1,405 @@
+/**
+ * Search Result Types — TypeScript interfaces and types for search results,
+ * engine configurations, scoring structures, and metadata used throughout
+ * the search API. All shared data contracts live here so every module
+ * can import from one canonical location.
+ */
+
+/**
+ * Search Engine Result Item
+ */
+export interface EngineResult {
+  /**
+   * The destination URL of the search result.
+   */
+  url?: string;
+  /**
+   * Alternative link field (same as url, depends on engine).
+   */
+  link?: string;
+  /**
+   * The title of the search result.
+   */
+  title: string;
+  /**
+   * The snippet/content description of the search result.
+   */
+  content: string;
+  /**
+   * Source URL for a thumbnail image if available.
+   */
+  thumbnail?: string;
+  /**
+   * The name of the engine that produced this result.
+   */
+  engine?: string;
+  /**
+   * If the result contains an iframe (e.g., embedded video), its source URL.
+   */
+  iframe_src?: string;
+  /**
+   * Author or creator of the content.
+   */
+  author?: string;
+  /**
+   * Latitude coordinate for location-based results.
+   */
+  latitude?: number;
+  /**
+   * Longitude coordinate for location-based results.
+   */
+  longitude?: number;
+  /**
+   * Main image source URL for the result.
+   */
+  img_src?: string;
+  /**
+   * Date the content was published, usually as a string representation.
+   */
+  publishedDate?: string;
+  /**
+   * The category of the result (e.g., general, images, videos).
+   */
+  category?: string;
+  /**
+   * Template to be used for rendering this result in the UI.
+   */
+  template?: string;
+}
+
+/**
+ * Definition of a Search Engine
+ */
+export interface Engine {
+  /**
+   * The unique identifier/name of the engine.
+   */
+  name: string;
+  /**
+   * Array of categories this engine supports.
+   */
+  categories?: string[];
+  /**
+   * Function to handle preparing and making the search request.
+   */
+  request: (query: string, params?: any) => Promise<any>;
+  /**
+   * Function to handle formatting the raw response into EngineResult[].
+   */
+  response: (response: any) => Promise<EngineResult[]>;
+}
+
+/**
+ * Modern Search Engine Function signature
+ */
+export interface EngineFunction {
+  (
+    /**
+     * The search query words and phrases
+     */
+    query: string,
+    /**
+     * The page number (optional)
+     */
+    page?: number,
+    /**
+     * Base domain to use for the request (optional)
+     */
+    baseDomain?: number,
+  ): Promise<EngineResult[]>;
+}
+
+/**
+ * Result Priority Level
+ */
+export type PriorityType = "low" | "normal" | "high";
+
+/**
+ * A combined result that merges duplicate results across different engines
+ */
+export interface MergedResult extends EngineResult {
+  /**
+   * Array of engine names that returned this result.
+   */
+  engines: string[];
+  /**
+   * The original rank positions of this result from its respective engines.
+   */
+  positions: number[];
+  /**
+   * A calculated score used for final ranking.
+   */
+  score: number;
+  /**
+   * Priority constraint for ranking calculations.
+   */
+  priority: PriorityType;
+  /**
+   * The primary category assigned to this merged result.
+   */
+  category?: string;
+  /**
+   * Desired UI template string.
+   */
+  template?: string;
+  /**
+   * A hashed unique identifier based on URL and title for deduplication.
+   */
+  resultHash?: string;
+}
+
+/**
+ * Weights assigned to individual engines for scoring algorithms
+ */
+export interface EngineWeight {
+  /**
+   * A numeric weight assigned to a specific engine.
+   */
+  [engineName: string]: number;
+}
+
+/**
+ * Weights assigned to specific categories
+ */
+export interface CategoryWeight {
+  /**
+   * A numeric weight assigned to a specific category.
+   */
+  [category: string]: number;
+}
+
+/**
+ * A rich entity or infobox result, often rendered separately from regular links
+ */
+export interface Infobox {
+  /**
+   * Title or main label of the infobox entity.
+   */
+  infobox: string;
+  /**
+   * Unique identifier for the infobox entity.
+   */
+  id?: string;
+  /**
+   * Text snippet or description of the entity.
+   */
+  content?: string;
+  /**
+   * Source URL for the primary image associated with the entity.
+   */
+  img_src?: string;
+  /**
+   * Important URLs related to the entity (e.g., Wikipedia page, official site).
+   */
+  urls?: Array<{
+    /**
+     * Link label or title.
+     */
+    title: string;
+    /**
+     * Target URL.
+     */
+    url: string;
+    /**
+     * Optional entity context.
+     */
+    entity?: string;
+  }>;
+  /**
+   * Key-value attributes describing the entity.
+   */
+  attributes?: Array<{
+    /**
+     * Label or name of the attribute.
+     */
+    label: string;
+    /**
+     * Value of the attribute.
+     */
+    value: string;
+    /**
+     * Optional entity context.
+     */
+    entity?: string;
+  }>;
+  /**
+   * The primary engine that provided this infobox.
+   */
+  engine: string;
+  /**
+   * Set of engines that provided matching infoboxes (for merging).
+   */
+  engines: Set<string>;
+}
+
+/**
+ * Tracks an engine that failed to respond or errored out
+ */
+export interface UnresponsiveEngine {
+  /**
+   * Name of the unresponsive engine.
+   */
+  engine: string;
+  /**
+   * Type or description of the error encountered.
+   */
+  errorType: string;
+  /**
+   * Whether the engine is currently suspended from further queries.
+   */
+  suspended: boolean;
+}
+
+/**
+ * Performance metric for an individual engine's request
+ */
+export interface Timing {
+  /**
+   * Name of the engine being timed.
+   */
+  engine: string;
+  /**
+   * Total elapsed time in milliseconds.
+   */
+  total: number;
+  /**
+   * The time taken specifically for network parsing or loading.
+   */
+  load: number;
+}
+
+/**
+ * Miscellaneous data returned by engines
+ */
+export interface EngineData {
+  /**
+   * Holds arbitrary key-value context or metadata specific to an engine.
+   */
+  [engineName: string]: {
+    [key: string]: any;
+  };
+}
+
+/**
+ * Configuration for a Search Category
+ */
+export interface CategoryConfig {
+  /**
+   * Internal ID or key name of the category.
+   */
+  name: string;
+  /**
+   * User-facing display name.
+   */
+  displayName: string;
+  /**
+   * Short description of what this category entails.
+   */
+  description: string;
+  /**
+   * Standard scoring multiplier for this category.
+   */
+  defaultWeight: number;
+}
+
+/**
+ * Metadata for a search engine mapping
+ */
+export interface EngineMetadata {
+  /**
+   * The name/identifier of the engine.
+   */
+  name: string;
+  /**
+   * The asynchronous function that runs the actual engine query.
+   */
+  fn: EngineFunction;
+  /**
+   * Categories the engine operates under.
+   */
+  categories: string[];
+  /**
+   * A short description or summary of the engine.
+   */
+  description?: string;
+}
+
+/**
+ * Engine Status Tracking
+ */
+export interface EngineStatus {
+  /**
+   * Name of the engine.
+   */
+  name: string;
+  /**
+   * Current health status of the engine.
+   */
+  status: "active" | "failed" | "disabled";
+  /**
+   * Timestamp of the last health check.
+   */
+  lastCheck: Date;
+  /**
+   * Timestamp of the last successful request.
+   */
+  lastSuccess: Date | null;
+  /**
+   * Timestamp of the last failed request.
+   */
+  lastFailure: Date | null;
+  /**
+   * Total number of consecutive failures.
+   */
+  failureCount: number;
+  /**
+   * Total number of successful requests.
+   */
+  successCount: number;
+  /**
+   * Total number of requests made to this engine.
+   */
+  totalRequests: number;
+  /**
+   * Average response time in milliseconds.
+   */
+  averageResponseTime: number;
+  /**
+   * The last error message encountered.
+   */
+  lastError?: string;
+  /**
+   * Categories this engine belongs to.
+   */
+  categories: string[];
+}
+
+/**
+ * Log entry for an engine request
+ */
+export interface EngineLog {
+  /**
+   * Timestamp of the log entry.
+   */
+  timestamp: Date;
+  /**
+   * Name of the engine.
+   */
+  engineName: string;
+  /**
+   * Status of the request.
+   */
+  status: "success" | "failure";
+  /**
+   * Response time in milliseconds.
+   */
+  responseTime: number;
+  /**
+   * Error message if the request failed.
+   */
+  error?: string;
+  /**
+   * The query that was made.
+   */
+  query?: string;
+}

package/test/api.test.ts

@@ -0,0 +1,128 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { Hono } from 'hono';
+import { Search } from '../src/search';
+import { validateEngineResults, TEST_QUERIES } from './test-utils';
+
+// Mock API setup
+const app = new Hono();
+const search = new Search();
+
+app.get('/search', async (c) => {
+    const query = c.req.query('q') || '';
+    const pageno = parseInt(c.req.query('pageno') || '1');
+    const engines = c.req.query('engines')?.split(',').filter(Boolean);
+    const categories = c.req.query('categories')?.split(',').filter(Boolean);
+
+    if (!query) {
+        return c.json({ error: 'Query parameter is required' }, 400);
+    }
+
+    try {
+        const results = await search.search(query, pageno, engines, categories);
+        return c.json({
+            query,
+            pageno,
+            engines: engines || 'all',
+            categories: categories || 'all',
+            results
+        });
+    } catch (error) {
+        return c.json({ error: 'Search failed', message: (error as Error).message }, 500);
+    }
+});
+
+describe('API Integration Tests', () => {
+    it('should handle basic search query', async () => {
+        const req = new Request('http://localhost/search?q=javascript');
+        const res = await app.request(req);
+        const data = await res.json();
+
+        expect(res.status).toBe(200);
+        expect(data.query).toBe('javascript');
+        expect(data.results).toBeDefined();
+        expect(Array.isArray(data.results)).toBe(true);
+        expect(validateEngineResults(data.results)).toBe(true);
+    }, 60000);
+
+    it('should require query parameter', async () => {
+        const req = new Request('http://localhost/search');
+        const res = await app.request(req);
+        const data = await res.json();
+
+        expect(res.status).toBe(400);
+        expect(data.error).toBe('Query parameter is required');
+    });
+
+    it('should filter by engines', async () => {
+        const req = new Request('http://localhost/search?q=react&engines=github,npm');
+        const res = await app.request(req);
+        const data = await res.json();
+
+        expect(res.status).toBe(200);
+        expect(data.engines).toEqual(['github', 'npm']);
+
+        if (data.results.length > 0) {
+            data.results.forEach((result: any) => {
+                expect(['github', 'npm']).toContain(result.engine);
+            });
+        }
+    }, 60000);
+
+    it('should filter by categories', async () => {
+        const req = new Request('http://localhost/search?q=ubuntu&categories=torrent');
+        const res = await app.request(req);
+        const data = await res.json();
+
+        expect(res.status).toBe(200);
+        expect(data.categories).toEqual(['torrent']);
+    }, 60000);
+
+    it('should handle pagination', async () => {
+        const req = new Request('http://localhost/search?q=python&pageno=2');
+        const res = await app.request(req);
+        const data = await res.json();
+
+        expect(res.status).toBe(200);
+        expect(data.pageno).toBe(2);
+    }, 60000);
+
+    describe('Common Query Tests', () => {
+        it('should search for "javascript"', async () => {
+            const req = new Request('http://localhost/search?q=javascript');
+            const res = await app.request(req);
+            const data = await res.json();
+
+            expect(res.status).toBe(200);
+            expect(data.results.length).toBeGreaterThan(0);
+        }, 60000);
+
+        it('should search for "linux"', async () => {
+            const req = new Request('http://localhost/search?q=linux');
+            const res = await app.request(req);
+            const data = await res.json();
+
+            expect(res.status).toBe(200);
+            expect(data.results.length).toBeGreaterThan(0);
+        }, 60000);
+
+        it('should search for "ubuntu torrent" in torrent engines', async () => {
+            const req = new Request('http://localhost/search?q=ubuntu&categories=torrent');
+            const res = await app.request(req);
+            const data = await res.json();
+
+            expect(res.status).toBe(200);
+            expect(data.results).toBeDefined();
+        }, 60000);
+
+        it('should search for "github" in code engines', async () => {
+            const req = new Request('http://localhost/search?q=react&engines=github');
+            const res = await app.request(req);
+            const data = await res.json();
+
+            expect(res.status).toBe(200);
+            if (data.results.length > 0) {
+                expect(data.results[0].engine).toBe('github');
+            }
+        }, 60000);
+    });
+});

package/test/autocomplete-ai.test.ts

@@ -0,0 +1,20 @@
+import { predictNextWordsWithSmallLocalModel } from "../src/suggest-next-words/autocomplete-ai";
+import { describe, it, expect } from "vitest";
+
+describe("predictNextWordsWithSmallLocalModel", () => {
+  it("should predict next words to complete the sentence", async () => {
+    const samplePrompts = [
+      // "ai broker has a lot of potential to",
+      // "what can help build ai is that",
+      "well-suited for sequence generation and can be",
+      "the flagship model is estimated to have",
+    ];
+
+    for (const prompt of samplePrompts) {
+      const result = await predictNextWordsWithSmallLocalModel(prompt);
+      console.log(prompt, "--->", result);
+
+      expect(result).toBeDefined();
+    }
+  });
+});

package/test/autocomplete-engines.test.ts

@@ -0,0 +1,131 @@
+/**
+ * Autocomplete Example
+ *
+ * Demonstrates how to use the autocomplete functionality
+ * to get search query suggestions from various search engines.
+ */
+
+import {
+  searchAutocomplete,
+  searchAutocompleteMulti,
+  backends,
+  // Individual backend functions
+  google,
+  duckduckgo,
+  brave,
+  wikipedia,
+  baidu,
+  qwant,
+  startpage,
+  yandex,
+} from "../src/suggest-next-words/autocomplete-search-engines";
+
+async function main() {
+  console.log("=== Search Autocomplete Examples ===\n");
+
+  // Example 1: Single backend autocomplete
+  console.log('Example 1: Google autocomplete for "typescript"');
+  const googleSuggestions = await searchAutocomplete("google", "typescript");
+  console.log("Suggestions:", googleSuggestions);
+  console.log("");
+
+  // Example 2: DuckDuckGo autocomplete
+  console.log('Example 2: DuckDuckGo autocomplete for "javascript"');
+  const ddgSuggestions = await searchAutocomplete("duckduckgo", "javascript");
+  console.log("Suggestions:", ddgSuggestions);
+  console.log("");
+
+  // Example 3: Wikipedia autocomplete with locale
+  console.log('Example 3: Wikipedia autocomplete for "quantum" (English)');
+  const wikiSuggestions = await searchAutocomplete(
+    "wikipedia",
+    "quantum",
+    "en-US"
+  );
+  console.log("Suggestions:", wikiSuggestions);
+  console.log("");
+
+  // Example 4: Multi-backend autocomplete
+  console.log('Example 4: Multi-backend autocomplete for "python"');
+  const multiSuggestions = await searchAutocompleteMulti(
+    ["google", "duckduckgo", "wikipedia"],
+    "python"
+  );
+  console.log("Merged suggestions:", multiSuggestions);
+  console.log("Total unique suggestions:", multiSuggestions.length);
+  console.log("");
+
+  // Example 5: Using backend functions directly
+  console.log("Example 5: Direct backend function calls");
+
+  console.log('\nGoogle suggestions for "react":');
+  const reactGoogle = await google("react");
+  console.log(reactGoogle);
+
+  console.log('\nBrave suggestions for "node":');
+  const nodeBrave = await brave("node");
+  console.log(nodeBrave);
+
+  console.log('\nWikipedia suggestions for "artificial intelligence":');
+  const aiWiki = await wikipedia("artificial intelligence", "en");
+  console.log(aiWiki);
+  console.log("");
+
+  // Example 6: Autocomplete with different locales
+  console.log("Example 6: Locale-specific autocomplete");
+
+  console.log("\nGoogle (English):");
+  const enGoogle = await google("machine learning", "en");
+  console.log(enGoogle.slice(0, 5));
+
+  console.log("\nGoogle (German):");
+  const deGoogle = await google("machine learning", "de");
+  console.log(deGoogle.slice(0, 5));
+
+  console.log("\nGoogle (French):");
+  const frGoogle = await google("machine learning", "fr");
+  console.log(frGoogle.slice(0, 5));
+  console.log("");
+
+  // Example 7: List all available backends
+  console.log("Example 7: Available autocomplete backends");
+  const availableBackends = Object.keys(backends);
+  console.log("Available backends:", availableBackends);
+  console.log("Total:", availableBackends.length);
+  console.log("");
+
+  // Example 8: Comparing results from different backends
+  console.log('Example 8: Comparing autocomplete results for "climate change"');
+  const query = "climate change";
+
+  for (const backendName of ["google", "duckduckgo", "brave", "wikipedia"]) {
+    const suggestions = await searchAutocomplete(backendName, query);
+    console.log(`\n${backendName}: ${suggestions.length} suggestions`);
+    console.log(
+      suggestions
+        .slice(0, 3)
+        .map((s) => `  - ${s}`)
+        .join("\n")
+    );
+  }
+  console.log("");
+
+  // Example 9: Asian language autocomplete
+  console.log("Example 9: Chinese autocomplete with Baidu");
+  const chineseSuggestions = await baidu("人工智能"); // "artificial intelligence" in Chinese
+  console.log("Suggestions:", chineseSuggestions);
+  console.log("");
+
+  console.log("Example 10: Russian autocomplete with Yandex");
+  const russianSuggestions = await yandex("программирование"); // "programming" in Russian
+  console.log("Suggestions:", russianSuggestions);
+  console.log("");
+
+  console.log("=== Examples Complete ===");
+}
+
+// Run the examples
+main().catch((error) => {
+  console.error("Error running examples:", error);
+  process.exit(1);
+});