search-web-api 1.0.13

package/src/search/search-query-executor.ts

@@ -0,0 +1,168 @@
+/**
+ * Search Query Executor — The main `Search` class that orchestrates a query
+ * across any subset of the registered engine adapters. It initialises engine
+ * health tracking and category registration on construction, runs engines in
+ * parallel with configurable per-engine and per-category weights, feeds results
+ * into a `ResultContainer` for deduplication and scoring, and returns a
+ * deduplicated, ranked list of `MergedResult` objects.
+ */
+
+import { engineStatusTracker } from "../registry/search-engine-status-tracker.js";
+import { categoryRegistry, CATEGORIES } from "../registry/search-engine-category-registry.js";
+import { ResultContainer } from "./search-result-container.js";
+import { ALL_ENGINES } from "./search-engines-registry-list.js";
+import { MergedResult, CategoryWeight, EngineMetadata } from "../types/search-result-types.js";
+
+export class Search {
+  private engines: EngineMetadata[] = ALL_ENGINES;
+
+  constructor() {
+    for (const engine of this.engines) {
+      engineStatusTracker.initEngine(engine.name, engine.categories);
+      categoryRegistry.registerEngine({
+        name: engine.name,
+        categories: engine.categories,
+        request: async () => {},
+        response: async () => [],
+      });
+    }
+  }
+
+  /**
+   * Run a search query against the specified engines or categories.
+   *
+   * @param query - Search terms to query.
+   * @param pageno - Result page number (default 1).
+   * @param engineNames - If provided, restrict to these engine names.
+   * @param categories - If provided (and engineNames is empty), restrict to these categories.
+   * @returns Deduplicated, scored, and grouped results.
+   */
+  async search(
+    query: string,
+    pageno: number = 1,
+    engineNames?: string[],
+    categories?: string[],
+  ): Promise<MergedResult[]> {
+    const resultContainer = new ResultContainer();
+    const encodedQuery = query.trim();
+
+    const engineWeights: { [key: string]: number } = {
+      google: 1.5,
+      bing: 1.3,
+      duckduckgo: 1.2,
+      brave: 1.1,
+      startpage: 1.1,
+      google_scholar: 1.4,
+      semantic_scholar: 1.3,
+      arxiv: 1.3,
+    };
+    resultContainer.setEngineWeights(engineWeights);
+
+    const categoryWeights: CategoryWeight = {};
+    for (const [key, config] of Object.entries(CATEGORIES)) {
+      categoryWeights[key] = config.defaultWeight;
+    }
+    resultContainer.setCategoryWeights(categoryWeights);
+
+    let enginesToUse: EngineMetadata[];
+
+    if (engineNames && engineNames.length > 0) {
+      enginesToUse = this.engines.filter((e) => engineNames.includes(e.name));
+    } else if (categories && categories.length > 0) {
+      enginesToUse = this.engines.filter((e) =>
+        e.categories.some((cat) => categories.includes(cat)),
+      );
+    } else {
+      enginesToUse = this.engines;
+    }
+
+    const promises = enginesToUse
+      .filter((engine) => {
+        if (!engineStatusTracker.isEngineHealthy(engine.name)) {
+          console.log(`Skipping unhealthy engine: ${engine.name}`);
+          return false;
+        }
+        return true;
+      })
+      .map(async (engine) => {
+        const startTime = Date.now();
+        try {
+          const engineResults = await engine.fn(encodedQuery, pageno);
+          const responseTime = Date.now() - startTime;
+          engineStatusTracker.recordSuccess(engine.name, responseTime, query);
+
+          if (engineResults && Array.isArray(engineResults)) {
+            for (const result of engineResults) {
+              if (!result.category && engine.categories.length > 0) {
+                result.category = engine.categories[0];
+              }
+            }
+            resultContainer.extend(engine.name, engineResults);
+          }
+        } catch (error) {
+          const responseTime = Date.now() - startTime;
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          engineStatusTracker.recordFailure(engine.name, errorMessage, responseTime, query);
+          console.error(`Error in engine ${engine.name}:`, errorMessage);
+        }
+      });
+
+    await Promise.all(promises);
+
+    resultContainer.close();
+
+    const orderedResults = resultContainer.getOrderedResults();
+
+    const stats = resultContainer.getStats();
+    console.log(
+      `Search "${query}": ${stats.totalResults} results, ${stats.duplicatesMerged} merged, avg ${stats.engineCoverage.toFixed(1)} engines/result`,
+    );
+
+    return orderedResults;
+  }
+
+  /**
+   * Search across multiple categories and combine the results.
+   *
+   * @param query - Search terms.
+   * @param categories - Categories to include (e.g. `['general', 'news', 'academic']`).
+   * @param pageno - Page number (default 1).
+   */
+  async searchByCategories(
+    query: string,
+    categories: string[],
+    pageno: number = 1,
+  ): Promise<MergedResult[]> {
+    return this.search(query, pageno, undefined, categories);
+  }
+
+  /** Return all registered engine names. */
+  getEngines(): string[] {
+    return this.engines.map((e) => e.name);
+  }
+
+  /** Return engine names that belong to the given category. */
+  getEnginesByCategory(category: string): string[] {
+    return this.engines.filter((e) => e.categories.includes(category)).map((e) => e.name);
+  }
+
+  /** Return all registered category names. */
+  getCategories(): string[] {
+    return categoryRegistry.getCategories();
+  }
+
+  /** Return per-category engine counts and totals. */
+  getCategoryStats() {
+    return categoryRegistry.getStats();
+  }
+
+  /** Return the current health status for a named engine. */
+  getEngineStatus(engineName: string) {
+    return engineStatusTracker.getStatus(engineName);
+  }
+
+  /** Return health status records for all registered engines. */
+  getAllEngineStatuses() {
+    return engineStatusTracker.getAllStatuses();
+  }
+}

package/src/search/search-result-container.ts

@@ -0,0 +1,421 @@
+/**
+ * Search Result Container — Central aggregator for results returned by multiple
+ * search engine adapters. Handles URL-based deduplication, merging of duplicate
+ * results (choosing longer content, preferring HTTPS URLs, unioning engine lists),
+ * position-weighted scoring with per-engine and per-category multipliers, and a
+ * two-pass category-grouping algorithm that clusters similar results together in
+ * the final output. Also tracks infoboxes, suggestions, answers, corrections,
+ * unresponsive engines, and per-engine timing metrics.
+ */
+
+import { EngineResult } from "../types/search-engine-interface.js";
+import {
+  PriorityType,
+  MergedResult,
+  EngineWeight,
+  CategoryWeight,
+  Infobox,
+  UnresponsiveEngine,
+  Timing,
+  EngineData,
+} from "../types/search-result-types.js";
+
+export class ResultContainer {
+  private mainResultsMap: Map<string, MergedResult> = new Map();
+  private mainResultsSorted: MergedResult[] | null = null;
+  private closed: boolean = false;
+  private engineWeights: EngineWeight = {};
+  private categoryWeights: CategoryWeight = {};
+
+  public infoboxes: Infobox[] = [];
+  public suggestions: Set<string> = new Set();
+  public answers: Map<string, any> = new Map();
+  public corrections: Set<string> = new Set();
+
+  public unresponsiveEngines: Set<UnresponsiveEngine> = new Set();
+  public timings: Timing[] = [];
+  public engineData: EngineData = {};
+
+  private numberofResults: number[] = [];
+  public paging: boolean = false;
+  public redirectUrl: string | null = null;
+
+  public onResult: (result: any) => boolean = () => true;
+
+  /**
+   * Configure engine weights for scoring. Higher weight = more influence from this engine.
+   */
+  setEngineWeights(weights: EngineWeight) {
+    this.engineWeights = weights;
+  }
+
+  /**
+   * Configure category weights for scoring. Applied as a multiplier on top of engine weights.
+   */
+  setCategoryWeights(weights: CategoryWeight) {
+    this.categoryWeights = weights;
+  }
+
+  private getEngineWeight(engineName: string): number {
+    return this.engineWeights[engineName] || 1.0;
+  }
+
+  private getCategoryWeight(category: string): number {
+    return this.categoryWeights[category] || 1.0;
+  }
+
+  /**
+   * Compute a deduplication hash for a result based on its normalised URL and title prefix.
+   */
+  private calculateResultHash(result: EngineResult): string {
+    const url = result.url || result.link || "";
+
+    try {
+      const urlObj = new URL(url);
+      let normalized = urlObj.hostname.replace(/^www\./, "") + urlObj.pathname;
+      normalized = normalized.replace(/\/$/, "").toLowerCase();
+
+      const titlePart = (result.title || "").toLowerCase().replace(/\s+/g, "");
+      return `${normalized}:${titlePart.substring(0, 50)}`;
+    } catch {
+      const fallback = (result.title || result.content || url).toLowerCase();
+      return fallback.replace(/\s+/g, "").substring(0, 100);
+    }
+  }
+
+  /**
+   * Merge a new engine result into an existing MergedResult, preferring longer
+   * content, non-empty optional fields, and HTTPS URLs.
+   */
+  private mergeTwoResults(origin: MergedResult, other: EngineResult): void {
+    if ((other.content || "").length > (origin.content || "").length) {
+      origin.content = other.content;
+    }
+
+    if ((other.title || "").length > (origin.title || "").length) {
+      origin.title = other.title;
+    }
+
+    if (!origin.img_src && other.img_src) origin.img_src = other.img_src;
+    if (!origin.thumbnail && other.thumbnail) origin.thumbnail = other.thumbnail;
+    if (!origin.publishedDate && other.publishedDate) origin.publishedDate = other.publishedDate;
+    if (!origin.author && other.author) origin.author = other.author;
+
+    const originUrl = origin.url || origin.link || "";
+    const otherUrl = other.url || other.link || "";
+    if (originUrl && !originUrl.startsWith("https://") && otherUrl.startsWith("https://")) {
+      origin.url = other.url;
+      origin.link = other.link;
+    }
+
+    if (other.engine && !origin.engines.includes(other.engine)) {
+      origin.engines.push(other.engine);
+    }
+  }
+
+  /**
+   * Add results from an engine. Ignored after the container is closed.
+   */
+  extend(engineName: string, results: EngineResult[]): void {
+    if (this.closed) {
+      console.warn("ResultContainer is closed, ignoring results from", engineName);
+      return;
+    }
+
+    let mainCount = 0;
+
+    for (const result of results) {
+      if (!this.onResult(result)) continue;
+
+      if ((result as any).suggestion) {
+        this.suggestions.add((result as any).suggestion);
+        continue;
+      }
+
+      if ((result as any).answer) {
+        const answer = (result as any).answer;
+        this.answers.set(answer, { engine: engineName, answer });
+        continue;
+      }
+
+      if ((result as any).correction) {
+        this.corrections.add((result as any).correction);
+        continue;
+      }
+
+      if ((result as any).infobox) {
+        this.mergeInfobox(result as any, engineName);
+        continue;
+      }
+
+      if ((result as any).number_of_results) {
+        this.numberofResults.push((result as any).number_of_results);
+        continue;
+      }
+
+      if ((result as any).engine_data) {
+        const key = (result as any).key || "data";
+        if (!this.engineData[engineName]) this.engineData[engineName] = {};
+        this.engineData[engineName][key] = (result as any).engine_data;
+        continue;
+      }
+
+      mainCount++;
+      this.mergeMainResult(result, mainCount, engineName);
+    }
+  }
+
+  private mergeInfobox(infoboxResult: any, engineName: string): void {
+    const newInfobox: Infobox = {
+      ...infoboxResult,
+      engine: engineName,
+      engines: new Set([engineName]),
+    };
+
+    if (newInfobox.id) {
+      const existing = this.infoboxes.find((ib) => ib.id === newInfobox.id);
+      if (existing) {
+        this.mergeTwoInfoboxes(existing, newInfobox);
+        return;
+      }
+    }
+
+    this.infoboxes.push(newInfobox);
+  }
+
+  private mergeTwoInfoboxes(origin: Infobox, other: Infobox): void {
+    other.engines.forEach((e) => origin.engines.add(e));
+
+    if (other.urls) {
+      if (!origin.urls) origin.urls = [];
+
+      for (const url2 of other.urls) {
+        const exists = origin.urls.some(
+          (url1) =>
+            (url2.entity && url1.entity === url2.entity) || url1.url === url2.url,
+        );
+        if (!exists) origin.urls.push(url2);
+      }
+    }
+
+    if (other.img_src && !origin.img_src) origin.img_src = other.img_src;
+
+    if (other.attributes) {
+      if (!origin.attributes) origin.attributes = [];
+
+      const existingLabels = new Set(
+        origin.attributes.map((a) => a.label || a.entity).filter(Boolean),
+      );
+
+      for (const attr of other.attributes) {
+        const key = attr.label || attr.entity;
+        if (key && !existingLabels.has(key)) origin.attributes.push(attr);
+      }
+    }
+
+    if (other.content && (!origin.content || other.content.length > origin.content.length)) {
+      origin.content = other.content;
+    }
+  }
+
+  private mergeMainResult(result: EngineResult, position: number, engineName: string): void {
+    const resultHash = this.calculateResultHash(result);
+    const existing = this.mainResultsMap.get(resultHash);
+
+    if (!existing) {
+      const mergedResult: MergedResult = {
+        ...result,
+        engine: engineName,
+        engines: [engineName],
+        positions: [position],
+        score: 0,
+        priority: (result as any).priority || "normal",
+        resultHash,
+      };
+      this.mainResultsMap.set(resultHash, mergedResult);
+    } else {
+      this.mergeTwoResults(existing, result);
+      existing.positions.push(position);
+    }
+  }
+
+  /**
+   * Score a result using position-weighted scoring with engine and category multipliers.
+   *
+   * Algorithm:
+   * - weight = product of all engine weights × category weight × number of positions
+   * - low priority: contributes nothing
+   * - high priority: adds full weight per position
+   * - normal priority: adds weight / position (inverse-position decay)
+   */
+  private calculateScore(result: MergedResult): number {
+    let weight = 1.0;
+
+    for (const engineName of result.engines) {
+      weight *= this.getEngineWeight(engineName);
+    }
+
+    if (result.category) {
+      weight *= this.getCategoryWeight(result.category);
+    }
+
+    weight *= result.positions.length;
+
+    let score = 0;
+
+    for (const position of result.positions) {
+      if (result.priority === "low") continue;
+      else if (result.priority === "high") score += weight;
+      else score += weight / position;
+    }
+
+    return score;
+  }
+
+  /**
+   * Freeze the container and compute final scores. Must be called before
+   * `getOrderedResults()`.
+   */
+  close(): void {
+    if (this.closed) return;
+
+    this.closed = true;
+
+    for (const result of this.mainResultsMap.values()) {
+      result.score = this.calculateScore(result);
+    }
+  }
+
+  /**
+   * Return results sorted by score with a category-grouping pass that clusters
+   * similar results within a sliding window of 20 positions (max 8 per group).
+   */
+  getOrderedResults(): MergedResult[] {
+    if (!this.closed) this.close();
+
+    if (this.mainResultsSorted) return this.mainResultsSorted;
+
+    const results = Array.from(this.mainResultsMap.values()).sort(
+      (a, b) => b.score - a.score,
+    );
+
+    const groupedResults: MergedResult[] = [];
+    const categoryPositions: Map<string, { index: number; count: number }> = new Map();
+
+    const maxCount = 8;
+    const maxDistance = 20;
+
+    for (const res of results) {
+      const hasImage = !!(res.thumbnail || res.img_src);
+      const categoryKey = `${res.category || "general"}:${res.template || "default"}:${hasImage ? "img" : "noimg"}`;
+
+      const group = categoryPositions.get(categoryKey);
+
+      if (
+        group &&
+        group.count > 0 &&
+        groupedResults.length - group.index < maxDistance
+      ) {
+        const insertIndex = group.index;
+        groupedResults.splice(insertIndex, 0, res);
+
+        for (const value of categoryPositions.values()) {
+          if (value.index >= insertIndex) value.index += 1;
+        }
+
+        group.count -= 1;
+      } else {
+        groupedResults.push(res);
+        categoryPositions.set(categoryKey, { index: groupedResults.length, count: maxCount });
+      }
+    }
+
+    this.mainResultsSorted = groupedResults;
+    return this.mainResultsSorted;
+  }
+
+  /**
+   * Return all results without ordering (useful for debugging).
+   */
+  getRawResults(): MergedResult[] {
+    return Array.from(this.mainResultsMap.values());
+  }
+
+  /**
+   * Track an engine that failed to respond during this search.
+   */
+  addUnresponsiveEngine(
+    engineName: string,
+    errorType: string,
+    suspended: boolean = false,
+  ): void {
+    if (this.closed) {
+      console.error("Cannot add unresponsive engine after container is closed");
+      return;
+    }
+    this.unresponsiveEngines.add({ engine: engineName, errorType, suspended });
+  }
+
+  /**
+   * Record timing information for an engine request.
+   */
+  addTiming(engineName: string, engineTime: number, pageLoadTime: number): void {
+    if (this.closed) {
+      console.error("Cannot add timing after container is closed");
+      return;
+    }
+    this.timings.push({ engine: engineName, total: engineTime, load: pageLoadTime });
+  }
+
+  /**
+   * Return the average number of results reported by engines, or 0 if fewer than
+   * the actual deduplicated count (indicating the average is not meaningful).
+   */
+  getNumberOfResults(): number {
+    if (!this.closed) {
+      console.error("Call to getNumberOfResults before close()");
+      return 0;
+    }
+
+    if (this.numberofResults.length === 0) return 0;
+
+    const sum = this.numberofResults.reduce((a, b) => a + b, 0);
+    const average = Math.floor(sum / this.numberofResults.length);
+    const actualResults = this.getOrderedResults().length;
+    return average < actualResults ? 0 : average;
+  }
+
+  /**
+   * Return all collected timing entries (only valid after `close()`).
+   */
+  getTimings(): Timing[] {
+    if (!this.closed) {
+      console.error("Call to getTimings before close()");
+      return [];
+    }
+    return this.timings;
+  }
+
+  /**
+   * Return a summary of result counts, scores, deduplication, and metadata.
+   */
+  getStats() {
+    const results = Array.from(this.mainResultsMap.values());
+
+    return {
+      totalResults: results.length,
+      averageScore: results.reduce((sum, r) => sum + r.score, 0) / results.length || 0,
+      engineCoverage:
+        results.reduce((sum, r) => sum + r.engines.length, 0) / results.length || 0,
+      duplicatesMerged: results.filter((r) => r.engines.length > 1).length,
+      categories: [...new Set(results.map((r) => r.category).filter(Boolean))],
+      suggestions: this.suggestions.size,
+      answers: this.answers.size,
+      corrections: this.corrections.size,
+      infoboxes: this.infoboxes.length,
+      unresponsiveEngines: this.unresponsiveEngines.size,
+      numberOfResults: this.getNumberOfResults(),
+      paging: this.paging,
+    };
+  }
+}

package/src/search-web-types.ts

@@ -0,0 +1,7 @@
+/**
+ * Search Web Types — Re-exports all shared TypeScript interfaces and types from
+ * the canonical types module. Covers EngineResult, MergedResult, Infobox,
+ * CategoryConfig, EngineMetadata, EngineStatus, EngineLog, and all supporting
+ * weight and tracking interfaces used across the search API.
+ */
+export * from "./types/search-result-types.js";

package/src/search.ts

@@ -0,0 +1,10 @@
+/**
+ * Search — Entry point re-exporting the Search orchestrator class and the
+ * complete ALL_ENGINES registry. The implementation lives in two focused modules:
+ * - `search/search-query-executor` — the Search class that runs queries in
+ *   parallel, deduplicates via ResultContainer, and returns scored results.
+ * - `search/search-engines-registry-list` — the static catalogue of all 75+
+ *   engine adapters organised by category.
+ */
+export { Search } from "./search/search-query-executor.js";
+export { ALL_ENGINES } from "./search/search-engines-registry-list.js";

package/src/sources/academic/arxiv.ts

@@ -0,0 +1,72 @@
+import { parseHTML } from "linkedom";
+import { EngineFunction } from "../../types/search-engine-interface.js";
+
+export const arxiv: EngineFunction = async (
+  query: string,
+  page: number | undefined
+) => {
+  const params = new URLSearchParams({
+    search_query: "all:" + query,
+    start: String(((page || 1) - 1) * 10),
+    max_results: "10",
+  });
+
+  const response = await fetch(
+    `https://export.arxiv.org/api/query?${params}`,
+    {
+      headers: {
+        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+        Accept: "application/atom+xml",
+      },
+    }
+  );
+
+  if (!response.ok) return [];
+
+  const text = await response.text();
+  const { document } = parseHTML(text);
+
+  return Array.from(document.querySelectorAll("entry"))
+    .map((entry) => {
+      const title =
+        entry
+          .querySelector("title")
+          ?.textContent?.replace(/\s+/g, " ")
+          .trim() || "";
+      const url = entry.querySelector("id")?.textContent?.trim() || "";
+      const abstract = (
+        entry
+          .querySelector("summary")
+          ?.textContent?.replace(/\s+/g, " ")
+          .trim() || ""
+      ).substring(0, 500);
+      const authors = Array.from(entry.querySelectorAll("author name"))
+        .map((n) => n.textContent?.trim())
+        .filter(Boolean);
+      const published = entry
+        .querySelector("published")
+        ?.textContent?.split("T")[0];
+      const categories = Array.from(entry.querySelectorAll("category"))
+        .map((cat) => cat.getAttribute("term"))
+        .filter(Boolean);
+      const meta = [
+        authors.length &&
+          `Authors: ${authors.slice(0, 3).join(", ")}${
+            authors.length > 3 ? " et al." : ""
+          }`,
+        published && `Published: ${published}`,
+        categories.length &&
+          `Categories: ${categories.slice(0, 3).join(", ")}`,
+      ]
+        .filter(Boolean)
+        .join(" | ");
+
+      return {
+        url,
+        title,
+        content: meta ? `${meta}\n\n${abstract}` : abstract,
+        engine: "arxiv",
+      };
+    })
+    .filter((r) => r.url && r.title);
+};