termsearch 0.3.0

package/src/search/engine.js

@@ -0,0 +1,231 @@
+// Search orchestrator — fan-out to all enabled providers, merge, rank, cache
+
+import path from 'path';
+import { makeTieredCache, searchCacheKey } from './cache.js';
+import { mergeSearchResultSets, rankResultsBySourceDiversity } from './ranking.js';
+import * as ddg from './providers/duckduckgo.js';
+import * as wikipedia from './providers/wikipedia.js';
+import * as brave from './providers/brave.js';
+import * as mojeek from './providers/mojeek.js';
+import * as searxng from './providers/searxng.js';
+
+let _searchCache = null;
+let _docCache = null;
+let _dataDir = null;
+
+export function initCaches(dataDir, cfg) {
+  _dataDir = dataDir;
+  const sc = cfg.search;
+  _searchCache = makeTieredCache(
+    sc.cache_l1_max_search,
+    path.join(dataDir, 'cache', 'search'),
+    sc.disk_max_search_entries,
+    sc.disk_max_search_bytes,
+  );
+  _docCache = makeTieredCache(
+    sc.cache_l1_max_docs,
+    path.join(dataDir, 'cache', 'docs'),
+    sc.disk_max_doc_entries,
+    sc.disk_max_doc_bytes,
+  );
+}
+
+export function getDocCache() {
+  return _docCache;
+}
+
+const PROVIDER_REGISTRY = {
+  duckduckgo: {
+    aliases: new Set(['duckduckgo', 'ddg']),
+    enabled: (_cfg) => true,
+    run: ddg.search,
+  },
+  wikipedia: {
+    aliases: new Set(['wikipedia', 'wiki']),
+    enabled: (_cfg) => true,
+    run: wikipedia.search,
+  },
+  brave: {
+    aliases: new Set(['brave']),
+    enabled: (cfg) => Boolean(cfg.brave?.enabled && cfg.brave?.api_key),
+    run: brave.search,
+  },
+  mojeek: {
+    aliases: new Set(['mojeek']),
+    enabled: (cfg) => Boolean(cfg.mojeek?.enabled && cfg.mojeek?.api_key),
+    run: mojeek.search,
+  },
+  searxng: {
+    aliases: new Set(['searxng', 'searx']),
+    enabled: (cfg) => Boolean(cfg.searxng?.enabled && cfg.searxng?.url),
+    run: searxng.search,
+  },
+};
+
+function normalizeRequestedEngines(input) {
+  if (!Array.isArray(input)) return [];
+  return [...new Set(
+    input
+      .map((item) => String(item || '').trim().toLowerCase())
+      .filter(Boolean)
+  )];
+}
+
+function resolveProviderPlan(cfg, requestedEngines = []) {
+  const requested = normalizeRequestedEngines(requestedEngines);
+  const enabledProviders = Object.keys(PROVIDER_REGISTRY).filter((name) => PROVIDER_REGISTRY[name].enabled(cfg));
+  if (requested.length === 0) {
+    return { providers: enabledProviders, searxEngines: [] };
+  }
+
+  const explicitProviders = new Set();
+  const searxEngines = [];
+
+  for (const engine of requested) {
+    const mapped = enabledProviders.find((provider) => PROVIDER_REGISTRY[provider].aliases.has(engine));
+    if (mapped) {
+      explicitProviders.add(mapped);
+    } else {
+      searxEngines.push(engine);
+    }
+  }
+
+  if (searxEngines.length > 0 && enabledProviders.includes('searxng')) {
+    explicitProviders.add('searxng');
+  }
+
+  const providers = [...explicitProviders].filter((name) => enabledProviders.includes(name));
+  if (providers.length === 0) {
+    return { providers: enabledProviders, searxEngines: [] };
+  }
+  return { providers, searxEngines };
+}
+
+async function runProvider(name, args) {
+  const provider = PROVIDER_REGISTRY[name];
+  if (!provider) return [];
+  try {
+    return await provider.run(args);
+  } catch {
+    return [];
+  }
+}
+
+// Run a search across all enabled providers and return merged, ranked results
+export async function search({ query, lang = 'en-US', safe = '1', page = 1, category = 'web', engines = [] }, cfg) {
+  if (!_searchCache) throw new Error('Caches not initialized — call initCaches() first');
+
+  const plan = resolveProviderPlan(cfg, engines);
+  const providerList = plan.providers;
+  const timeoutMs = cfg.search.timeout_ms;
+  const cacheEngines = providerList.length ? providerList : ['none'];
+  const cacheKey = searchCacheKey(query, lang, safe, cacheEngines, 'full', category, page);
+  const cached = _searchCache.get(cacheKey);
+  if (cached) return cached;
+
+  const tasks = providerList.map((providerName) =>
+    runProvider(providerName, {
+      query,
+      lang,
+      safe,
+      page,
+      category,
+      config: cfg,
+      timeoutMs,
+      engines: providerName === 'searxng' ? plan.searxEngines : [],
+    })
+  );
+
+  const allResults = await Promise.all(tasks);
+
+  // Merge all provider results
+  let merged = [];
+  for (const provResults of allResults) {
+    merged = mergeSearchResultSets(merged, provResults);
+  }
+
+  // Rank by source diversity
+  const ranked = rankResultsBySourceDiversity(merged);
+
+  const response = {
+    results: ranked,
+    query,
+    lang,
+    page: Number(page),
+    total: ranked.length,
+    providers: providerList,
+    category,
+  };
+
+  _searchCache.set(cacheKey, response, cfg.search.cache_ttl_search_ms);
+  return response;
+}
+
+// Streaming search: returns fast results first (DDG), then merges full results
+export async function* searchStream({ query, lang = 'en-US', safe = '1', page = 1, category = 'web', engines = [] }, cfg) {
+  const plan = resolveProviderPlan(cfg, engines);
+  const providerList = plan.providers;
+  if (providerList.length === 0) {
+    yield { tier: 'full', results: [], providers: [] };
+    return;
+  }
+
+  const timeoutMs = cfg.search.timeout_ms;
+  const fastProvider = providerList.includes('duckduckgo')
+    ? 'duckduckgo'
+    : providerList[0];
+  const fastResults = await runProvider(fastProvider, {
+    query,
+    lang,
+    safe,
+    page,
+    category,
+    config: cfg,
+    timeoutMs,
+    engines: fastProvider === 'searxng' ? plan.searxEngines : [],
+  });
+  if (fastResults.length > 0) {
+    yield { tier: 'fast', results: rankResultsBySourceDiversity(fastResults), providers: [fastProvider] };
+  }
+
+  const remainingProviders = providerList.filter((name) => name !== fastProvider);
+  const tasks = remainingProviders.map((providerName) =>
+    runProvider(providerName, {
+      query,
+      lang,
+      safe,
+      page,
+      category,
+      config: cfg,
+      timeoutMs,
+      engines: providerName === 'searxng' ? plan.searxEngines : [],
+    })
+  );
+
+  const additional = await Promise.all(tasks);
+  let full = fastResults.slice();
+  for (const r of additional) {
+    full = mergeSearchResultSets(full, r);
+  }
+  const fullRanked = rankResultsBySourceDiversity(full);
+
+  // Cache the full result
+  const cacheEngines = providerList.length ? providerList : ['none'];
+  const cacheKey = searchCacheKey(query, lang, safe, cacheEngines, 'full', category, page);
+  _searchCache?.set(cacheKey, {
+    results: fullRanked,
+    query, lang, page: Number(page), total: fullRanked.length,
+    providers: providerList,
+    category,
+  }, cfg.search.cache_ttl_search_ms);
+
+  yield { tier: 'full', results: fullRanked, providers: providerList };
+}
+
+export function getEnabledProviders(cfg) {
+  const providers = ['duckduckgo', 'wikipedia'];
+  if (cfg.brave?.enabled && cfg.brave?.api_key) providers.push('brave');
+  if (cfg.mojeek?.enabled && cfg.mojeek?.api_key) providers.push('mojeek');
+  if (cfg.searxng?.enabled && cfg.searxng?.url) providers.push('searxng');
+  return providers;
+}

package/src/search/providers/brave.js

@@ -0,0 +1,57 @@
+// Brave Search API provider — requires API key (configured via web UI)
+
+function localeToIso639(locale) {
+  const map = {
+    'it-IT': 'it', 'en-US': 'en', 'es-ES': 'es',
+    'fr-FR': 'fr', 'de-DE': 'de', 'pt-PT': 'pt',
+    'ru-RU': 'ru', 'zh-CN': 'zh', 'ja-JP': 'ja',
+  };
+  return map[locale] || '';
+}
+
+export async function search({ query, lang = 'en-US', safe = '1', page = 1, config, timeoutMs = 12000 }) {
+  const apiKey = config?.brave?.api_key;
+  const apiBase = (config?.brave?.api_base || 'https://api.search.brave.com/res/v1').replace(/\/$/, '');
+  if (!apiKey) return [];
+
+  const isoLang = localeToIso639(lang);
+  const resultCount = config?.search?.result_count || 10;
+  const params = new URLSearchParams({
+    q: query,
+    count: String(resultCount),
+    offset: String((Number(page) - 1) * resultCount),
+  });
+  if (isoLang) params.set('search_lang', isoLang);
+  if (safe === '2') params.set('safesearch', 'strict');
+  else if (safe === '1') params.set('safesearch', 'moderate');
+  else params.set('safesearch', 'off');
+
+  const ac = new AbortController();
+  const timer = setTimeout(() => ac.abort(), timeoutMs);
+  try {
+    const r = await fetch(`${apiBase}/web/search?${params.toString()}`, {
+      headers: {
+        'Accept': 'application/json',
+        'Accept-Encoding': 'gzip',
+        'X-Subscription-Token': apiKey,
+        'User-Agent': 'TermSearch/1.0',
+      },
+      signal: ac.signal,
+    });
+    clearTimeout(timer);
+    if (!r.ok) return [];
+    const data = await r.json();
+    return (data?.web?.results || []).map((item) => ({
+      title: item.title || '',
+      url: item.url || '',
+      snippet: item.description || '',
+      engine: 'brave-api',
+      score: 0,
+      publishedDate: item.age || null,
+      thumbnail_src: item.thumbnail?.src || null,
+    }));
+  } catch {
+    clearTimeout(timer);
+    return [];
+  }
+}

package/src/search/providers/duckduckgo.js

@@ -0,0 +1,148 @@
+// DuckDuckGo HTML scraper — zero API key required
+// Uses html.duckduckgo.com/html/ (maintained for non-JS clients / accessibility)
+
+const DDG_ENDPOINT = 'https://html.duckduckgo.com/html/';
+const DDG_FALLBACK = 'https://lite.duckduckgo.com/lite/';
+
+const USER_AGENTS = [
+  'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0',
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+];
+
+function randomUA() {
+  return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
+}
+
+// DDG safe search parameter
+function safeParam(safe) {
+  if (safe === '2') return '-1'; // strict
+  if (safe === '0') return '-2'; // off
+  return '-1';                   // moderate → strict (default)
+}
+
+// DDG language/region parameter
+function langParam(lang) {
+  const map = {
+    'it-IT': 'it-it', 'en-US': 'us-en', 'es-ES': 'es-es',
+    'fr-FR': 'fr-fr', 'de-DE': 'de-de', 'pt-PT': 'pt-pt',
+    'ru-RU': 'ru-ru', 'zh-CN': 'cn-zh', 'ja-JP': 'jp-ja',
+  };
+  return map[lang] || 'wt-wt';
+}
+
+// Parse DDG HTML result — returns { title, url, snippet } or null
+function parseResult(html, startIdx) {
+  // Extract result URL from <a class="result__a" href="...">
+  const aMatch = html.slice(startIdx).match(/<a[^>]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
+  if (!aMatch) return null;
+
+  let url = aMatch[1];
+  // DDG wraps URLs in redirect — extract real URL
+  if (url.startsWith('//duckduckgo.com/l/?')) {
+    try {
+      const uddg = new URL('https:' + url).searchParams.get('uddg') || '';
+      if (uddg) url = decodeURIComponent(uddg);
+    } catch { /* keep raw */ }
+  }
+  if (!url.startsWith('http')) return null;
+
+  const title = aMatch[2].replace(/<[^>]+>/g, '').trim();
+  if (!title || !url) return null;
+
+  // Snippet: next <a class="result__snippet"> after the title link
+  const snippetChunk = html.slice(startIdx, startIdx + 3000);
+  const snippetMatch = snippetChunk.match(/class="result__snippet"[^>]*>([\s\S]*?)<\/a>/i);
+  const snippet = snippetMatch
+    ? snippetMatch[1].replace(/<[^>]+>/g, '').replace(/&amp;/g, '&').replace(/&nbsp;/g, ' ').trim()
+    : '';
+
+  return { title, url, snippet };
+}
+
+// Parse DDG lite HTML — simpler table-based format
+function parseLiteHtml(html) {
+  const results = [];
+  const linkRe = /<a[^>]*class="result-link"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
+  const snippetRe = /<td[^>]*class="result-snippet"[^>]*>([\s\S]*?)<\/td>/gi;
+  const links = [...html.matchAll(linkRe)];
+  const snippets = [...html.matchAll(snippetRe)];
+  for (let i = 0; i < links.length && results.length < 15; i++) {
+    let url = links[i][1];
+    if (url.includes('duckduckgo.com/l/?')) {
+      try { url = decodeURIComponent(new URL('https://x.com' + url.replace(/^https?:\/\/[^/]+/, '')).searchParams.get('uddg') || url); } catch { /* ok */ }
+    }
+    if (!url.startsWith('http')) continue;
+    const title = links[i][2].replace(/<[^>]+>/g, '').trim();
+    const snippet = snippets[i]
+      ? snippets[i][1].replace(/<[^>]+>/g, '').replace(/&amp;/g, '&').replace(/&nbsp;/g, ' ').trim()
+      : '';
+    if (title && url) results.push({ title, url, snippet, engine: 'duckduckgo', score: 0 });
+  }
+  return results;
+}
+
+async function fetchDDG(endpoint, formData, timeoutMs) {
+  const ac = new AbortController();
+  const timer = setTimeout(() => ac.abort(), timeoutMs);
+  try {
+    const r = await fetch(endpoint, {
+      method: 'POST',
+      headers: {
+        'User-Agent': randomUA(),
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'Origin': 'https://duckduckgo.com',
+        'Referer': 'https://duckduckgo.com/',
+      },
+      body: formData,
+      signal: ac.signal,
+    });
+    clearTimeout(timer);
+    if (!r.ok) return null;
+    const html = await r.text();
+    // Detect captcha/block page
+    if (html.includes('challenge-form') || html.includes('Sorry, you have been blocked')) return null;
+    return html;
+  } catch {
+    clearTimeout(timer);
+    return null;
+  }
+}
+
+export async function search({ query, lang = 'en-US', safe = '1', page = 1, timeoutMs = 12000 }) {
+  const kp = safeParam(safe);
+  const kl = langParam(lang);
+  const offset = (Number(page) - 1) * 10;
+
+  const params = new URLSearchParams({
+    q: query,
+    kp,
+    kl,
+    kf: '-1', // site icons off (faster)
+    s: String(offset),
+  });
+  const formData = params.toString();
+
+  let html = await fetchDDG(DDG_ENDPOINT, formData, timeoutMs);
+
+  // Fallback to lite endpoint
+  if (!html) {
+    const liteParams = new URLSearchParams({ q: query, s: String(offset) });
+    html = await fetchDDG(DDG_FALLBACK, liteParams.toString(), timeoutMs);
+    if (!html) return [];
+    return parseLiteHtml(html);
+  }
+
+  // Parse main HTML endpoint
+  const results = [];
+  const resultRe = /class="results_links|class="result results_links/gi;
+  let match;
+  while ((match = resultRe.exec(html)) !== null && results.length < 15) {
+    const r = parseResult(html, match.index);
+    if (r) results.push({ ...r, engine: 'duckduckgo', score: 0 });
+  }
+
+  return results;
+}

package/src/search/providers/mojeek.js

@@ -0,0 +1,56 @@
+// Mojeek Search API provider — requires API key (configured via web UI)
+
+function localeToIso639(locale) {
+  const map = {
+    'it-IT': 'it', 'en-US': 'en', 'es-ES': 'es',
+    'fr-FR': 'fr', 'de-DE': 'de', 'pt-PT': 'pt',
+    'ru-RU': 'ru', 'zh-CN': 'zh', 'ja-JP': 'ja',
+  };
+  return map[locale] || '';
+}
+
+export async function search({ query, lang = 'en-US', page = 1, config, timeoutMs = 12000 }) {
+  const apiKey = config?.mojeek?.api_key;
+  const apiBase = (config?.mojeek?.api_base || 'https://api.mojeek.com').replace(/\/$/, '');
+  if (!apiKey) return [];
+
+  const isoLang = localeToIso639(lang);
+  const resultCount = config?.search?.result_count || 10;
+  const params = new URLSearchParams({
+    api_key: apiKey,
+    q: query,
+    t: String(resultCount),
+    s: String(((Number(page) - 1) * resultCount) + 1),
+    fmt: 'json',
+  });
+  if (isoLang) {
+    params.set('lb', isoLang.toUpperCase());
+    params.set('lbb', '100');
+  }
+
+  const ac = new AbortController();
+  const timer = setTimeout(() => ac.abort(), timeoutMs);
+  try {
+    const r = await fetch(`${apiBase}/search?${params.toString()}`, {
+      headers: {
+        'Accept': 'application/json',
+        'User-Agent': 'TermSearch/1.0',
+      },
+      signal: ac.signal,
+    });
+    clearTimeout(timer);
+    if (!r.ok) return [];
+    const data = await r.json();
+    return (data?.response?.results || []).map((item) => ({
+      title: item.title || '',
+      url: item.url || '',
+      snippet: item.desc || '',
+      engine: 'mojeek-api',
+      score: Number(item.score || 0),
+      publishedDate: item.date || null,
+    }));
+  } catch {
+    clearTimeout(timer);
+    return [];
+  }
+}

package/src/search/providers/searxng.js

@@ -0,0 +1,53 @@
+// SearXNG proxy provider — for power users who run their own SearXNG instance
+
+function mapCategory(category) {
+  const c = String(category || 'web').toLowerCase();
+  if (c === 'images') return 'images';
+  if (c === 'news') return 'news';
+  return 'general';
+}
+
+export async function search({ query, lang = 'en-US', safe = '1', page = 1, category = 'web', engines = [], config, timeoutMs = 15000 }) {
+  const searxngUrl = config?.searxng?.url;
+  if (!searxngUrl) return [];
+
+  const base = searxngUrl.replace(/\/$/, '');
+  const params = new URLSearchParams({
+    q: query,
+    format: 'json',
+    language: lang,
+    safesearch: safe === '2' ? '2' : safe === '0' ? '0' : '1',
+    pageno: String(page),
+    categories: mapCategory(category),
+  });
+  if (Array.isArray(engines) && engines.length > 0) {
+    params.set('engines', engines.join(','));
+  }
+
+  const ac = new AbortController();
+  const timer = setTimeout(() => ac.abort(), timeoutMs);
+  try {
+    const r = await fetch(`${base}/search?${params.toString()}`, {
+      headers: {
+        'Accept': 'application/json',
+        'User-Agent': 'TermSearch/1.0',
+      },
+      signal: ac.signal,
+    });
+    clearTimeout(timer);
+    if (!r.ok) return [];
+    const data = await r.json();
+    return (data?.results || []).map((item) => ({
+      title: item.title || '',
+      url: item.url || '',
+      snippet: item.content || '',
+      engine: item.engine ? `searxng:${item.engine}` : 'searxng',
+      score: Number(item.score || 0),
+      publishedDate: item.publishedDate || null,
+      thumbnail_src: item.thumbnail || null,
+    }));
+  } catch {
+    clearTimeout(timer);
+    return [];
+  }
+}

package/src/search/providers/wikipedia.js

@@ -0,0 +1,70 @@
+// Wikipedia REST API provider — zero API key required
+// Uses the MediaWiki action API for full-text search
+
+const API_BASE = 'https://{lang}.wikipedia.org/w/api.php';
+
+// Map locale to Wikipedia language subdomain
+function langCode(locale) {
+  const map = {
+    'it-IT': 'it', 'en-US': 'en', 'es-ES': 'es',
+    'fr-FR': 'fr', 'de-DE': 'de', 'pt-PT': 'pt',
+    'ru-RU': 'ru', 'zh-CN': 'zh', 'ja-JP': 'ja',
+  };
+  return map[locale] || 'en';
+}
+
+function stripHtmlBasic(html) {
+  return (html || '')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&amp;/g, '&')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+export async function search({ query, lang = 'en-US', page = 1, timeoutMs = 12000 }) {
+  const lc = langCode(lang);
+  const endpoint = API_BASE.replace('{lang}', lc);
+  const offset = (Number(page) - 1) * 5;
+
+  const params = new URLSearchParams({
+    action: 'query',
+    list: 'search',
+    srsearch: query,
+    srlimit: '6',
+    sroffset: String(offset),
+    srprop: 'snippet|titlesnippet|sectiontitle',
+    format: 'json',
+    origin: '*',
+  });
+
+  const ac = new AbortController();
+  const timer = setTimeout(() => ac.abort(), timeoutMs);
+  try {
+    const r = await fetch(`${endpoint}?${params.toString()}`, {
+      headers: {
+        'Accept': 'application/json',
+        'User-Agent': 'TermSearch/1.0 (personal search engine; https://github.com/DioNanos/termsearch)',
+      },
+      signal: ac.signal,
+    });
+    clearTimeout(timer);
+    if (!r.ok) return [];
+
+    const data = await r.json();
+    const items = data?.query?.search || [];
+    return items.map((item) => ({
+      title: stripHtmlBasic(item.title || ''),
+      url: `https://${lc}.wikipedia.org/wiki/${encodeURIComponent((item.title || '').replace(/ /g, '_'))}`,
+      snippet: stripHtmlBasic(item.snippet || ''),
+      engine: 'wikipedia',
+      score: 1.0,
+      publishedDate: null,
+    }));
+  } catch {
+    clearTimeout(timer);
+    return [];
+  }
+}