termsearch 0.3.0

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "termsearch",
+  "version": "0.3.0",
+  "description": "Personal search engine for Termux/Linux/macOS — zero-config, privacy-first, AI-optional",
+  "type": "module",
+  "bin": {
+    "termsearch": "./bin/termsearch.js"
+  },
+  "main": "./src/server.js",
+  "files": [
+    "bin/",
+    "src/",
+    "scripts/",
+    "frontend/dist/",
+    "README.md",
+    "LICENSE",
+    "config.example.json"
+  ],
+  "scripts": {
+    "start": "node ./bin/termsearch.js",
+    "dev": "node --watch ./src/server.js",
+    "postinstall": "node scripts/postinstall.js"
+  },
+  "dependencies": {
+    "express": "^4.18.2"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "keywords": [
+    "search",
+    "metasearch",
+    "personal",
+    "terminal",
+    "termux",
+    "privacy",
+    "ai",
+    "duckduckgo",
+    "wikipedia",
+    "self-hosted"
+  ],
+  "author": "DAG",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/DioNanos/termsearch"
+  }
+}

package/scripts/postinstall.js

@@ -0,0 +1,84 @@
+#!/usr/bin/env node
+// Post-install check — runs after npm install -g termsearch
+// Never throws: warnings only, install must always succeed.
+
+import { execSync } from 'child_process';
+import os from 'os';
+import fs from 'fs';
+import path from 'path';
+
+const RESET  = '\x1b[0m';
+const GREEN  = '\x1b[32m';
+const YELLOW = '\x1b[33m';
+const CYAN   = '\x1b[36m';
+const BOLD   = '\x1b[1m';
+
+function ok(msg)   { console.log(`  ${GREEN}✓${RESET} ${msg}`); }
+function warn(msg) { console.log(`  ${YELLOW}⚠${RESET} ${msg}`); }
+function info(msg) { console.log(`  ${CYAN}→${RESET} ${msg}`); }
+
+try {
+  console.log('');
+  console.log(`${BOLD}  TermSearch — post-install check${RESET}`);
+  console.log('');
+
+  // ── Node.js version ──────────────────────────────────────────────────────
+  const [major] = process.versions.node.split('.').map(Number);
+  if (major >= 18) {
+    ok(`Node.js ${process.versions.node}`);
+  } else {
+    warn(`Node.js ${process.versions.node} detected — requires ≥ 18.0.0`);
+    warn('Please upgrade: https://nodejs.org');
+  }
+
+  // ── Platform ─────────────────────────────────────────────────────────────
+  const isTermux = process.env.PREFIX?.includes('com.termux') ||
+                   fs.existsSync('/data/data/com.termux');
+  if (isTermux) {
+    ok('Platform: Termux (Android) — compatible');
+  } else if (process.platform === 'darwin') {
+    ok('Platform: macOS — compatible');
+  } else if (process.platform === 'linux') {
+    ok('Platform: Linux — compatible');
+  } else {
+    warn(`Platform: ${process.platform} — not officially tested`);
+  }
+
+  // ── Home dir / data dir writable ─────────────────────────────────────────
+  const dataDir = path.join(os.homedir(), '.termsearch');
+  try {
+    fs.mkdirSync(dataDir, { recursive: true });
+    fs.accessSync(dataDir, fs.constants.W_OK);
+    ok(`Data dir: ${dataDir}`);
+  } catch {
+    warn(`Cannot write to ${dataDir} — check permissions`);
+  }
+
+  // ── Systemd hint (Linux only, non-Termux) ─────────────────────────────────
+  if (process.platform === 'linux' && !isTermux) {
+    try {
+      execSync('systemctl --user status > /dev/null 2>&1', { stdio: 'ignore' });
+      ok('systemd --user: available (autostart supported)');
+    } catch {
+      warn('systemd --user: not available — autostart will use manual method');
+    }
+  }
+
+  // ── Termux:Boot hint ──────────────────────────────────────────────────────
+  if (isTermux) {
+    const bootDir = path.join(os.homedir(), '.termux', 'boot');
+    if (fs.existsSync(bootDir)) {
+      ok('Termux:Boot: found — autostart supported');
+    } else {
+      info('Termux:Boot: install the app for autostart support');
+      info('  → F-Droid: search "Termux:Boot"');
+    }
+  }
+
+  console.log('');
+  info(`Run ${BOLD}termsearch${RESET}${CYAN} then open http://localhost:3000`);
+  console.log('');
+
+} catch {
+  // Never block the install
+}

package/src/ai/orchestrator.js

@@ -0,0 +1,163 @@
+// AI orchestrator — coordinates the 2-phase agentic summary flow
+
+import { call, stream, classifyAiError } from './providers/openai-compat.js';
+import { buildFetchDecisionPrompt, parseFetchDecision, buildAgenticSummaryPrompt, extractAiSites, scoreResultsFromSummary } from './summary.js';
+import { batchFetch } from '../fetch/document.js';
+
+function mapAiErrorCode(error) { return classifyAiError(error); }
+function mapAiErrorMessage(error) {
+  switch (classifyAiError(error)) {
+    case 'ai_provider_auth': return 'AI provider authentication failed. Check your API key in Settings.';
+    case 'ai_rate_limited_provider': return 'AI provider rate limited. Try again shortly.';
+    case 'ai_timeout_provider': return 'AI provider timed out. Try again shortly.';
+    case 'ai_provider_unavailable': case 'ai_provider_unreachable': return 'AI provider temporarily unavailable.';
+    default: return 'AI unavailable. Check endpoint in Settings.';
+  }
+}
+function mapAiErrorStatus(error) {
+  switch (classifyAiError(error)) {
+    case 'ai_rate_limited_provider': return 429;
+    case 'ai_timeout_provider': return 504;
+    case 'ai_provider_auth': return 503;
+    default: return 502;
+  }
+}
+
+// Refine a query using AI (non-streaming, quick call)
+// Returns { content, model } or throws
+export async function aiQueryRefine(prompt, aiConfig) {
+  return call(prompt, {
+    apiBase: aiConfig.api_base,
+    apiKey: aiConfig.api_key,
+    model: aiConfig.model,
+    maxTokens: 200,
+    timeoutMs: 5000,
+    jsonMode: true,
+  });
+}
+
+// Generate AI summary with optional streaming
+// If onToken is provided, streams tokens as they arrive
+// Returns { summary, sites, fetchedCount, fetchedUrls, model, error? }
+export async function generateSummary({
+  query,
+  lang = 'en-US',
+  results = [],
+  session = [],
+  onToken = null,
+  docCache = null,
+}, aiConfig) {
+  if (!aiConfig?.enabled || !aiConfig?.api_base || !aiConfig?.model) {
+    return { error: 'ai_not_configured', message: 'AI not configured. Add endpoint in Settings.' };
+  }
+
+  const ai = {
+    apiBase: aiConfig.api_base,
+    apiKey: aiConfig.api_key,
+    model: aiConfig.model,
+    maxTokens: aiConfig.max_tokens || 1200,
+    timeoutMs: aiConfig.timeout_ms || 90_000,
+  };
+
+  try {
+    // Phase 1: AI decides which URLs to fetch
+    const phase1Prompt = buildFetchDecisionPrompt({
+      query,
+      results,
+      maxFetch: aiConfig.fetch_soft_cap || 10,
+      session,
+    });
+
+    let phase1Result = null;
+    try {
+      phase1Result = await call(phase1Prompt, {
+        ...ai,
+        maxTokens: 300,
+        timeoutMs: 8000,
+        jsonMode: true,
+      });
+    } catch {
+      // Phase 1 failure is non-fatal — fall back to fetching top results
+    }
+
+    const allResultUrls = results.slice(0, 10).map((r) => r.url).filter(Boolean);
+    const { urls: urlsToFetch } = parseFetchDecision(phase1Result?.content, allResultUrls);
+
+    // Fetch the selected URLs
+    let documents = [];
+    if (urlsToFetch.length > 0) {
+      const fetched = await batchFetch(
+        urlsToFetch.slice(0, aiConfig.fetch_hard_cap || 15),
+        { timeoutMs: 12000, docCache }
+      );
+      documents = fetched.filter((d) => d.status === 'ok' && d.content);
+    }
+
+    // Fallback: if no docs fetched, try top 2 results directly
+    if (documents.length === 0 && allResultUrls.length > 0) {
+      const fallback = await batchFetch(allResultUrls.slice(0, 2), { timeoutMs: 10000, docCache });
+      documents = fallback.filter((d) => d.status === 'ok' && d.content);
+    }
+
+    // Phase 2: synthesize summary
+    const phase2Prompt = buildAgenticSummaryPrompt({ query, lang, results, documents, session });
+
+    let summaryText = '';
+    let summaryModel = ai.model;
+
+    if (typeof onToken === 'function') {
+      // Streaming mode
+      const streamResult = await stream(phase2Prompt, onToken, {
+        ...ai,
+        systemPrompt: 'You are a search assistant. Write your answer directly. Do not include reasoning or thinking.',
+      });
+      summaryText = streamResult.content;
+      summaryModel = streamResult.model;
+    } else {
+      // Non-streaming mode
+      const result = await call(phase2Prompt, {
+        ...ai,
+        systemPrompt: 'You are a search assistant. Write your answer directly. Do not include reasoning or thinking.',
+      });
+      summaryText = result.content;
+      summaryModel = result.model;
+    }
+
+    const sites = extractAiSites(summaryText);
+    const scoredResults = scoreResultsFromSummary(results, summaryText, urlsToFetch);
+
+    return {
+      summary: summaryText,
+      sites,
+      fetchedCount: documents.length,
+      fetchedUrls: urlsToFetch,
+      scoredResults,
+      model: summaryModel,
+    };
+  } catch (error) {
+    return {
+      error: mapAiErrorCode(error),
+      message: mapAiErrorMessage(error),
+      status: mapAiErrorStatus(error),
+    };
+  }
+}
+
+// Test the AI connection with a simple completion
+export async function testConnection(aiConfig) {
+  if (!aiConfig?.api_base || !aiConfig?.model) {
+    return { ok: false, error: 'Missing api_base or model' };
+  }
+  try {
+    const result = await call('Say "OK" and nothing else.', {
+      apiBase: aiConfig.api_base,
+      apiKey: aiConfig.api_key,
+      model: aiConfig.model,
+      maxTokens: 10,
+      timeoutMs: 10000,
+    });
+    return { ok: true, model: result.model, response: result.content.slice(0, 50) };
+  } catch (error) {
+    return { ok: false, error: mapAiErrorMessage(error), code: mapAiErrorCode(error) };
+  }
+}

package/src/ai/providers/openai-compat.js

@@ -0,0 +1,255 @@
+// Generic OpenAI-compatible API client
+// Works with: Ollama, llama.cpp, LM Studio, OpenAI, Groq, Chutes.ai, any compatible endpoint
+
+const RATE_LIMIT_RETRY_DELAYS_MS = [3000, 6000];
+const TRANSIENT_RETRY_DELAYS_MS = [1500, 3000];
+
+const THINKING_START_RE = /^(?:Thinking Process:|Let me (?:analyze|think)|I (?:need to|will|should)|The user (?:is|wants|asked)|Looking at (?:the|these)|Analyzing (?:the|this))/i;
+
+function stripThinkingPrefix(text) {
+  if (!THINKING_START_RE.test(text)) return text;
+  const answerStart = text.search(/\n\n(?=##|###|\*\*[A-Z]|[A-Z][a-zÀ-ÿ])/);
+  if (answerStart !== -1) return text.slice(answerStart).trim();
+  const parts = text.split(/\n\n+/);
+  const firstReal = parts.findIndex((p, i) => i > 0 && !/^\d+\.|^Let me|^I |^The user|^Thinking|^Analyzing/i.test(p.trim()));
+  if (firstReal > 0) return parts.slice(firstReal).join('\n\n').trim();
+  return text;
+}
+
+function extractThinking(raw) {
+  const thinkMatch = raw.match(/^\s*<think>([\s\S]*?)<\/think>\s*/);
+  if (thinkMatch) {
+    return { reasoning: thinkMatch[1].trim(), content: raw.slice(thinkMatch[0].length).trim() };
+  }
+  const blockMatch = raw.match(/^(?:Thinking Process:|Let me|I need to|The user|Looking at)[\s\S]*?(\{[\s\S]*\})\s*$/);
+  if (blockMatch) {
+    return { reasoning: raw.slice(0, raw.lastIndexOf(blockMatch[1])).trim(), content: blockMatch[1].trim() };
+  }
+  return { reasoning: '', content: raw };
+}
+
+function buildAiError(code, message) {
+  const error = new Error(message);
+  error.aiCode = code;
+  return error;
+}
+
+export function classifyAiError(error) {
+  const explicit = String(error?.aiCode || '').trim();
+  if (explicit) return explicit;
+  const raw = String(error?.message || error || '').trim().toLowerCase();
+  if (!raw) return 'ai_unavailable';
+  if (raw.includes(' 401') || raw.includes(' 403') || raw.includes('authentication failed') || raw.includes('invalid token')) return 'ai_provider_auth';
+  if (raw.includes(' 429') || raw.includes('rate limit') || raw.includes('too many requests')) return 'ai_rate_limited_provider';
+  if (raw.includes('aborted') || raw.includes('timeout') || raw.includes('timed out')) return 'ai_timeout_provider';
+  if (raw.includes('fetch failed') || raw.includes('networkerror') || raw.includes('network error')) return 'ai_provider_unreachable';
+  if (raw.includes(' 500') || raw.includes(' 502') || raw.includes(' 503') || raw.includes(' 504')) return 'ai_provider_unavailable';
+  return 'ai_unavailable';
+}
+
+function shouldRetry(error) {
+  const code = classifyAiError(error);
+  return code === 'ai_rate_limited_provider' || code === 'ai_timeout_provider' || code === 'ai_provider_unreachable' || code === 'ai_provider_unavailable';
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// Single non-streaming call with retry logic
+// Returns { content, reasoning, model } or throws
+export async function call(prompt, {
+  apiBase,
+  apiKey,
+  model,
+  maxTokens = 1200,
+  timeoutMs = 90_000,
+  systemPrompt = null,
+  jsonMode = false,
+  temperature = 0.3,
+} = {}) {
+  const base = String(apiBase || '').replace(/\/$/, '');
+  if (!base || !model) throw buildAiError('ai_unavailable', 'AI not configured');
+
+  const messages = systemPrompt
+    ? [{ role: 'system', content: systemPrompt }, { role: 'user', content: prompt }]
+    : [{ role: 'user', content: prompt }];
+  const bodyBase = { messages, max_tokens: maxTokens, temperature };
+  if (jsonMode) bodyBase.response_format = { type: 'json_object' };
+
+  const headers = { 'Content-Type': 'application/json' };
+  if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`;
+
+  let rateLimitAttempt = 0;
+  let transientAttempt = 0;
+
+  while (true) {
+    const ac = new AbortController();
+    const timer = setTimeout(() => ac.abort(), timeoutMs);
+    try {
+      const response = await fetch(`${base}/chat/completions`, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify({ ...bodyBase, model }),
+        signal: ac.signal,
+      });
+
+      if (response.status === 401 || response.status === 403) {
+        throw buildAiError('ai_provider_auth', `ai ${response.status}: authentication failed`);
+      }
+      if (response.status === 429) {
+        if (rateLimitAttempt < RATE_LIMIT_RETRY_DELAYS_MS.length) {
+          const retryAfter = Number(response.headers.get('retry-after') || 0);
+          const delay = retryAfter > 0 ? retryAfter * 1000 : RATE_LIMIT_RETRY_DELAYS_MS[rateLimitAttempt++];
+          console.warn(`[AI] rate limited, retrying in ${delay}ms`);
+          await sleep(delay);
+          continue;
+        }
+        throw buildAiError('ai_rate_limited_provider', 'ai 429: rate limit exceeded');
+      }
+      if (!response.ok) {
+        const errBody = await response.text().catch(() => '');
+        const err = new Error(`ai ${response.status}: ${errBody.slice(0, 220)}`);
+        if (shouldRetry(err) && classifyAiError(err) !== 'ai_rate_limited_provider' && transientAttempt < TRANSIENT_RETRY_DELAYS_MS.length) {
+          const delay = TRANSIENT_RETRY_DELAYS_MS[transientAttempt++];
+          await sleep(delay);
+          continue;
+        }
+        throw err;
+      }
+
+      const data = await response.json();
+      const msg = data.choices?.[0]?.message || {};
+      const rawContent = stripThinkingPrefix((msg.content || '').trim());
+      const { reasoning, content } = extractThinking(rawContent);
+      return {
+        content,
+        reasoning: (msg.reasoning_content || reasoning || '').trim(),
+        model: data.model || model,
+      };
+    } catch (error) {
+      clearTimeout(timer);
+      if (shouldRetry(error) && classifyAiError(error) !== 'ai_rate_limited_provider' && transientAttempt < TRANSIENT_RETRY_DELAYS_MS.length) {
+        const delay = TRANSIENT_RETRY_DELAYS_MS[transientAttempt++];
+        console.warn(`[AI] transient error, retrying in ${delay}ms:`, error.message);
+        await sleep(delay);
+        continue;
+      }
+      throw error;
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+}
+
+// Streaming call — emits tokens via onToken(chunk, fullContent) callback
+// Returns { content, reasoning, model } when done
+export async function stream(prompt, onToken, {
+  apiBase,
+  apiKey,
+  model,
+  maxTokens = 1200,
+  timeoutMs = 90_000,
+  systemPrompt = 'You are a search assistant. Write your answer directly. Do not include reasoning or thinking.',
+  temperature = 0.3,
+} = {}) {
+  const base = String(apiBase || '').replace(/\/$/, '');
+  if (!base || !model) throw buildAiError('ai_unavailable', 'AI not configured');
+
+  const messages = systemPrompt
+    ? [{ role: 'system', content: systemPrompt }, { role: 'user', content: prompt }]
+    : [{ role: 'user', content: prompt }];
+  const body = JSON.stringify({ messages, max_tokens: maxTokens, temperature, model, stream: true });
+
+  const headers = { 'Content-Type': 'application/json' };
+  if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`;
+
+  let rateLimitAttempt = 0;
+  let transientAttempt = 0;
+
+  while (true) {
+    const ac = new AbortController();
+    const timer = setTimeout(() => ac.abort(), timeoutMs);
+    let emittedAny = false;
+    try {
+      const response = await fetch(`${base}/chat/completions`, {
+        method: 'POST',
+        headers,
+        body,
+        signal: ac.signal,
+      });
+
+      if (response.status === 401 || response.status === 403) {
+        throw buildAiError('ai_provider_auth', `ai ${response.status}: authentication failed`);
+      }
+      if (response.status === 429) {
+        if (rateLimitAttempt < RATE_LIMIT_RETRY_DELAYS_MS.length) {
+          const retryAfter = Number(response.headers.get('retry-after') || 0);
+          const delay = retryAfter > 0 ? retryAfter * 1000 : RATE_LIMIT_RETRY_DELAYS_MS[rateLimitAttempt++];
+          await sleep(delay);
+          continue;
+        }
+        return { content: '', reasoning: '', model };
+      }
+      if (!response.ok) {
+        const errBody = await response.text().catch(() => '');
+        const err = new Error(`ai ${response.status}: ${errBody.slice(0, 220)}`);
+        if (shouldRetry(err) && !emittedAny && transientAttempt < TRANSIENT_RETRY_DELAYS_MS.length) {
+          await sleep(TRANSIENT_RETRY_DELAYS_MS[transientAttempt++]);
+          continue;
+        }
+        throw err;
+      }
+
+      const reader = response.body.getReader();
+      const decoder = new TextDecoder();
+      let fullContent = '';
+      let fullReasoning = '';
+      let buffer = '';
+      let inThinking = false;
+      let streamModel = '';
+
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split('\n');
+        buffer = lines.pop() || '';
+        for (const line of lines) {
+          if (!line.startsWith('data: ')) continue;
+          const data = line.slice(6).trim();
+          if (data === '[DONE]') continue;
+          try {
+            const parsed = JSON.parse(data);
+            if (!streamModel && parsed.model) streamModel = parsed.model;
+            const delta = parsed.choices?.[0]?.delta || {};
+            if (delta.reasoning_content) fullReasoning += delta.reasoning_content;
+            if (delta.content) {
+              let chunk = delta.content;
+              if (inThinking) {
+                const end = chunk.indexOf('</think>');
+                if (end !== -1) { fullReasoning += chunk.slice(0, end); chunk = chunk.slice(end + 8); inThinking = false; }
+                else { fullReasoning += chunk; continue; }
+              } else if (chunk.includes('<think>')) {
+                const start = chunk.indexOf('<think>');
+                const end = chunk.indexOf('</think>');
+                if (end !== -1) { fullReasoning += chunk.slice(start + 7, end); chunk = chunk.slice(0, start) + chunk.slice(end + 8); }
+                else { inThinking = true; fullReasoning += chunk.slice(start + 7); chunk = chunk.slice(0, start); }
+              }
+              if (chunk) { emittedAny = true; fullContent += chunk; onToken(chunk, fullContent); }
+            }
+          } catch { /* ignore SSE parse errors */ }
+        }
+      }
+      clearTimeout(timer);
+      return { content: stripThinkingPrefix(fullContent.trim()), reasoning: fullReasoning.trim(), model: streamModel || model };
+    } catch (error) {
+      clearTimeout(timer);
+      if (!emittedAny && shouldRetry(error) && classifyAiError(error) !== 'ai_rate_limited_provider' && transientAttempt < TRANSIENT_RETRY_DELAYS_MS.length) {
+        await sleep(TRANSIENT_RETRY_DELAYS_MS[transientAttempt++]);
+        continue;
+      }
+      throw error;
+    }
+  }
+}

package/src/ai/query.js

@@ -0,0 +1,54 @@
+// AI query refinement — Phase 0 (parallel with search, optional)
+
+import { call } from './providers/openai-compat.js';
+
+function buildQueryInterpretPrompt({ query, lang }) {
+  const langName = {
+    'it-IT': 'Italian', 'en-US': 'English', 'es-ES': 'Spanish',
+    'fr-FR': 'French', 'de-DE': 'German', 'pt-PT': 'Portuguese',
+  }[lang] || 'English';
+
+  return `Analyze this search query and respond in JSON only.
+
+Query: "${query}"
+User language: ${langName}
+
+Respond with this exact JSON structure:
+{
+  "refined_query": "improved version of the query (or same if already good)",
+  "intent": "one of: definition, how_to, news, research, comparison, navigation, other",
+  "also_search": ["optional alternative query 1", "optional alternative query 2"]
+}
+
+Rules:
+- refined_query: fix typos, expand acronyms, clarify ambiguous terms — keep it concise
+- intent: classify what the user is looking for
+- also_search: at most 2 useful variant queries, empty array if not applicable
+- JSON only, no explanation`;
+}
+
+// Returns { refined_query, intent, also_search } or null on failure
+export async function refineQuery({ query, lang = 'en-US' }, aiConfig) {
+  if (!aiConfig?.enabled || !aiConfig?.api_base || !aiConfig?.model) return null;
+
+  try {
+    const result = await call(buildQueryInterpretPrompt({ query, lang }), {
+      apiBase: aiConfig.api_base,
+      apiKey: aiConfig.api_key,
+      model: aiConfig.model,
+      maxTokens: 200,
+      timeoutMs: 5000,
+      jsonMode: true,
+    });
+
+    if (!result?.content) return null;
+    const parsed = JSON.parse(result.content);
+    return {
+      refined_query: String(parsed.refined_query || query).slice(0, 240),
+      intent: String(parsed.intent || 'other'),
+      also_search: Array.isArray(parsed.also_search) ? parsed.also_search.slice(0, 2).map(String) : [],
+    };
+  } catch {
+    return null;
+  }
+}