termsearch 0.3.0

package/src/ai/summary.js

@@ -0,0 +1,120 @@
+// AI summary generation — 2-phase agentic flow
+// Phase 1: decide which URLs to fetch
+// Phase 2: synthesize summary from fetched content
+
+const LANG_NAMES = {
+  'it-IT': 'Italian', 'en-US': 'English', 'es-ES': 'Spanish',
+  'fr-FR': 'French', 'de-DE': 'German', 'pt-PT': 'Portuguese',
+  'ru-RU': 'Russian', 'zh-CN': 'Chinese', 'ja-JP': 'Japanese',
+};
+
+function getLanguageOutputRule(lang) {
+  if (lang === 'it-IT') return 'FINAL OUTPUT LANGUAGE RULE: write ONLY in Italian.';
+  if (lang === 'es-ES') return 'FINAL OUTPUT LANGUAGE RULE: write ONLY in Spanish.';
+  if (lang === 'fr-FR') return 'FINAL OUTPUT LANGUAGE RULE: write ONLY in French.';
+  if (lang === 'de-DE') return 'FINAL OUTPUT LANGUAGE RULE: write ONLY in German.';
+  return 'FINAL OUTPUT LANGUAGE RULE: write in the language the user query is in.';
+}
+
+// Build Phase 1 prompt: AI decides which URLs to fetch
+export function buildFetchDecisionPrompt({ query, results, maxFetch = 10, session = [] }) {
+  const list = results.slice(0, 10).map((r, i) =>
+    `[${i + 1}] ${r.title} — ${r.snippet || '(no snippet)'}\n    URL: ${r.url}${r.publishedDate ? `\n    Published: ${r.publishedDate}` : ''}${r.engine ? `\n    Engine: ${r.engine}` : ''}`
+  ).join('\n');
+
+  const sessionBlock = session.length
+    ? `\n=== SESSION CONTEXT ===\n${session.map((s, i) => `${i + 1}. "${s.q}" → ${s.r}`).join('\n')}\n`
+    : '';
+
+  return `You are a search agent. Decide which URLs to read to answer the query.
+Reply ONLY with valid JSON, no text outside the JSON:
+{"fetch":["url1","url2"],"reason":"brief reason"}
+
+RULE: You MUST fetch at least 1 URL unless the snippet already contains a complete, definitive answer.
+Fetch 1-${maxFetch} URLs. Prefer Wikipedia, official sites, reputable sources. Avoid logins, PDFs, redirects.
+${sessionBlock}
+Query: ${query}
+
+Results:
+${list}`;
+}
+
+// Parse Phase 1 response: extract list of URLs to fetch
+export function parseFetchDecision(rawContent, allResultUrls = []) {
+  if (!rawContent) return { urls: allResultUrls.slice(0, 5), reason: '' };
+  try {
+    const json = rawContent.match(/\{[\s\S]*\}/)?.[0];
+    if (!json) throw new Error('no JSON');
+    const parsed = JSON.parse(json);
+    const urls = (parsed.fetch || [])
+      .filter((u) => typeof u === 'string' && /^https?:\/\//.test(u))
+      .slice(0, 20);
+    return { urls, reason: String(parsed.reason || '').slice(0, 200) };
+  } catch {
+    // Fallback: fetch top 3 results
+    return { urls: allResultUrls.slice(0, 3), reason: 'fallback' };
+  }
+}
+
+// Build Phase 2 prompt: synthesize summary from results + fetched documents
+export function buildAgenticSummaryPrompt({ query, lang = 'en-US', results, documents, session = [] }) {
+  const langName = LANG_NAMES[lang] || 'English';
+
+  const fetchedSection = documents.length
+    ? documents.map((doc, i) =>
+        `[F${i + 1}] ${doc.title}\nURL: ${doc.url}\nContent:\n${doc.content.slice(0, 3000)}`
+      ).join('\n\n---\n\n')
+    : null;
+
+  const snippets = results.slice(0, 8).map((r, i) =>
+    `[${i + 1}] ${r.title}\nURL: ${r.url}\nSnippet: ${r.snippet || 'n/a'}`
+  ).join('\n\n');
+
+  const sessionItems = Array.isArray(session) && session.length
+    ? `=== SEARCH SESSION ===\n${session.slice(-4).map((s, i) => `${i + 1}. "${s.q}" → ${s.r}`).join('\n')}\n\n`
+    : '';
+
+  return `You are a search assistant. Answer the query based EXCLUSIVELY on the provided sources.
+NEVER use internal knowledge or training data.
+IMPORTANT: Web page contents have already been extracted below. Never say "I cannot access websites".
+LANGUAGE: Respond in ${langName}.
+${getLanguageOutputRule(lang)}
+
+RESPONSE RULES:
+- 1-2 sentences: direct answer to the query
+- 3-5 bullet points with specific facts, numbers, names from sources
+- Cite inline: [F1][F2] for pages read, [1][2] for snippets
+- If sources conflict on key facts, note it
+- Do not speculate or invent
+- If the answer lists specific sites/tools/services, add at the end:
+  SITES_AI: https://url1, https://url2, https://url3
+
+${sessionItems}SEARCH QUERY: ${query}
+
+${fetchedSection ? `=== PAGES READ ===\n${fetchedSection}\n\n=== SEARCH RESULTS ===\n${snippets}` : `=== SEARCH RESULTS ===\n${snippets}`}
+
+Answer now based only on these sources.`;
+}
+
+// Extract AI-curated site URLs from summary text
+export function extractAiSites(summaryText) {
+  const match = summaryText.match(/SITES_AI:\s*([^\n]+)/);
+  if (!match) return [];
+  return match[1]
+    .split(',')
+    .map((u) => u.trim())
+    .filter((u) => /^https?:\/\//.test(u))
+    .slice(0, 10);
+}
+
+// Score results for reordering based on AI citations
+export function scoreResultsFromSummary(results, summaryText, fetchedUrls = []) {
+  const fetchedSet = new Set(fetchedUrls.map((u) => String(u).toLowerCase()));
+  return results.map((r) => {
+    const urlLower = String(r.url || '').toLowerCase();
+    const isFetched = fetchedSet.has(urlLower);
+    const isCited = summaryText.includes(r.url);
+    const boost = isFetched ? 2 : isCited ? 1 : 0;
+    return { ...r, aiBoost: boost };
+  });
+}

package/src/api/middleware.js

@@ -0,0 +1,91 @@
+// Express middleware: security headers, rate limiting, IP utilities
+
+export function applySecurityHeaders(res) {
+  res.setHeader('X-Content-Type-Options', 'nosniff');
+  res.setHeader('Referrer-Policy', 'strict-origin-when-cross-origin');
+  res.setHeader('Permissions-Policy', 'microphone=(), geolocation=(), camera=()');
+  res.setHeader('Cross-Origin-Resource-Policy', 'same-origin');
+}
+
+export function sendJson(res, status, payload) {
+  applySecurityHeaders(res);
+  res.status(status).json(payload);
+}
+
+export function normalizeIp(value) {
+  const raw = String(value || '').trim();
+  if (!raw) return 'unknown';
+  return raw.replace(/^::ffff:/, '') || 'unknown';
+}
+
+export function isLoopbackIp(value) {
+  const ip = normalizeIp(value);
+  return ip === '127.0.0.1' || ip === '::1' || ip === 'localhost';
+}
+
+export function getClientIp(req) {
+  const remoteIp = normalizeIp(req.socket?.remoteAddress || req.ip || '');
+  if (isLoopbackIp(remoteIp)) {
+    const realIp = req.headers['x-real-ip'];
+    if (typeof realIp === 'string' && realIp.trim()) return normalizeIp(realIp);
+    const forwarded = req.headers['x-forwarded-for'];
+    if (typeof forwarded === 'string' && forwarded.trim()) {
+      return normalizeIp(forwarded.split(',')[0].trim());
+    }
+  }
+  return remoteIp;
+}
+
+// Rate limiting: sliding window per IP
+export function checkWindowRateLimit(store, ip, windowMs, limit) {
+  const key = normalizeIp(ip);
+  const now = Date.now();
+  const bucket = store.get(key) || [];
+  const recent = bucket.filter((ts) => now - ts < windowMs);
+  if (recent.length >= limit) { store.set(key, recent); return false; }
+  recent.push(now);
+  store.set(key, recent);
+  return true;
+}
+
+export function sendRateLimited(res, { windowMs, message = 'Too many requests' }) {
+  res.setHeader('Retry-After', String(Math.max(1, Math.ceil(windowMs / 1000))));
+  return sendJson(res, 429, { error: 'rate_limited', message });
+}
+
+// Middleware factory that creates rate limit stores and checkers
+export function createRateLimiters(cfg) {
+  const generalStore = new Map();
+  const searchStore = new Map();
+  const aiStore = new Map();
+
+  // Prune expired entries every 5 minutes
+  const pruneInterval = setInterval(() => {
+    const now = Date.now();
+    for (const store of [generalStore, searchStore, aiStore]) {
+      for (const [key, bucket] of store) {
+        const fresh = bucket.filter((ts) => now - ts < 3_600_000);
+        if (fresh.length === 0) store.delete(key);
+        else store.set(key, fresh);
+      }
+    }
+  }, 5 * 60 * 1000);
+  pruneInterval.unref?.();
+
+  const rl = cfg.rate_limit;
+  const aiCfg = cfg.ai;
+
+  return {
+    checkGeneral: (ip) => checkWindowRateLimit(generalStore, ip, rl.window_ms, rl.general_per_min),
+    checkSearch: (ip) => checkWindowRateLimit(searchStore, ip, rl.window_ms, rl.search_per_min),
+    checkAi: (ip) => checkWindowRateLimit(aiStore, ip, aiCfg.rate_window_ms, aiCfg.rate_limit),
+    windowMs: rl.window_ms,
+    aiWindowMs: aiCfg.rate_window_ms,
+  };
+}
+
+// Express middleware: attach client IP to req
+export function ipMiddleware(req, _res, next) {
+  req.clientIp = getClientIp(req);
+  next();
+}

package/src/api/routes.js

@@ -0,0 +1,461 @@
+// All API route handlers
+
+import express from 'express';
+import { search, searchStream, getEnabledProviders, getDocCache } from '../search/engine.js';
+import { batchFetch, fetchReadableDocument } from '../fetch/document.js';
+import { generateSummary, testConnection } from '../ai/orchestrator.js';
+import { refineQuery } from '../ai/query.js';
+import { sendJson, sendRateLimited, applySecurityHeaders } from './middleware.js';
+import { getStatus as autostartStatus, setEnabled as autostartSetEnabled } from '../autostart/manager.js';
+import { detectProfileTarget, scanProfile, PROFILER_PLATFORMS } from '../profiler/scanner.js';
+import { fetchBlueskyPosts, fetchBlueskyActors, fetchGdeltArticles } from '../social/search.js';
+import { scrapeTPB, scrape1337x, extractMagnetFromUrl } from '../torrent/scrapers.js';
+
+const APP_VERSION = '0.3.0';
+const ALLOWED_CATEGORIES = new Set(['web', 'images', 'news']);
+
+function parseCategory(raw) {
+  const category = String(raw || 'web').trim().toLowerCase();
+  return ALLOWED_CATEGORIES.has(category) ? category : 'web';
+}
+
+function parseEngines(raw) {
+  if (!raw) return [];
+  const source = Array.isArray(raw) ? raw.join(',') : String(raw);
+  return [...new Set(
+    source
+      .split(',')
+      .map((entry) => entry.trim().toLowerCase())
+      .filter(Boolean)
+      .slice(0, 12)
+  )];
+}
+
+export function createRouter(config, rateLimiters) {
+  const router = express.Router();
+
+  // ─── Health ──────────────────────────────────────────────────────────────
+  router.get('/api/health', (req, res) => {
+    const cfg = config.getConfig();
+    sendJson(res, 200, {
+      status: 'ok',
+      version: APP_VERSION,
+      providers: getEnabledProviders(cfg),
+      ai_enabled: Boolean(cfg.ai?.enabled && cfg.ai?.api_base && cfg.ai?.model),
+      ai_model: cfg.ai?.model || null,
+    });
+  });
+
+  // ─── OpenAPI ─────────────────────────────────────────────────────────────
+  router.get('/api/openapi.json', (_req, res) => {
+    applySecurityHeaders(res);
+    res.json({
+      openapi: '3.1.0',
+      info: {
+        title: 'TermSearch API',
+        version: APP_VERSION,
+      },
+      paths: {
+        '/api/health': { get: { summary: 'Service health' } },
+        '/api/search': { get: { summary: 'Search results (JSON)' } },
+        '/api/search-stream': { get: { summary: 'Progressive search (SSE)' } },
+        '/api/fetch': { post: { summary: 'Fetch readable documents' } },
+        '/api/ai-summary': { post: { summary: 'AI summary (SSE/JSON)' } },
+        '/api/ai-query': { post: { summary: 'AI query refinement' } },
+        '/api/social-search': { get: { summary: 'Bluesky + GDELT search' } },
+        '/api/profiler': { get: { summary: 'Social profile scanner' } },
+        '/api/torrent-search': { post: { summary: 'Torrent direct scraping' } },
+        '/api/magnet': { post: { summary: 'Extract magnet from page URL' } },
+        '/api/scan': { post: { summary: 'Scan site pages by query' } },
+        '/api/config': { get: { summary: 'Read config (masked)' }, post: { summary: 'Update config' } },
+      },
+    });
+  });
+
+  // ─── Search (single response) ─────────────────────────────────────────────
+  router.get('/api/search', async (req, res) => {
+    const ip = req.clientIp;
+    if (!rateLimiters.checkSearch(ip)) {
+      return sendRateLimited(res, { windowMs: rateLimiters.windowMs });
+    }
+
+    const cfg = config.getConfig();
+    const q = String(req.query.q || '').trim();
+    if (!q) return sendJson(res, 400, { error: 'missing_query', message: 'q parameter required' });
+    if (q.length > cfg.search.max_query_length) return sendJson(res, 400, { error: 'query_too_long' });
+
+    const lang = String(req.query.lang || 'en-US');
+    const safe = String(req.query.safe || '1');
+    const page = Number(req.query.page || '1');
+    const category = parseCategory(req.query.cat);
+    const engines = parseEngines(req.query.engines);
+
+    try {
+      const result = await search({ query: q, lang, safe, page, category, engines }, cfg);
+      applySecurityHeaders(res);
+      res.json(result);
+    } catch (error) {
+      sendJson(res, 500, { error: 'search_failed', message: error.message });
+    }
+  });
+
+  // ─── Search stream (SSE) ──────────────────────────────────────────────────
+  router.get('/api/search-stream', async (req, res) => {
+    const ip = req.clientIp;
+    if (!rateLimiters.checkSearch(ip)) {
+      return sendRateLimited(res, { windowMs: rateLimiters.windowMs });
+    }
+
+    const cfg = config.getConfig();
+    const q = String(req.query.q || '').trim();
+    if (!q) return sendJson(res, 400, { error: 'missing_query' });
+    if (q.length > cfg.search.max_query_length) return sendJson(res, 400, { error: 'query_too_long' });
+
+    const lang = String(req.query.lang || 'en-US');
+    const safe = String(req.query.safe || '1');
+    const page = Number(req.query.page || '1');
+    const category = parseCategory(req.query.cat);
+    const engines = parseEngines(req.query.engines);
+
+    applySecurityHeaders(res);
+    res.setHeader('Content-Type', 'text/event-stream');
+    res.setHeader('Cache-Control', 'no-cache');
+    res.setHeader('Connection', 'keep-alive');
+    res.flushHeaders?.();
+
+    const send = (data) => res.write(`data: ${JSON.stringify(data)}\n\n`);
+
+    try {
+      for await (const chunk of searchStream({ query: q, lang, safe, page, category, engines }, cfg)) {
+        if (chunk.tier === 'fast') {
+          send({
+            batch: 'fast',
+            query: q,
+            lang,
+            results: chunk.results || [],
+            providers: chunk.providers || [],
+          });
+        } else {
+          send({
+            batch: 'full',
+            query: q,
+            lang,
+            results: chunk.results || [],
+            allResults: chunk.results || [],
+            providers: chunk.providers || [],
+            degraded: false,
+            engineStats: { responded: chunk.providers || [], failed: [], unstable: [], health: {} },
+          });
+        }
+      }
+      send({ done: true, providers: getEnabledProviders(cfg) });
+    } catch (error) {
+      send({ error: 'search_failed', message: error.message });
+    } finally {
+      res.end();
+    }
+  });
+
+  // ─── Fetch document(s) ────────────────────────────────────────────────────
+  router.post('/api/fetch', express.json({ limit: '32kb' }), async (req, res) => {
+    const ip = req.clientIp;
+    if (!rateLimiters.checkGeneral(ip)) {
+      return sendRateLimited(res, { windowMs: rateLimiters.windowMs });
+    }
+
+    const cfg = config.getConfig();
+    const urls = req.body?.urls;
+    if (!Array.isArray(urls) || urls.length === 0) {
+      return sendJson(res, 400, { error: 'missing_urls' });
+    }
+    if (urls.length > 10) return sendJson(res, 400, { error: 'too_many_urls', max: 10 });
+
+    const results = await batchFetch(urls.slice(0, 10), {
+      timeoutMs: cfg.search.timeout_ms,
+      docCache: getDocCache(),
+    });
+    applySecurityHeaders(res);
+    res.json({ results });
+  });
+
+  // ─── AI query refinement ──────────────────────────────────────────────────
+  router.post('/api/ai-query', express.json({ limit: '16kb' }), async (req, res) => {
+    const ip = req.clientIp;
+    if (!rateLimiters.checkAi(ip)) {
+      return sendRateLimited(res, { windowMs: rateLimiters.aiWindowMs });
+    }
+
+    const cfg = config.getConfig();
+    if (!cfg.ai?.enabled) return sendJson(res, 200, { refined_query: req.body?.query, intent: 'other', also_search: [] });
+
+    const query = String(req.body?.query || '').trim();
+    const lang = String(req.body?.lang || 'en-US');
+    if (!query) return sendJson(res, 400, { error: 'missing_query' });
+
+    const result = await refineQuery({ query, lang }, cfg.ai);
+    applySecurityHeaders(res);
+    res.json(result || { refined_query: query, intent: 'other', also_search: [] });
+  });
+
+  // ─── AI summary (SSE streaming) ────────────────────────────────────────────
+  router.post('/api/ai-summary', express.json({ limit: '256kb' }), async (req, res) => {
+    const ip = req.clientIp;
+    if (!rateLimiters.checkAi(ip)) {
+      return sendRateLimited(res, { windowMs: rateLimiters.aiWindowMs });
+    }
+
+    const cfg = config.getConfig();
+    if (!cfg.ai?.enabled || !cfg.ai?.api_base || !cfg.ai?.model) {
+      return sendJson(res, 200, {
+        error: 'ai_not_configured',
+        message: 'AI not configured. Go to Settings to add your endpoint.',
+      });
+    }
+
+    const query = String(req.body?.query || '').trim();
+    const lang = String(req.body?.lang || 'en-US');
+    const results = Array.isArray(req.body?.results) ? req.body.results : [];
+    const session = Array.isArray(req.body?.session) ? req.body.session.slice(-4) : [];
+    const streamMode = req.body?.stream !== false;
+
+    if (!query) return sendJson(res, 400, { error: 'missing_query' });
+
+    if (streamMode) {
+      applySecurityHeaders(res);
+      res.setHeader('Content-Type', 'text/event-stream');
+      res.setHeader('Cache-Control', 'no-cache');
+      res.setHeader('Connection', 'keep-alive');
+      res.flushHeaders?.();
+
+      const sendEvent = (event, data) => res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
+
+      const result = await generateSummary(
+        {
+          query, lang, results, session,
+          onToken: (chunk) => sendEvent('token', { chunk }),
+          docCache: getDocCache(),
+        },
+        cfg.ai
+      );
+
+      if (result.error) {
+        sendEvent('error', { error: result.error, message: result.message });
+      } else {
+        sendEvent('done', {
+          sites: result.sites,
+          fetchedCount: result.fetchedCount,
+          scoredResults: result.scoredResults,
+          model: result.model,
+        });
+      }
+      return res.end();
+    }
+
+    // Non-streaming mode
+    const result = await generateSummary({ query, lang, results, session, docCache: getDocCache() }, cfg.ai);
+    if (result.error) {
+      return sendJson(res, result.status || 502, { error: result.error, message: result.message });
+    }
+    applySecurityHeaders(res);
+    res.json(result);
+  });
+
+  // ─── Config (public — keys masked) ────────────────────────────────────────
+  router.get('/api/config', (req, res) => {
+    applySecurityHeaders(res);
+    res.json(config.getPublicConfig());
+  });
+
+  // ─── Config update ─────────────────────────────────────────────────────────
+  router.post('/api/config', express.json({ limit: '16kb' }), (req, res) => {
+    const body = req.body;
+    if (!body || typeof body !== 'object') {
+      return sendJson(res, 400, { error: 'invalid_body' });
+    }
+    // Whitelist accepted config keys to prevent unexpected writes
+    const allowed = ['port', 'host', 'ai', 'brave', 'mojeek', 'searxng', 'search', 'rate_limit'];
+    const filtered = {};
+    for (const key of allowed) {
+      if (key in body) filtered[key] = body[key];
+    }
+    try {
+      config.update(filtered);
+      applySecurityHeaders(res);
+      res.json({ ok: true, config: config.getPublicConfig() });
+    } catch (error) {
+      sendJson(res, 500, { error: 'config_save_failed', message: error.message });
+    }
+  });
+
+  // ─── Test AI connection ────────────────────────────────────────────────────
+  router.post('/api/config/test-ai', express.json({ limit: '8kb' }), async (req, res) => {
+    const cfg = config.getConfig();
+    const body = req.body || {};
+    const testCfg = {
+      api_base: String(body.api_base || cfg.ai?.api_base || ''),
+      api_key: String(body.api_key || cfg.ai?.api_key || ''),
+      model: String(body.model || cfg.ai?.model || ''),
+    };
+    const result = await testConnection(testCfg);
+    applySecurityHeaders(res);
+    res.json(result);
+  });
+
+  router.get('/api/config/test-ai', (_req, res) => {
+    sendJson(res, 405, { error: 'method_not_allowed', message: 'Use POST /api/config/test-ai' });
+  });
+
+  // ─── Test search provider ─────────────────────────────────────────────────
+  router.get('/api/config/test-provider/:name', async (req, res) => {
+    const cfg = config.getConfig();
+    const name = String(req.params.name || '');
+    const testQuery = 'test';
+
+    try {
+      let results = [];
+      if (name === 'duckduckgo') {
+        const { search: ddgSearch } = await import('../search/providers/duckduckgo.js');
+        results = await ddgSearch({ query: testQuery, timeoutMs: 8000 });
+      } else if (name === 'wikipedia') {
+        const { search: wikiSearch } = await import('../search/providers/wikipedia.js');
+        results = await wikiSearch({ query: testQuery, timeoutMs: 8000 });
+      } else if (name === 'brave') {
+        const { search: braveSearch } = await import('../search/providers/brave.js');
+        results = await braveSearch({ query: testQuery, config: cfg, timeoutMs: 8000 });
+      } else if (name === 'mojeek') {
+        const { search: mojeekSearch } = await import('../search/providers/mojeek.js');
+        results = await mojeekSearch({ query: testQuery, config: cfg, timeoutMs: 8000 });
+      } else if (name === 'searxng') {
+        const { search: searxSearch } = await import('../search/providers/searxng.js');
+        results = await searxSearch({ query: testQuery, config: cfg, timeoutMs: 8000 });
+      } else {
+        return sendJson(res, 400, { error: 'unknown_provider' });
+      }
+      applySecurityHeaders(res);
+      res.json({ ok: results.length > 0, count: results.length, sample: results.slice(0, 2) });
+    } catch (error) {
+      sendJson(res, 200, { ok: false, error: error.message });
+    }
+  });
+
+  // ─── Stats ────────────────────────────────────────────────────────────────
+  router.get('/api/stats', (req, res) => {
+    applySecurityHeaders(res);
+    // TODO: implement persistent stats counter
+    res.json({ searches: 0, uptime_ms: process.uptime() * 1000 });
+  });
+
+  // ─── Profiler ─────────────────────────────────────────────────────────────────
+  router.get('/api/profiler', async (req, res) => {
+    const ip  = req.clientIp;
+    if (!rateLimiters.checkGeneral(ip)) return sendRateLimited(res, { windowMs: rateLimiters.windowMs });
+    const raw = String(req.query.q || '').trim();
+    if (!raw) return sendJson(res, 400, { error: 'missing_query', message: 'q required (URL or @handle)' });
+    const target = detectProfileTarget(raw);
+    if (!target) return sendJson(res, 400, { error: 'not_a_profile', message: 'Could not detect a social profile in query' });
+    try {
+      applySecurityHeaders(res);
+      const result = await scanProfile(target);
+      res.json(result);
+    } catch (error) {
+      sendJson(res, 500, { error: 'profiler_failed', message: error.message });
+    }
+  });
+
+  // ─── Social search ─────────────────────────────────────────────────────────
+  router.get('/api/social-search', async (req, res) => {
+    const ip  = req.clientIp;
+    if (!rateLimiters.checkGeneral(ip)) return sendRateLimited(res, { windowMs: rateLimiters.windowMs });
+    const q       = String(req.query.q || '').trim();
+    if (!q) return sendJson(res, 400, { error: 'missing_query' });
+    const limit   = Math.min(parseInt(req.query.limit) || 25, 50);
+    const sources = String(req.query.sources || 'bluesky,gdelt').split(',').map((s) => s.trim()).filter(Boolean);
+    const taskMap = {};
+    if (sources.includes('bluesky')) {
+      taskMap.bluesky_posts  = fetchBlueskyPosts(q, limit);
+      taskMap.bluesky_actors = fetchBlueskyActors(q, Math.min(limit, 20));
+    }
+    if (sources.includes('gdelt')) taskMap.gdelt = fetchGdeltArticles(q, limit);
+    const keys    = Object.keys(taskMap);
+    const settled = await Promise.allSettled(Object.values(taskMap));
+    const results = {};
+    keys.forEach((key, i) => { results[key] = settled[i].status === 'fulfilled' ? settled[i].value : []; });
+    const total   = Object.values(results).reduce((s, arr) => s + arr.length, 0);
+    applySecurityHeaders(res);
+    res.json({ query: q, total, results });
+  });
+
+  // ─── Torrent search ─────────────────────────────────────────────────────────
+  router.post('/api/torrent-search', express.json(), async (req, res) => {
+    const ip = req.clientIp;
+    if (!rateLimiters.checkGeneral(ip)) return sendRateLimited(res, { windowMs: rateLimiters.windowMs });
+    const query = String(req.body?.q || req.body?.query || '').trim().slice(0, 200);
+    if (!query) return sendJson(res, 400, { error: 'missing_query', message: 'q required' });
+    try {
+      const [tpb, lxx] = await Promise.allSettled([scrapeTPB(query, 8), scrape1337x(query, 7)]);
+      const results = [
+        ...(tpb.status === 'fulfilled' ? tpb.value : []),
+        ...(lxx.status === 'fulfilled' ? lxx.value : []),
+      ];
+      applySecurityHeaders(res);
+      res.json({ results, source: results.length ? 'tpb+1337x' : 'none' });
+    } catch (error) {
+      sendJson(res, 502, { error: 'scrape_failed', message: error.message });
+    }
+  });
+
+  router.post('/api/magnet', express.json(), async (req, res) => {
+    const ip = req.clientIp;
+    if (!rateLimiters.checkGeneral(ip)) return sendRateLimited(res, { windowMs: rateLimiters.windowMs });
+    const rawUrl = String(req.body?.url || '').trim();
+    if (!rawUrl || !/^https?:\/\//.test(rawUrl)) return sendJson(res, 400, { error: 'invalid_url' });
+    try {
+      const magnet = await extractMagnetFromUrl(rawUrl);
+      applySecurityHeaders(res);
+      res.json({ magnet });
+    } catch (error) {
+      sendJson(res, error.message.includes('SSRF') ? 400 : 502, { error: 'fetch_failed', message: error.message });
+    }
+  });
+
+  // ─── Site scan ──────────────────────────────────────────────────────────────
+  router.post('/api/scan', express.json(), async (req, res) => {
+    const ip = req.clientIp;
+    if (!rateLimiters.checkGeneral(ip)) return sendRateLimited(res, { windowMs: rateLimiters.windowMs });
+    const rawUrl   = String(req.body?.url || '').trim();
+    const query    = String(req.body?.query || '').trim().slice(0, 200);
+    const maxPages = Math.min(Number(req.body?.max_pages) || 4, 8);
+    if (!rawUrl || !query) return sendJson(res, 400, { error: 'invalid_input', message: 'url and query required' });
+    try {
+      const { scanSitePages } = await import('../fetch/document.js');
+      const pages = await scanSitePages(rawUrl, query, maxPages);
+      applySecurityHeaders(res);
+      res.json({ pages: pages.map((p) => ({ url: p.url, title: p.title, content: p.content.slice(0, 3000) })) });
+    } catch (error) {
+      sendJson(res, 502, { error: 'scan_failed', message: error.message });
+    }
+  });
+
+  // ─── Autostart ────────────────────────────────────────────────────────────
+  router.get('/api/autostart', (req, res) => {
+    applySecurityHeaders(res);
+    try {
+      res.json(autostartStatus());
+    } catch (error) {
+      sendJson(res, 500, { error: 'autostart_check_failed', message: error.message });
+    }
+  });
+
+  router.post('/api/autostart', express.json(), (req, res) => {
+    applySecurityHeaders(res);
+    const enable = Boolean(req.body?.enabled);
+    try {
+      const status = autostartSetEnabled(enable);
+      res.json({ ok: true, ...status });
+    } catch (error) {
+      sendJson(res, 500, { error: 'autostart_failed', message: error.message });
+    }
+  });
+
+  return router;
+}