skyloom 1.14.6 → 1.14.8

package/src/tools/builtin.ts

@@ -88,6 +88,145 @@ export function fenceCheck(resolvedPath: string): string | null {
   return `Error: 路径越界 — 工作区围栏已启用 (SKYLOOM_WORKSPACE_FENCE=1)，'${resolvedPath}' 在根目录 '${root}' 之外。`;
 }
 
+/* ── Web search helpers ───────────────────────────────────────────────────
+   Multi-engine fallback. DuckDuckGo's Instant Answer JSON API only returns
+   "abstracts" and is blank for ~90% of real queries; HTML scraping is what
+   actually works. In CN networks, DDG/Bing may be unreachable — Baidu/Sogou
+   serve as fallbacks. Each parser is intentionally tolerant: HTML changes
+   over time, so we extract loosely and let the engine list provide redundancy.
+   ────────────────────────────────────────────────────────────────────────── */
+interface SearchResult { title: string; url: string; snippet: string }
+
+const SEARCH_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36';
+
+async function fetchHtml(url: string, timeoutMs = 12000): Promise<string> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetch(url, {
+      headers: {
+        'User-Agent': SEARCH_UA,
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+      },
+      signal: controller.signal,
+    });
+    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+    return await res.text();
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function decodeHtmlEntities(s: string): string {
+  return s
+    .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, ' ')
+    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n, 10)))
+    .replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCharCode(parseInt(n, 16)));
+}
+
+function stripTags(s: string): string {
+  return decodeHtmlEntities(s.replace(/<[^>]+>/g, '')).replace(/\s+/g, ' ').trim();
+}
+
+function unwrapDdgRedirect(href: string): string {
+  // DuckDuckGo HTML wraps results in /l/?uddg=<encoded-url>
+  const m = href.match(/[?&]uddg=([^&]+)/);
+  if (m) { try { return decodeURIComponent(m[1]); } catch { /* fall through */ } }
+  if (href.startsWith('//')) return 'https:' + href;
+  return href;
+}
+
+function unwrapBaiduRedirect(href: string): string {
+  // Baidu uses opaque /link?url=... redirects; we can't resolve without another request.
+  // Return as-is; consumer can still click through.
+  return href;
+}
+
+async function searchDuckDuckGo(query: string, max: number): Promise<SearchResult[]> {
+  const html = await fetchHtml(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`);
+  const results: SearchResult[] = [];
+  const re = /<a[^>]+class="[^"]*result__a[^"]*"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a[^>]+class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/a>/gi;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(html)) && results.length < max) {
+    results.push({ url: unwrapDdgRedirect(m[1]), title: stripTags(m[2]), snippet: stripTags(m[3]) });
+  }
+  return results;
+}
+
+async function searchBing(query: string, max: number): Promise<SearchResult[]> {
+  const html = await fetchHtml(`https://www.bing.com/search?q=${encodeURIComponent(query)}&setlang=zh-cn`);
+  const results: SearchResult[] = [];
+  const liRe = /<li class="b_algo"[\s\S]*?<\/li>/gi;
+  const items = html.match(liRe) || [];
+  for (const item of items) {
+    if (results.length >= max) break;
+    const a = item.match(/<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
+    if (!a) continue;
+    const snipMatch =
+      item.match(/<p class="b_lineclamp[^"]*"[^>]*>([\s\S]*?)<\/p>/i) ||
+      item.match(/<div class="b_caption"[\s\S]*?<p[^>]*>([\s\S]*?)<\/p>/i) ||
+      item.match(/<p[^>]*>([\s\S]*?)<\/p>/i);
+    results.push({ url: a[1], title: stripTags(a[2]), snippet: snipMatch ? stripTags(snipMatch[1]) : '' });
+  }
+  return results;
+}
+
+async function searchBaidu(query: string, max: number): Promise<SearchResult[]> {
+  const html = await fetchHtml(`https://www.baidu.com/s?wd=${encodeURIComponent(query)}`);
+  const results: SearchResult[] = [];
+  // Baidu nests divs aggressively; anchor on <h3> ... <a href>...</a> and look
+  // for the nearest abstract block following.
+  const re = /<h3[^>]*>[\s\S]{0,500}?<a[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(html)) && results.length < max) {
+    const url = unwrapBaiduRedirect(m[1]);
+    const title = stripTags(m[2]);
+    if (!title || !/^https?:\/\//.test(url)) continue;
+    const after = html.slice(re.lastIndex, re.lastIndex + 4000);
+    const snipMatch =
+      after.match(/<span class="content-right[^"]*"[^>]*>([\s\S]*?)<\/span>/i) ||
+      after.match(/<div class="c-abstract[^"]*"[^>]*>([\s\S]*?)<\/div>/i) ||
+      after.match(/<span[^>]*content[^"]*"[^>]*>([\s\S]{20,400}?)<\/span>/i) ||
+      after.match(/<p[^>]*>([\s\S]{20,400}?)<\/p>/i);
+    results.push({ url, title, snippet: snipMatch ? stripTags(snipMatch[1]) : '' });
+  }
+  return results;
+}
+
+async function searchSogou(query: string, max: number): Promise<SearchResult[]> {
+  const html = await fetchHtml(`https://www.sogou.com/web?query=${encodeURIComponent(query)}`);
+  const results: SearchResult[] = [];
+  const divRe = /<div[^>]+class="vrwrap"[\s\S]*?(?=<div[^>]+class="vrwrap"|$)/gi;
+  const items = html.match(divRe) || [];
+  for (const item of items) {
+    if (results.length >= max) break;
+    const a = item.match(/<h3[^>]*>[\s\S]*?<a[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
+    if (!a) continue;
+    let url = a[1];
+    if (url.startsWith('/link?')) url = 'https://www.sogou.com' + url;
+    const snipMatch =
+      item.match(/<div[^>]+class="(?:str_info|fz-mid|space-txt)[^"]*"[^>]*>([\s\S]*?)<\/div>/i) ||
+      item.match(/<p[^>]*>([\s\S]{20,400}?)<\/p>/i);
+    results.push({ url, title: stripTags(a[2]), snippet: snipMatch ? stripTags(snipMatch[1]) : '' });
+  }
+  return results;
+}
+
+async function runSearchEngine(engine: string, query: string, max: number): Promise<SearchResult[]> {
+  let results: SearchResult[];
+  switch (engine) {
+    case 'duckduckgo': case 'ddg': results = await searchDuckDuckGo(query, max); break;
+    case 'bing': results = await searchBing(query, max); break;
+    case 'baidu': results = await searchBaidu(query, max); break;
+    case 'sogou': results = await searchSogou(query, max); break;
+    default: throw new Error(`unknown search engine: ${engine}`);
+  }
+  // Drop placeholder/JS-anchor entries from inline answer cards.
+  return results.filter((r) => r.title && /^https?:\/\//i.test(r.url));
+}
+
 /**
  * Register all built-in tools into the given registry.
  */
@@ -295,33 +434,43 @@ export function registerBuiltinTools(registry: ToolRegistry): void {
 
   registry.register({
     name: 'web_search',
-    description: 'Search the web for information. Returns search results with titles and snippets.',
+    description: 'Search the web for information. Returns search results with titles, URLs and snippets.',
     parameters: [
       { name: 'query', type: 'string', description: 'Search query', required: true },
+      { name: 'engine', type: 'string', description: 'Optional engine: duckduckgo|bing|baidu|sogou. Default: auto (tries each until one returns results)', required: false },
+      { name: 'max_results', type: 'number', description: 'Max results to return (default 8, capped at 20)', required: false },
     ],
     handler: async (params) => {
-      // Simplified web search using a basic approach
-      try {
-        const query = encodeURIComponent(params.query as string);
-        const url = `https://api.duckduckgo.com/?q=${query}&format=json`;
-        const response = await fetch(url);
-        const data = await response.json() as Record<string, any>;
-        const results: string[] = [];
-        if (data.AbstractText) results.push(`Abstract: ${data.AbstractText}`);
-        if (data.RelatedTopics) {
-          for (const topic of data.RelatedTopics.slice(0, 10)) {
-            if (topic.Text) results.push(`- ${topic.Text}`);
-            else if (topic.Topics) {
-              for (const sub of topic.Topics.slice(0, 5)) {
-                if (sub.Text) results.push(`- ${sub.Text}`);
-              }
-            }
+      const query = String(params.query || '').trim();
+      if (!query) return 'Error: query is required';
+      const max = Math.max(1, Math.min(20, Math.floor(Number(params.max_results) || 8)));
+      const explicit = String(params.engine || '').trim().toLowerCase();
+      const envEngine = String(process.env.SKYLOOM_SEARCH_ENGINE || '').trim().toLowerCase();
+      const order = explicit
+        ? [explicit]
+        : envEngine
+        ? [envEngine, 'duckduckgo', 'bing', 'baidu', 'sogou']
+        : ['duckduckgo', 'bing', 'baidu', 'sogou'];
+      const seen = new Set<string>();
+      const tried: string[] = [];
+      for (const eng of order) {
+        if (seen.has(eng)) continue;
+        seen.add(eng);
+        tried.push(eng);
+        try {
+          const results = await runSearchEngine(eng, query, max);
+          if (results && results.length > 0) {
+            const head = `Search results (${eng}, ${results.length}):`;
+            const body = results
+              .map((r, i) => `${i + 1}. ${r.title}\n   ${r.url}${r.snippet ? `\n   ${r.snippet}` : ''}`)
+              .join('\n');
+            return `${head}\n${body}`;
           }
+        } catch (e: any) {
+          log.warn('web_search_engine_failed', { engine: eng, error: String(e?.message || e) });
         }
-        return results.length > 0 ? results.join('\n') : 'No search results found.';
-      } catch (e) {
-        return `Search error: ${e}`;
       }
+      return `No search results found (tried: ${tried.join(', ')}). Set SKYLOOM_SEARCH_ENGINE to pin an engine, or try a different query.`;
     },
   });