thincoder 0.10.0 → 0.11.1

package/src/tools/web.mjs

@@ -1,11 +1,60 @@
-import {
-  DESC,
-  truncate,
-  readBodyText,
-  stripTags,
-  htmlToText
-} from "./shared.mjs";
-import { join } from "node:path";
+import { DESC, truncate, stripTags, htmlToText, isPrivateHost } from "./shared.mjs";
+import { URL } from "node:url";
+import { resolveWebProxy, proxyFetch } from "../proxy.mjs";
+
+export const UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+const FETCH_TIMEOUT = 15_000
+
+// ── Web search (Bing; direct by default, through proxy when configured and web toggle on) ──
+
+function extractBing(html) {
+  const results = []
+  // RSS format: <item><title>..</title><link>..</link><description>..</description></item>
+  for (const m of html.matchAll(/<item>([\s\S]*?)<\/item>/gi)) {
+    const block = m[1]
+    const title = block.match(/<title>([\s\S]*?)<\/title>/)?.[1] ?? ""
+    const href = block.match(/<link>([\s\S]*?)<\/link>/)?.[1] ?? ""
+    const snippet = block.match(/<description>([\s\S]*?)<\/description>/)?.[1] ?? ""
+    if (!href) continue
+    results.push({ href, title: stripTags(title), snippet: stripTags(snippet) })
+  }
+  // Fallback: HTML b_algo blocks (older server-rendered pages)
+  if (results.length === 0) {
+    const blocks = html.split('<li class="b_algo"').slice(1)
+    for (const block of blocks) {
+      const link = block.match(/<h2[^>]*><a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/)
+      if (!link) continue
+      const snippet = block.match(/<p[^>]*>([\s\S]*?)<\/p>/)
+      results.push({ href: link[1], title: stripTags(link[2]), snippet: snippet ? stripTags(snippet[1]) : "" })
+    }
+  }
+  return results
+}
+
+function bingUrl(query, page) {
+  let u = `https://www.bing.com/search?q=${encodeURIComponent(query)}&format=rss&setlang=en`
+  if (page > 1) u += `&first=${(page - 1) * 10 + 1}`
+  return u
+}
+
+const ENGINES = [{ name: "bing", label: "Bing", url: bingUrl, extract: extractBing, ua: UA }]
+const ENGINE_NAMES = ENGINES.map(e => e.name)
+
+async function fetchEngine(engine, query, page, ctx) {
+  const ctrl = new AbortController()
+  const timer = setTimeout(() => ctrl.abort(), FETCH_TIMEOUT)
+  try {
+    const response = await proxyFetch(engine.url(query, page), {
+      headers: { "User-Agent": engine.ua, "Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8" },
+      signal: ctrl.signal,
+    }, resolveWebProxy(ctx))
+    if (!response.ok) return null
+    const html = await response.text()
+    const results = engine.extract(html)
+    return { engine: engine.name, results }
+  } catch { return null }
+  finally { clearTimeout(timer) }
+}
 
 export const websearchTool = {
   name: "websearch",
@@ -14,108 +63,85 @@ export const websearchTool = {
     type: "object",
     properties: {
       query: { type: "string", description: "Search query" },
-      limit: { type: "number", description: "Max results (default 8)" },
+      limit: { type: "number", description: "Max results (default 8, max 20)" },
+      engine: { type: "string", enum: ENGINE_NAMES, description: "Specific engine — \"bing\" (Bing). Omit to search all engines concurrently." },
+      page: { type: "number", description: "Page number for pagination (1-based, default 1). Only used when engine is specified." },
     },
     required: ["query"],
   },
   readonly: true,
   async execute(args, ctx) {
-    const limit = args.limit ?? 8
-    const url = `https://www.bing.com/search?q=${encodeURIComponent(args.query)}`
-    let html
-    try {
-      const response = await fetch(url, {
-        headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" },
-        signal: ctx?.signal
-          ? AbortSignal.any([ctx.signal, AbortSignal.timeout(15_000)])
-          : AbortSignal.timeout(15_000),
-      })
-      if (!response.ok) throw new Error(`HTTP ${response.status}`)
-      html = await readBodyText(response)
-    } catch (error) {
-      throw new Error(`websearch request failed: ${error.cause?.code ?? error.message}`)
+    const limit = Math.min(args.limit ?? 8, 20)
+    const page = Math.max(1, args.page ?? 1)
+    if (args.engine) {
+      const engine = ENGINES.find(e => e.name === args.engine)
+      if (!engine) return `Unknown engine '${args.engine}'. Available: ${ENGINE_NAMES.join(", ")}`
+      const fetched = await fetchEngine(engine, args.query, page, ctx)
+      if (!fetched || fetched.results.length === 0) return `(no results from ${engine.label})`
+      return truncate(fetched.results.slice(0, limit).map((r, i) => `${i + 1}. ${r.title}\n   ${r.href}\n   ${r.snippet}`).join("\n\n"))
     }
-
-    // Result block <li class="b_algo">: <h2><a href>title</a></h2> + <p>snippet</p>
-    const blocks = html.split('<li class="b_algo"').slice(1)
-    const results = []
-    for (const block of blocks) {
-      const link = block.match(/<h2[^>]*><a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/)
-      if (!link) continue
-      const snippet = block.match(/<p[^>]*>([\s\S]*?)<\/p>/)
-      results.push({
-        href: link[1],
-        title: stripTags(link[2]),
-        snippet: snippet ? stripTags(snippet[1]) : "",
-      })
-      if (results.length >= limit) break
+    const promises = ENGINES.map(e => fetchEngine(e, args.query, 1, ctx))
+    const fetched = (await Promise.all(promises)).filter(Boolean)
+    if (fetched.length === 0) return "(no results — all search engines failed)"
+    const merged = [], indexes = fetched.map(() => 0)
+    let done = false
+    while (!done && merged.length < limit) {
+      done = true
+      for (let i = 0; i < fetched.length; i++) {
+        if (indexes[i] < fetched[i].results.length) {
+          merged.push({ ...fetched[i].results[indexes[i]], _engine: fetched[i].engine })
+          indexes[i]++; done = false
+          if (merged.length >= limit) break
+        }
+      }
     }
-    if (results.length === 0) return "(no results)"
-    return truncate(
-      results.map((r, i) => `${i + 1}. ${r.title}\n   ${r.href}\n   ${r.snippet}`).join("\n\n"),
-    )
+    return truncate(merged.slice(0, limit).map((r, i) => `${i + 1}. [${r._engine}] ${r.title}\n   ${r.href}\n   ${r.snippet}`).join("\n\n"))
   },
 }
 
+// ── Fetch tool (with proxy support) ──────
 
-// ---------------------------------------------------------------- ls
-
-/** SSRF protection: block internal private-network/metadata endpoints (localhost allowed — user's dev server, tests depend on it) */
 function isPrivateUrl(urlStr) {
-  let u
-  try { u = new URL(urlStr) } catch { return true }
-  const host = u.hostname.toLowerCase()
-  // localhost / 127.x allowed (user's dev server on local machine, test mock server)
-  if (host === "localhost" || host === "0.0.0.0" || host.endsWith(".localhost")) return false
-  if (host === "127.0.0.1" || host.startsWith("127.")) return false
-  // cloud metadata endpoint
-  if (host === "169.254.169.254" || host === "metadata.google.internal") return true
-  // IPv4 private ranges
-  const m = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/)
-  if (m) {
-    const [a, b] = [Number(m[1]), Number(m[2])]
-    if (a === 10) return true
-    if (a === 172 && b >= 16 && b <= 31) return true
-    if (a === 192 && b === 168) return true
-    if (a === 169 && b === 254) return true
-    if (a === 0) return true
-  }
-  // IPv6 loopback / link-local / unique local addresses
-  if (host === "::1" || host === "fe80::1" || host.startsWith("fc") || host.startsWith("fd")) return true
-  return false
+  let u; try { u = new URL(urlStr) } catch { return true }
+  return isPrivateHost(u.hostname)
+}
+
+// proxyFetch returns a native Response (Headers object, needs .get()) without proxy,
+// but a Response-like with a plain lowercase-keyed Record through the CONNECT tunnel — handle both.
+function headerOf(res, name) {
+  const h = res.headers
+  if (!h) return null
+  if (typeof h.get === "function") return h.get(name)
+  return h[name.toLowerCase()] ?? null
 }
 
 export const fetchTool = {
   name: "fetch",
   description: DESC("fetch"),
-  parameters: {
-    type: "object",
-    properties: {
-      url: { type: "string", description: "http/https URL" },
-    },
-    required: ["url"],
-  },
+  parameters: { type: "object", properties: { url: { type: "string", description: "http/https URL" } }, required: ["url"] },
   readonly: true,
   async execute(args, ctx) {
     if (!/^https?:\/\//.test(args.url)) throw new Error("url must start with http:// or https://")
     if (isPrivateUrl(args.url)) throw new Error("fetch blocked: internal/private/metadata addresses are not allowed")
-    let response
     try {
-      response = await fetch(args.url, {
-        headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" },
-        redirect: "follow",
-        signal: ctx?.signal
-          ? AbortSignal.any([ctx.signal, AbortSignal.timeout(20_000)])
-          : AbortSignal.timeout(20_000),
-      })
-    } catch (error) {
-      throw new Error(`fetch failed: ${error.cause?.code ?? error.message}`)
-    }
-    if (!response.ok) throw new Error(`fetch failed: HTTP ${response.status}`)
-
-    const contentType = response.headers.get("content-type") ?? ""
-    const body = await readBodyText(response)
-    if (!contentType.includes("text/html")) return truncate(body)
-    return truncate(htmlToText(body))
+      const proxyUri = resolveWebProxy(ctx)
+      const response = await proxyFetch(args.url, { headers: { "User-Agent": UA } }, proxyUri)
+      if (!response.ok) {
+        if ([301, 302, 307, 308].includes(response.status)) {
+          const loc = headerOf(response, "location")
+          if (loc) {
+            const r2 = await proxyFetch(loc, { headers: { "User-Agent": UA } }, proxyUri)
+            if (!r2.ok) throw new Error(`fetch failed: HTTP ${r2.status}`)
+            const ct2 = headerOf(r2, "content-type") ?? ""
+            const b2 = await r2.text()
+            return ct2.includes("text/html") ? truncate(htmlToText(b2)) : truncate(b2)
+          }
+        }
+        throw new Error(`fetch failed: HTTP ${response.status}`)
+      }
+      const ct = headerOf(response, "content-type") ?? ""
+      const body = await response.text()
+      return ct.includes("text/html") ? truncate(htmlToText(body)) : truncate(body)
+    } catch (e) { throw new Error(`fetch failed: ${e.cause?.code ?? e.message}`) }
   },
 }

package/src/tools/websearch.md

@@ -1,11 +1,13 @@
-Search the web (Bing). Returns result titles, URLs, and snippets. Use for looking up current information, docs, error messages.
+Search the web via Bing. Returns result titles, URLs, and snippets. Use for looking up current information, docs, error messages.
 
 Parameters:
 - query (required): Search query
-- limit: Max results (default 8)
+- limit: Max results (default 8, max 20)
+- engine: Specific engine to use — "bing" (Bing). Omit to search all engines concurrently.
+- page: Page number for pagination (1-based, default 1). Only used when engine is specified.
 
 Notes:
 - Before searching the web, call `memory_search` first — you may already know the answer from a previous session. Only reach for websearch if memory comes up empty.
 - Use this for information that is NOT in the local codebase — current docs, error messages, API references
 - Follow up with `fetch` to read full pages from the results
-- Results are scraped from Bing HTML — some formatting may be imperfect
+- Proxy support: set `"proxy": {"uri": "http://host:port", "web": true}` in config.json

package/src/tui/agent-turn.mjs

@@ -10,6 +10,9 @@ import { ansi, C } from "./ansi.mjs"
  *         handleSlash, summarize } */
 export async function runAgentTurn(ctx, text) {
   const { agent, state, pushLine, pushLabel, render, scheduleRender, ensureAssistantLabel, askPermission, askQuestion, handleSlash, summarize } = ctx
+  // 可注入覆盖（测试用）；默认走真实实现
+  const runAgentImpl = ctx.runAgent ?? runAgent
+  const saveSessionImpl = ctx.saveSession ?? saveSession
   pushLabel(`❯ You:`, ansi.bold + C.user)
   pushLine(text, C.text)
 
@@ -133,12 +136,21 @@ export async function runAgentTurn(ctx, text) {
       const stream = state.toolStreams[name]
       const panel = state.outputPanels[name]
       if (panel) {
-        // Tool with output panel: mark panel as done, conversation gets only summary
-        panel.done = true
         delete state.toolStreams[name]
-        const summary = formatPanelSummary(name, result)
-        if (summary) pushLine(`  ${summary}`, C.dim)
-        // Clear panel after 3 seconds
+        panel._pendingDone = true  // defer done until next render cycle flushes it
+        scheduleRender()  // trigger one final render while panel is still alive
+        if (name === "advisor") {
+          const text = String(result ?? "")
+          const lines = text.split("\n")
+          const maxShow = Math.min(60, lines.length)
+          // Push as single multiline block so formatTables aligns MD table columns
+          const shown = lines.slice(0, maxShow).map((l) => `  ${l.slice(0, 200)}`).join("\n")
+          pushLine(shown, C.advisor)
+          if (lines.length > maxShow) pushLine(`  ... (${lines.length - maxShow} more lines — call advisor again or scroll through the tool result for full output)`, C.dim)
+        } else {
+          const summary = formatPanelSummary(name, result)
+          if (summary) pushLine(`  ${summary}`, C.dim)
+        }
         setTimeout(() => {
           delete state.outputPanels[name]
           if (state.processing) render()
@@ -164,6 +176,7 @@ export async function runAgentTurn(ctx, text) {
       }
       panel.text = (panel.text ?? "") + chunk
       if (panel.text.length > 4000) panel.text = panel.text.slice(-4000)
+      panel.seq = (panel.seq ?? 0) + 1 // render-loop cache key: survives the 4000-char cap
       scheduleRender()
     },
     onPermissionRequest: (name, args) => askPermission(name, args),
@@ -197,99 +210,97 @@ export async function runAgentTurn(ctx, text) {
       pushLine(`  [task] ${done}/${items.length}${current ? ` ▶ ${current.title}` : ""}`, C.dim)
       render()
     },
-    onAdvisor: (note) => {
-      pushLine(`  [advisor] ${note.replace(/\n/g, "\n  ")}`, C.advisor)
-    },
     // Incremental save: flush to disk every 5 tool turns — mid-crash loss window shrinks from an entire round to a few turns
     onTurnEnd: (() => {
       let n = 0
       return () => {
         if (++n % 5 !== 0) return
-        try { saveSession(agent, state.lines) } catch (e) { console.error(`[session] incremental save failed: ${e.message}`) }
+        try { saveSessionImpl(agent, state.lines) } catch (e) { console.error(`[session] incremental save failed: ${e.message}`) }
       }
     })(),
   }
 
-  for (let resume = false; ; resume = true) {
-    try {
-      await runAgent(agent, text, callbacks, { signal: state.controller.signal, resume })
-      flushStream()
-      break // Normal completion, exit loop
-    } catch (error) {
-      flushStream()
-      if (error.name === "AbortError" || state.controller?.signal.aborted) {
-        // Ctrl+I inject: the signal was aborted with an interrupt message — the agent loop
-        // may have already injected it into history, but the aborted signal prevents retry.
-        // Recreate the controller and resume from the same context.
-        if (state.controller?.signal?.reason?.interrupt) {
+  // try/finally: every exit path — including an unexpected throw inside the catch
+  // block (e.g. the continue-permission UI) — must stop the ticker and reset state,
+  // otherwise the 1s render interval leaks and keeps firing forever.
+  try {
+    for (let resume = false; ; resume = true) {
+      try {
+        await runAgentImpl(agent, text, callbacks, { signal: state.controller.signal, resume })
+        flushStream()
+        break // Normal completion, exit loop
+      } catch (error) {
+        flushStream()
+        if (error.name === "AbortError" || state.controller?.signal.aborted) {
+          // Ctrl+I inject: the signal was aborted with an interrupt message — the agent loop
+          // may have already injected it into history, but the aborted signal prevents retry.
+          // Recreate the controller and resume from the same context.
+          if (state.controller?.signal?.reason?.interrupt) {
+            state.controller = new AbortController()
+            resume = true
+            continue
+          }
+          pushLine("[stopped]", C.warn)
+          break
+        }
+        if (error instanceof ContinueError) {
+          pushLabel(`❯ Continue`, ansi.bold + C.warn)
+          pushLine(`Ran ${error.turn} turns (limit ${error.turn}). Continue?`, C.warn)
+          // Pause to ask: reuse permission mechanism
+          const willContinue = await new Promise((resolve) => {
+            state.permission = {
+              name: "continue",
+              args: { turns: error.turn },
+              resolve,
+            }
+            state.status = `Continue after ${error.turn} turns?`
+            render()
+          })
+          state.permission = null
+          if (!willContinue) {
+            pushLine("[continue cancelled]", C.warn)
+            break
+          }
+          pushLine("[continuing…]", C.tool)
+          // Recreate AbortController: once aborted, resume immediately fails (defensive; current path unreachable but tightly coupled)
           state.controller = new AbortController()
-          resume = true
           continue
         }
-        pushLine("[stopped]", C.warn)
+        pushLine(`[error] ${error.message}`, C.error)
         break
       }
-      if (error instanceof ContinueError) {
-        pushLabel(`❯ Continue`, ansi.bold + C.warn)
-        pushLine(`Ran ${error.turn} turns (limit ${error.turn}). Continue?`, C.warn)
-        // Pause to ask: reuse permission mechanism
-        const willContinue = await new Promise((resolve) => {
-          state.permission = {
-            name: "continue",
-            args: { turns: error.turn },
-            resolve,
-          }
-          state.status = `Continue after ${error.turn} turns?`
-          render()
-        })
-        state.permission = null
-        if (!willContinue) {
-          pushLine("[continue cancelled]", C.warn)
-          break
-        }
-        pushLine("[continuing…]", C.tool)
-        // Recreate AbortController: once aborted, resume immediately fails (defensive; current path unreachable but tightly coupled)
-        state.controller = new AbortController()
-        continue
-      }
-      pushLine(`[error] ${error.message}`, C.error)
-      break
     }
+  } finally {
+    clearInterval(ticker)
+    state.processing = false
+    state.subTasks = {}
+    state.controller = null
+    state.status = "Ready"
+    // Auto-collapse todo panel when all tasks done (matching kimi-code TUI; agent.tasks are preserved)
+    if (state.tasks.length > 0 && state.tasks.every((t) => t.status === "done")) {
+      state.tasks = []
+    }
+    // Save session after every turn (survives crashes)
+    try {
+      saveSessionImpl(agent, state.lines)
+    } catch {
+      // Save failure doesn't interrupt usage
+    }
+    render()
   }
 
-  clearInterval(ticker)
-  state.processing = false
-  state.subTasks = {}
-  state.controller = null
-  state.status = "Ready"
-  // Auto-collapse todo panel when all tasks done (matching kimi-code TUI; agent.tasks are preserved)
-  if (state.tasks.length > 0 && state.tasks.every((t) => t.status === "done")) {
-    state.tasks = []
-  }
-  // Save session after every turn (survives crashes)
-  try {
-    saveSession(agent, state.lines)
-  } catch {
-    // Save failure doesn't interrupt usage
-  }
-  render()
-
   // Queued messages: auto-process next one
-  if (state.queue.length > 0) {
+  while (state.queue.length > 0 && !state.processing) {
     const next = state.queue.shift()
-    // Queued slash commands execute directly
+    // Queued slash commands execute directly — check every item, not just the first
     if (next.text.startsWith("/")) {
       await handleSlash(next.text)
       render()
-      // After slash command completes, continue checking queue
-      if (state.queue.length > 0 && !state.processing) {
-        const next2 = state.queue.shift()
-        await runAgentTurn(ctx, next2.text)
-      }
-    } else {
-      pushLabel(`❯ You: (from queue)`, ansi.bold + C.user)
-      await runAgentTurn(ctx, next.text)
+      continue
     }
+    pushLabel(`❯ You: (from queue)`, ansi.bold + C.user)
+    await runAgentTurn(ctx, next.text)
+    return
   }
 }