thincoder 0.7.8 → 0.8.1

package/src/cli/distill-command.mjs

@@ -0,0 +1,85 @@
+import { join } from "node:path"
+import { loadConfig, configPath } from "../config.mjs"
+import { createMemory } from "../memory.mjs"
+import { teamConfig, gitAuthor } from "./make-agent.mjs"
+import { setupWizard } from "./setup-wizard.mjs"
+import { askPermission } from "./permission.mjs"
+
+/** 缺 key 时的统一提示 */
+function noKeyMessage() {
+  return `还没有配置 API key。运行 thincoder 进入 TUI，用 /provider add 和 /provider key 配置；或直接编辑 ${configPath}`
+}
+
+/** thincoder distill <transcript-file> [--yes] [--scope=...]
+ *  返回退出码：0=成功，1=错误 */
+export async function distillCommand(args, exitSoon) {
+  const flags = {}
+  const positional = []
+  for (const a of args) {
+    const m = a.match(/^--([\w-]+)(?:=(.*))?$/)
+    if (m) flags[m[1]] = m[2] ?? true
+    else positional.push(a)
+  }
+  const file = positional[0]
+  if (!file) {
+    console.error("Usage: thincoder distill <transcript-file> [--yes] [--scope=personal|project|team]")
+    return 1
+  }
+  const { readFile } = await import("node:fs/promises")
+  const transcript = await readFile(file, "utf8")
+
+  const config = loadConfig()
+  let provider = config.provider
+  if (!provider.apiKey) {
+    if (!process.stdin.isTTY) {
+      console.error(noKeyMessage())
+      return 1
+    }
+    provider = await setupWizard()
+    if (!provider) {
+      return 1
+    }
+  }
+  const memory = createMemory({ dbPath: config.memory.dbPath })
+  const team = teamConfig(config)
+  const { extractCandidates, saveCandidate } = await import("../distill.mjs")
+
+  console.error("[distill] extracting candidates...")
+  let candidates
+  try {
+    candidates = await extractCandidates(provider, transcript)
+  } catch (error) {
+    console.error(`[distill] ${error.message}`)
+    return 1
+  }
+  if (candidates.length === 0) {
+    console.log("No distillable knowledge found in this session.")
+    return 0
+  }
+
+  const opts = {
+    projectDir: config.memory.projectDir ? join(process.cwd(), config.memory.projectDir) : null,
+    team,
+    author: gitAuthor(),
+  }
+  let saved = 0
+  for (const c of candidates) {
+    if (flags.scope) c.scope = flags.scope
+    console.log(`\n--- candidate ---`)
+    console.log(`[${c.type}] ${c.title}  (scope: ${c.scope})`)
+    console.log(c.content)
+    if (c.type === "rule") {
+      console.log("(rule 类知识通常建议手动撰写；确认提取吗？)")
+    }
+    const accept = flags.yes ? true : await askPermission("distill-save", { title: c.title })
+    if (!accept) {
+      console.log("skipped")
+      continue
+    }
+    const where = await saveCandidate(memory, c, opts)
+    console.log(`saved -> ${where}`)
+    saved++
+  }
+  console.log(`\nDistilled ${saved}/${candidates.length} entries.`)
+  return 0
+}

package/src/cli/make-agent.mjs

@@ -0,0 +1,85 @@
+import { execSync } from "node:child_process"
+import { join } from "node:path"
+import { createAgent } from "../agent.mjs"
+import { loadConfig, configDir } from "../config.mjs"
+import { createMemory, memoryTools, syncDir, codeSearchTool, docSearchTool } from "../memory.mjs"
+import { repoOutlineTool } from "../tools/repomap.mjs"
+import { builtinTools } from "../tools/index.mjs"
+
+/** 组装一个带记忆的 agent（同步各层索引后返回） */
+export async function makeAgent() {
+  const config = loadConfig()
+  const provider = config.provider
+  const providers = config.providersList
+  const memory = createMemory({ dbPath: config.memory.dbPath })
+  // 向量检索：配了 embedding 就启用（惰性生成向量，首次搜索时补算）
+  if (config.embedding?.apiKey) {
+    const { createEmbedder } = await import("../embedding.mjs")
+    memory.embedder = createEmbedder(config.embedding)
+  }
+  const cwd = process.cwd()
+  // code/doc 索引按 origin（项目根目录）隔离：检索只查本项目
+  memory.codeOrigin = cwd
+  // Project 层：启动时同步 .thincoder/memory/ 目录到索引（有就同步，没有就跳过）
+  if (config.memory.projectDir) {
+    memory.projectOrigin = join(cwd, config.memory.projectDir)
+    await syncDir(memory, { layer: "project", dir: memory.projectOrigin })
+  }
+  // Team 层（可选）：首次自动 clone；启动只索引本地目录，拉取远端走显式 thincoder sync
+  const team = teamConfig(config)
+  if (team) {
+    const { ensureClone } = await import("../git/gitmem.mjs")
+    await ensureClone(team)
+    await syncDir(memory, { layer: "team", dir: team.dir })
+  }
+  const baseTools = [...builtinTools, ...memoryTools(memory, { cwd, projectDir: config.memory.projectDir, author: gitAuthor(), team }), codeSearchTool(memory), docSearchTool(memory), repoOutlineTool(memory.db, cwd)]
+
+  // MCP servers：并行连接（一个死 server 不会拖住启动），失败的收集警告（TUI 下 stderr 不可见，通过 agent 对象传递）
+  const mcpServers = config.mcp?.servers ?? []
+  let mcpTools = []
+  const mcpWarnings = []
+  if (mcpServers.length) {
+    const { connectMcpServer } = await import("../mcp.mjs")
+    const results = await Promise.allSettled(mcpServers.map((srv) => connectMcpServer(srv)))
+    for (let i = 0; i < results.length; i++) {
+      const r = results[i]
+      if (r.status === "fulfilled") {
+        mcpTools = mcpTools.concat(r.value)
+      } else {
+        const srv = mcpServers[i]
+        const msg = `MCP server "${srv.name ?? srv.command}" failed to connect: ${r.reason?.message ?? r.reason}`
+        console.error(`[mcp] ${msg}`)
+        mcpWarnings.push(msg)
+      }
+    }
+  }
+
+  const agent = createAgent({
+    provider,
+    tools: [...baseTools, ...mcpTools],
+    config,
+    cwd,
+    memory,
+  })
+  agent.providers = providers
+  agent.activeProvider = config.activeProvider
+  agent._mcpWarnings = mcpWarnings
+  return agent
+}
+
+/** 读取 team 配置并补全默认目录；未配置返回 null */
+export function teamConfig(config) {
+  const team = config.memory?.team
+  if (!team?.repo) return null
+  const name = team.name ?? "default"
+  return { name, repo: team.repo, dir: team.dir ?? join(configDir, "teams", name) }
+}
+
+/** 条目作者：git config user.name 兜底 unknown */
+export function gitAuthor() {
+  try {
+    return execSync("git config user.name", { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim() || "unknown"
+  } catch {
+    return "unknown"
+  }
+}

package/src/cli/memory-command.mjs

@@ -0,0 +1,63 @@
+import { put, remove, search, list } from "../memory.mjs"
+
+/** thincoder memory <list|search|put|remove> 子命令 */
+export async function memoryCommand(memory, args) {
+  const [sub, ...rest] = args
+
+  const flags = {}
+  const positional = []
+  for (const a of rest) {
+    const m = a.match(/^--([\w-]+)=(.*)$/)
+    if (m) flags[m[1]] = m[2]
+    else positional.push(a)
+  }
+
+  switch (sub) {
+    case "list": {
+      const entries = await list(memory, { type: flags.type })
+      printEntries(entries)
+      break
+    }
+    case "search": {
+      const query = positional.join(" ")
+      if (!query) {
+        console.error("Usage: thincoder memory search <query>")
+        return 1
+      }
+      printEntries(await search(memory, query, { limit: 10 }))
+      break
+    }
+    case "put": {
+      if (!flags.type || !flags.title || !flags.content) {
+        console.error("Usage: thincoder memory put --type=<rule|knowledge|decision|pattern> --title=<t> --content=<c> [--tags=<t>]")
+        return 1
+      }
+      const id = await put(memory, { type: flags.type, title: flags.title, content: flags.content, tags: flags.tags ?? "" })
+      console.log(`Saved (id=${id})`)
+      break
+    }
+    case "remove": {
+      const id = Number(positional[0])
+      if (!id) {
+        console.error("Usage: thincoder memory remove <id>")
+        return 1
+      }
+      console.log((await remove(memory, id)) ? `Removed #${id}` : `No entry #${id}`)
+      break
+    }
+    default:
+      console.error("Usage: thincoder memory <list|search|put|remove>")
+      return 1
+  }
+}
+
+function printEntries(entries) {
+  if (entries.length === 0) {
+    console.log("(no entries)")
+    return
+  }
+  for (const e of entries) {
+    console.log(`#${e.id} [${e.type}] ${e.title}${e.tags ? `  (${e.tags})` : ""}`)
+    console.log(`  ${e.content.split("\n")[0].slice(0, 100)}`)
+  }
+}

package/src/cli/permission.mjs

@@ -0,0 +1,41 @@
+import { createInterface } from "node:readline"
+
+/** CLI 版工具参数摘要（截断长 JSON） */
+export function summarize(toolArgs) {
+  const s = JSON.stringify(toolArgs)
+  return s.length > 120 ? s.slice(0, 120) + "..." : s
+}
+
+/** 权限请求的关键信息（按工具定制），与 TUI 的 formatPermission 对齐。name 可能带子 agent 前缀（"coder/bash"），取基名匹配 */
+export function formatPermission(name, args) {
+  const cap = (s, n = 1000) => (s.length > n ? `${s.slice(0, n)}…(共 ${s.length} 字符)` : s)
+  const base = name.includes("/") ? name.split("/").pop() : name
+  if (base === "bash") return cap(args.command ?? "")
+  if (base === "write") return `${args.path}（写入 ${(args.content ?? "").length} 字符）\n${cap(args.content ?? "", 1000)}`
+  if (base === "edit") {
+    const oldLines = cap(args.old_string ?? "", 500).split("\n").map((l) => `- ${l}`).join("\n")
+    const newLines = cap(args.new_string ?? "", 500).split("\n").map((l) => `+ ${l}`).join("\n")
+    return `${args.path}\n${oldLines}\n  ↓\n${newLines}`
+  }
+  if (base === "delete") return `${args.path}${args.force ? "（force：跟踪文件也删）" : ""}`
+  if (base === "subagent") return cap(args.task ?? "", 500)
+  if (base === "memory_put") return `[${args.type ?? ""}] ${args.title ?? ""}\n${cap(args.content ?? "", 500)}`
+  return cap(summarize(args), 300)
+}
+
+/** 权限确认：TTY 下交互询问 y/n；非交互环境默认拒绝（安全优先） */
+export async function askPermission(name, toolArgs) {
+  if (!process.stdin.isTTY) {
+    console.error(`\n[deny] ${name} (non-interactive, side-effect tools require a TTY)`)
+    return false
+  }
+  const rl = createInterface({ input: process.stdin, output: process.stderr })
+  try {
+    const answer = await new Promise((resolve) => {
+      rl.question(`\n[allow?] ${name}\n${formatPermission(name, toolArgs)}\n(y/N) `, resolve)
+    })
+    return answer.trim().toLowerCase() === "y"
+  } finally {
+    rl.close()
+  }
+}

package/src/cli/setup-wizard.mjs

@@ -0,0 +1,70 @@
+import { existsSync, readFileSync } from "node:fs"
+import { createInterface } from "node:readline"
+import { configPath, saveConfig, PROVIDER_PRESETS } from "../config.mjs"
+
+/** 首次使用（TTY 下的 chat/distill）：问答式配置一个 provider 并落盘，返回运行时 provider；取消返回 null */
+export async function setupWizard() {
+  // 自带缓冲的提问器：rl.question 在输入被管道/快速粘贴时会丢行（问题注册前 line 已到达）
+  const rl = createInterface({ input: process.stdin, terminal: false })
+  const buffered = []
+  let waiter = null
+  rl.on("line", (line) => {
+    if (waiter) {
+      const w = waiter
+      waiter = null
+      w(line)
+    } else {
+      buffered.push(line)
+    }
+  })
+  const ask = (q) =>
+    new Promise((resolve) => {
+      process.stderr.write(q)
+      if (buffered.length) resolve(buffered.shift())
+      else waiter = resolve
+    })
+  try {
+    const presets = Object.entries(PROVIDER_PRESETS)
+    console.error("首次使用，先配置一个模型提供商：")
+    presets.forEach(([n, p], i) => console.error(`  ${i + 1}. ${n.padEnd(10)} ${p.desc}`))
+    console.error(`  ${presets.length + 1}. 自定义端点`)
+    const choice = Number((await ask(`选择 [1-${presets.length + 1}]: `)).trim())
+    let name, baseURL, model
+    if (choice === presets.length + 1) {
+      name = (await ask("名称（如 my-openai）: ")).trim()
+      baseURL = (await ask("baseURL（如 https://api.openai.com/v1）: ")).trim().replace(/\/+$/, "")
+      model = (await ask("模型（如 gpt-4o）: ")).trim()
+      if (!name || !/^https?:\/\//.test(baseURL) || !model) {
+        console.error("输入不完整或 baseURL 不合法，已取消")
+        return null
+      }
+    } else if (choice >= 1 && choice <= presets.length) {
+      name = presets[choice - 1][0]
+      baseURL = presets[choice - 1][1].baseURL
+      model = presets[choice - 1][1].model
+    } else {
+      console.error("无效选择，已取消")
+      return null
+    }
+    const apiKey = (await ask(`${name} 的 API key: `)).trim()
+    if (!apiKey) {
+      console.error("key 不能为空，已取消")
+      return null
+    }
+    const embedKey = (await ask("可选：embedding API key（SiliconFlow，向量检索用；回车跳过）: ")).trim()
+    const raw = existsSync(configPath) ? JSON.parse(readFileSync(configPath, "utf8")) : {}
+    const providers = raw.providers?.length ? raw.providers : []
+    const existing = providers.find((p) => p.name === name)
+    if (existing) Object.assign(existing, { baseURL, model, apiKey })
+    else providers.push({ name, baseURL, model, apiKey })
+    raw.providers = providers
+    raw.activeProvider = name
+    if (embedKey) raw.embedding = { ...(raw.embedding ?? {}), apiKey: embedKey }
+    saveConfig(raw)
+    console.error(`配置完成：${name} / ${model}（已写入 ${configPath}）`)
+    console.error(embedKey ? "向量检索已启用\n" : "（未配 embedding key：记忆为纯文本检索，之后在 config.json 的 embedding.apiKey 补上即可开启向量检索）\n")
+    return { name, baseURL, model, apiKey }
+  } finally {
+    rl.close()
+  }
+}

package/src/config.mjs

@@ -16,8 +16,8 @@ export const configPath = join(configDir, "config.json")
 export const PROVIDER_PRESETS = {
   deepseek: { baseURL: "https://api.deepseek.com/v1", model: "deepseek-v4-pro", thinking: { type: "enabled" }, reasoningEffort: "max", maxTokens: 393216, desc: "DeepSeek" },
   kimi:     { baseURL: "https://api.moonshot.cn/v1", model: "kimi-k3", thinking: null, reasoningEffort: "max", maxTokens: 131072, desc: "Kimi / Moonshot" },
-  glm:      { baseURL: "https://open.bigmodel.cn/api/paas/v4", model: "glm-5.2", thinking: { type: "enabled" }, reasoningEffort: "max", maxTokens: 131072, desc: "智谱 GLM" },
-  qwen:     { baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", model: "qwen3.7-max", maxTokens: 131072, desc: "通义千问" },
+  glm:      { baseURL: "https://open.bigmodel.cn/api/paas/v4", model: "glm-5.2", thinking: { type: "enabled" }, reasoningEffort: "max", maxTokens: 131072, desc: "Zhipu GLM" },
+  qwen:     { baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", model: "qwen3.7-max", maxTokens: 131072, desc: "Qwen / Alibaba" },
   minimax:  { baseURL: "https://api.minimax.chat/v1", chatPath: "/text/chatcompletion_v2", model: "MiniMax-M3", maxTokens: 131072, desc: "MiniMax" },
 }
 
@@ -131,8 +131,8 @@ export function findProvider(providers, name) {
   if (name) {
     const found = providers.find((p) => p.name === name)
     if (found) return found
-    const available = providers.map((p) => p.name).join(", ") || "(空)"
-    throw new Error(`activeProvider "${name}" 不在 providers 列表中（可用: ${available}），请检查配置是否打错字: ${configPath}`)
+    const available = providers.map((p) => p.name).join(", ") || "(empty)"
+    throw new Error(`activeProvider "${name}" not in providers list (available: ${available}); check for a typo in: ${configPath}`)
   }
   return providers[0] ?? { name: "default", baseURL: "", model: "" }
 }
@@ -148,7 +148,7 @@ export function loadConfig() {
     try {
       config = JSON.parse(readFileSync(configPath, "utf8"))
     } catch (error) {
-      throw new Error(`配置文件不是合法 JSON，请检查或删除: ${configPath}\n  ${error.message}`)
+      throw new Error(`Config file is not valid JSON, check or delete it: ${configPath}\n  ${error.message}`)
     }
   }
 
@@ -190,9 +190,6 @@ export function loadConfig() {
     const keyVar = envMap[merged.activeProvider]
     if (keyVar && process.env[keyVar]) runtimeProvider.apiKey = process.env[keyVar]
   }
-  if (!runtimeProvider.apiKey) {
-    runtimeProvider.apiKey = process.env.THINCODER_API_KEY
-  }
 
   // embedding apiKey
   if (!merged.embedding.apiKey) {

package/src/context.mjs

@@ -5,14 +5,8 @@
  * 压缩策略：保留最早 2 条 + 最近 N 条，中间由 LLM 摘要成一条（学 kimi-code，简化版）。
  */
 
-import { chat } from "./provider.mjs"
-
-/** 粗估一段文本的 token 数：ASCII 约 4 字符 1 token，CJK 等非 ASCII 约 1 字符 1 token */
-function estimateText(s) {
-  let nonAscii = 0
-  for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) > 0x7f) nonAscii++
-  return Math.ceil((s.length - nonAscii) / 4) + nonAscii
-}
+import { chat } from "./provider/index.mjs"
+import { estimateText } from "./provider/rate.mjs"
 
 /** 粗估一组消息的 token 数（正文 + 思考链 + tool_calls 参数） */
 export function estimateTokens(messages) {
@@ -208,15 +202,18 @@ const OVERSIZE_CONTENT_LIMIT = 8_000
  * 不动 reasoning_content（DeepSeek/Kimi 回传协议）与 tool_calls 配对结构，无协议 400 风险。
  * 只在 compressIfNeeded 判定超阈值后调用。返回是否有消息被截断。
  */
-export function shrinkOversized(agent) {
+export function shrinkOversized(agent, limit = OVERSIZE_CONTENT_LIMIT) {
   let shrunk = false
   for (const m of agent.history) {
     if ((m.role !== "user" && m.role !== "tool") || typeof m.content !== "string") continue
-    if (m.content.length <= OVERSIZE_CONTENT_LIMIT) continue
+    if (m.content.length <= limit) continue
+    // 截断保留首尾，中间换桩说明；keepHead/keepTail 按比例但不超过 limit 的 50%/25%
+    const keepHead = Math.min(Math.floor(limit * 0.5), 4000)
+    const keepTail = Math.min(Math.floor(limit * 0.25), 2000)
     m.content =
-      m.content.slice(0, 4_000) +
-      `\n[... ${m.content.length - 6_000} chars truncated — single message too large for context window ...]\n` +
-      m.content.slice(-2_000)
+      m.content.slice(0, keepHead) +
+      `\n[... ${m.content.length - keepHead - keepTail} chars truncated — single message too large for context window ...]\n` +
+      m.content.slice(-keepTail)
     shrunk = true
   }
   if (shrunk) {

package/src/distill.mjs

@@ -4,9 +4,9 @@
  * 绝不做会话结束后的全自动沉淀。
  */
 
-import { chat } from "./provider.mjs"
+import { chat } from "./provider/index.mjs"
 import { put, putMarkdown } from "./memory.mjs"
-import { commitAndPush } from "./gitmem.mjs"
+import { commitAndPush } from "./git/gitmem.mjs"
 
 const DISTILL_PROMPT = `你是知识提取器。阅读下面的 agent 工作会话记录，提取值得跨会话长期记住的知识。
 
@@ -47,7 +47,8 @@ export async function extractCandidates(provider, transcript) {
   const res = await chat(provider, {
     messages: [{ role: "user", content: DISTILL_PROMPT + transcript }],
   })
-  const match = res.content.match(/\[[\s\S]*\]/)
+  // 非贪婪匹配第一个 JSON 数组（贪婪 [\s\S]* 会跨多个数组把中间文本也吃进去）
+  const match = res.content.match(/\[[\s\S]*?\]/)
   if (!match) return []
   try {
     const parsed = JSON.parse(match[0])

package/src/embedding.mjs

@@ -5,7 +5,7 @@
  * 向量在入库前归一化，之后点积即余弦相似度。
  */
 
-import { RETRYABLE_STATUS } from "./provider.mjs"
+import { RETRYABLE_STATUS } from "./provider/index.mjs"
 const MAX_RETRIES = 3
 const BATCH_SIZE = 32 // 单次请求的文本数上限（SiliconFlow 限制内）
 
@@ -48,8 +48,9 @@ export async function embed(embedder, texts, { signal } = {}) {
 
 /** 余弦相似度（输入均已归一化，点积即余弦） */
 export function cosine(a, b) {
+  if (a.length !== b.length) return 0
   let sum = 0
-  const n = Math.min(a.length, b.length)
+  const n = a.length
   for (let i = 0; i < n; i++) sum += a[i] * b[i]
   return sum
 }
@@ -63,6 +64,7 @@ export function toBlob(vec) {
 export function fromBlob(buf) {
   // BLOB 可能来自 Buffer 池，byteOffset 不保证 4 对齐，直接建视图会 RangeError——先复制对齐
   if (buf.byteOffset % 4 !== 0) buf = new Uint8Array(buf)
+  if (buf.byteLength % 4 !== 0) return new Float32Array(0)
   return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4)
 }
 

package/src/{checkpoint.mjs → git/checkpoint.mjs}

@@ -9,7 +9,7 @@ import { createHash } from "node:crypto"
 import { existsSync } from "node:fs"
 import { cp, mkdir, readFile, readdir, rm, writeFile, copyFile } from "node:fs/promises"
 import { dirname, join, relative } from "node:path"
-import { configDir } from "./config.mjs"
+import { configDir } from "../config.mjs"
 
 const MAX_CHECKPOINTS = 20
 

package/src/mcp/helpers.mjs

@@ -0,0 +1,37 @@
+/**
+ * mcp/helpers.mjs — MCP 共享工具函数与常量
+ */
+
+export const INIT_TIMEOUT_MS = 30_000
+export const CALL_TIMEOUT_MS = 120_000
+export const ENDPOINT_WAIT_MS = 5_000
+
+let nextRpcId = 0
+export function rpcId() {
+  return String(++nextRpcId)
+}
+
+export function withTimeout(promise, ms) {
+  let timer
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(new Error(`timeout after ${ms}ms`)), ms)
+    timer.unref?.()
+  })
+  return Promise.race([promise.finally(() => clearTimeout(timer)), timeout])
+}
+
+export function quoteArg(s) {
+  return /[\s"]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s
+}
+
+export function withAuthToken(wsUrl, authorization) {
+  if (!authorization) return wsUrl
+  const token = authorization.replace(/^Bearer\s+/i, "")
+  const u = new URL(wsUrl)
+  u.searchParams.set("token", token)
+  return u.href
+}
+
+export function sanitizeToolName(name) {
+  return name.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 64)
+}