thincoder 0.12.59 → 0.12.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -4
- package/README.md +10 -8
- package/bin/thincoder.mjs +99 -133
- package/package.json +6 -4
- package/src/abort-provenance.mjs +116 -0
- package/src/acp/bridge.mjs +45 -21
- package/src/acp.mjs +6 -1
- package/src/advisor/citations.mjs +83 -21
- package/src/advisor/compaction.mjs +174 -0
- package/src/advisor/loop.mjs +293 -0
- package/src/advisor/messages.mjs +59 -137
- package/src/advisor/project-context.mjs +194 -0
- package/src/advisor/repos.mjs +17 -40
- package/src/advisor/run.mjs +156 -359
- package/src/advisor/truncate.mjs +57 -0
- package/src/advisor.mjs +27 -7
- package/src/agent/completion.mjs +17 -11
- package/src/agent/dispatch.mjs +145 -27
- package/src/agent/helpers.mjs +107 -13
- package/src/agent/record-results.mjs +55 -11
- package/src/agent/relay-prefix.mjs +39 -0
- package/src/agent/run-stages.mjs +242 -0
- package/src/agent/setup-reminders.mjs +69 -0
- package/src/agent/setup.mjs +107 -127
- package/src/agent/spawn-child.mjs +55 -13
- package/src/agent-tools/advisor-async.mjs +346 -0
- package/src/agent-tools/advisor-settle.mjs +231 -0
- package/src/agent-tools/advisor.mjs +167 -116
- package/src/agent-tools/async-settle.mjs +191 -0
- package/src/agent-tools/batch-segment.mjs +195 -0
- package/src/agent-tools/consult.mjs +139 -107
- package/src/agent-tools/design-token.mjs +117 -0
- package/src/agent-tools/digest-budget.mjs +76 -0
- package/src/agent-tools/eng.mjs +24 -29
- package/src/agent-tools/escalate-async.mjs +289 -0
- package/src/agent-tools/read-history.mjs +166 -32
- package/src/agent-tools/recent-changes.mjs +2 -1
- package/src/agent-tools/review-streak.mjs +93 -0
- package/src/agent-tools/settings.mjs +137 -34
- package/src/agent-tools/subagent-actions.mjs +180 -133
- package/src/agent-tools/subagent-async.mjs +184 -177
- package/src/agent-tools/subagent-panel.mjs +160 -0
- package/src/agent-tools/subagent-run.mjs +205 -0
- package/src/agent-tools/subagent-scheduler.mjs +100 -27
- package/src/agent-tools/subagent-spawn.mjs +453 -0
- package/src/agent-tools/subagent.mjs +256 -578
- package/src/agent-tools/verify.mjs +119 -292
- package/src/agent-tools.mjs +1 -0
- package/src/agent.mjs +89 -205
- package/src/cli/distill-command.mjs +12 -6
- package/src/cli/make-agent.mjs +26 -8
- package/src/cli/memory-command.mjs +4 -3
- package/src/cli/permission.mjs +2 -2
- package/src/cli/setup-wizard.mjs +42 -17
- package/src/completions.mjs +114 -0
- package/src/config-migrate.mjs +70 -0
- package/src/config.mjs +180 -63
- package/src/context.mjs +5 -147
- package/src/conventions.mjs +223 -0
- package/src/crash-reports.mjs +128 -0
- package/src/distill.mjs +11 -11
- package/src/expand-home.mjs +16 -0
- package/src/explore-distill.mjs +155 -0
- package/src/generate-title.mjs +1 -1
- package/src/hooks.mjs +7 -3
- package/src/memory/code-index.mjs +9 -3
- package/src/memory/code-sync.mjs +72 -32
- package/src/memory/core.mjs +6 -193
- package/src/memory/delete.mjs +236 -0
- package/src/memory/docs.mjs +68 -54
- package/src/memory/file-walk.mjs +109 -0
- package/src/memory/schema.mjs +15 -3
- package/src/memory.mjs +3 -1
- package/src/model-ref.mjs +66 -0
- package/src/model-specs.mjs +42 -8
- package/src/peer-domains.mjs +265 -0
- package/src/peer-instances.mjs +231 -0
- package/src/prompt-overlays.mjs +82 -0
- package/src/prompts/advisor-design.md +18 -75
- package/src/prompts/advisor-round1.md +14 -67
- package/src/prompts/advisor-round2.md +15 -51
- package/src/prompts/advisor-round3.md +15 -51
- package/src/prompts/common.md +115 -0
- package/src/prompts/consult-base.md +5 -23
- package/src/prompts/discipline-engineering.md +217 -0
- package/src/prompts/discipline-normal.md +179 -0
- package/src/prompts/persona-coder.md +21 -0
- package/src/prompts/persona-eng-coder.md +37 -0
- package/src/prompts/persona-eng-designer.md +55 -0
- package/src/prompts/persona-engineering.md +54 -0
- package/src/prompts/persona-explore.md +15 -0
- package/src/prompts/persona-normal.md +27 -0
- package/src/prompts/persona-plan.md +26 -0
- package/src/provider/anthropic.mjs +4 -4
- package/src/provider/core.mjs +18 -98
- package/src/provider/errors.mjs +101 -0
- package/src/provider/google.mjs +5 -6
- package/src/provider/index.mjs +2 -1
- package/src/provider/list-models.mjs +93 -0
- package/src/provider/rate.mjs +2 -1
- package/src/provider/responses.mjs +5 -3
- package/src/provider/retry.mjs +8 -45
- package/src/provider/sse.mjs +3 -4
- package/src/proxy.mjs +9 -14
- package/src/session-gc.mjs +214 -0
- package/src/session-guard.mjs +47 -0
- package/src/session-rename.mjs +38 -0
- package/src/session-slots.mjs +188 -60
- package/src/session.mjs +104 -124
- package/src/token-ttl.mjs +274 -0
- package/src/tools/{system.mjs → bash.mjs} +19 -221
- package/src/tools/checklist-sync.mjs +181 -0
- package/src/tools/checklist.mjs +52 -39
- package/src/tools/edit-batch.mjs +109 -10
- package/src/tools/edit-diff.mjs +110 -27
- package/src/tools/edit.md +17 -12
- package/src/tools/execute.mjs +31 -4
- package/src/tools/file.mjs +41 -16
- package/src/tools/git.md +1 -1
- package/src/tools/git.mjs +23 -34
- package/src/tools/glob-dialect.mjs +130 -0
- package/src/tools/glob.md +3 -3
- package/src/tools/grep.md +1 -1
- package/src/tools/index.mjs +9 -8
- package/src/tools/ops.mjs +188 -3
- package/src/tools/patch.mjs +3 -3
- package/src/tools/question.md +4 -0
- package/src/tools/question.mjs +26 -0
- package/src/tools/read.md +1 -2
- package/src/tools/read_image.md +1 -1
- package/src/tools/search.mjs +236 -0
- package/src/tools/shared.mjs +14 -13
- package/src/tools/wait_for.md +22 -0
- package/src/tui/agent-turn.mjs +36 -228
- package/src/tui/ansi.mjs +2 -0
- package/src/tui/clipboard.mjs +7 -1
- package/src/tui/cmd-advisor.mjs +3 -2
- package/src/tui/cmd-config.mjs +142 -30
- package/src/tui/cmd-eng.mjs +28 -40
- package/src/tui/cmd-exit.mjs +6 -8
- package/src/tui/cmd-mcp.mjs +8 -2
- package/src/tui/cmd-model.mjs +14 -12
- package/src/tui/cmd-new.mjs +3 -2
- package/src/tui/cmd-reindex.mjs +7 -0
- package/src/tui/cmd-session.mjs +19 -4
- package/src/tui/cmd-submodel.mjs +8 -5
- package/src/tui/cmd-think.mjs +10 -10
- package/src/tui/cmd-undo.mjs +4 -3
- package/src/tui/cmd-upgrade.mjs +19 -4
- package/src/tui/config-helpers.mjs +28 -16
- package/src/tui/distill-cmd.mjs +1 -1
- package/src/tui/index.mjs +40 -38
- package/src/tui/interaction.mjs +3 -3
- package/src/tui/key-handler.mjs +61 -17
- package/src/tui/key-modes.mjs +86 -8
- package/src/tui/layout.mjs +18 -10
- package/src/tui/model-catalog.mjs +89 -0
- package/src/tui/model-picker.mjs +498 -0
- package/src/tui/mouse.mjs +52 -9
- package/src/tui/pickers.mjs +28 -392
- package/src/tui/render-frame.mjs +32 -16
- package/src/tui/render-loop.mjs +2 -0
- package/src/tui/render-segments.mjs +12 -9
- package/src/tui/render.mjs +37 -5
- package/src/tui/slash-commands.mjs +2 -2
- package/src/tui/startup.mjs +4 -0
- package/src/tui/subagent-blocks.mjs +106 -295
- package/src/tui/subagent-children.mjs +162 -0
- package/src/tui/subagent-freeze.mjs +169 -0
- package/src/tui/subagent-panel.mjs +24 -31
- package/src/tui/suspension-drive.mjs +297 -0
- package/src/tui/tool-args.mjs +7 -5
- package/src/tui/tool-display.mjs +143 -0
- package/src/tui/tool-events.mjs +56 -185
- package/src/tui/tui-lifecycle.mjs +46 -4
- package/src/tui/update-notice.mjs +4 -0
- package/src/tui/wizard.mjs +61 -21
- package/src/tui/wrapped-spawn.mjs +38 -0
- package/src/prompts/coder.md +0 -56
- package/src/prompts/discipline.md +0 -102
- package/src/prompts/eng-coder.md +0 -44
- package/src/prompts/engineering-sub.md +0 -34
- package/src/prompts/engineering.md +0 -340
- package/src/prompts/explore.md +0 -21
- package/src/prompts/main.md +0 -56
- package/src/prompts/methodology-template.md +0 -58
- package/src/prompts/plan.md +0 -16
- package/src/prompts/system.md +0 -63
- package/src/tools/pdf-parse-text.mjs +0 -497
- package/src/tools/pdf-parse-xref.mjs +0 -499
- package/src/tools/pdf.mjs +0 -155
- package/src/tools/read_pdf.md +0 -21
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* explore-distill.mjs — End-of-run exploration distillation (AGENT-LOOP §13 +
|
|
3
|
+
* CONTEXT-COMPACTION §5, 2026-08-23). 2026-09-05 module-split: moved verbatim out of
|
|
4
|
+
* context.mjs (524 > 500 hard limit). The main agent's machine line is flooded by inline
|
|
5
|
+
* step-by-step exploration (read/grep/...). At run end we distill THIS run's exploration
|
|
6
|
+
* tool-results into one semantic summary note that replaces them in the machine line,
|
|
7
|
+
* while agent._fullHistory (the human line) stays untouched. VS Code compact.mjs
|
|
8
|
+
* distillation mirrors this module (same-name file in thincoder-vscode/src, cross-repo
|
|
9
|
+
* parity anchors point here).
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { chat } from "./provider/index.mjs"
|
|
13
|
+
|
|
14
|
+
/** Read-only knowledge tools counted as "exploration" (execute writes files → never exploration). */
|
|
15
|
+
export const EXPLORE_TOOLS = new Set([
|
|
16
|
+
"read", "grep", "glob", "ls", "code_search", "doc_search", "repo_outline",
|
|
17
|
+
])
|
|
18
|
+
|
|
19
|
+
/** Summary prompt for turning a burst of exploration results into a semantic summary. */
|
|
20
|
+
export const EXPLORE_SUMMARY_PROMPT = `You are distilling exploration tool results. Summarize the following read-only codebase exploration into a compact semantic summary for the main agent's own context.
|
|
21
|
+
|
|
22
|
+
Requirements:
|
|
23
|
+
- Capture WHAT was discovered, WHERE (which files / directories / symbols), and the KEY CONCLUSIONS — do not list tool calls mechanically
|
|
24
|
+
- Keep actionable facts the main agent needs to continue: code locations, function names, file paths, structure, and open questions the exploration raised
|
|
25
|
+
- Drop raw tool-output noise, repeated lines, and verbatim file dumps — keep only what must be remembered
|
|
26
|
+
- Be honest: mark anything not actually verified as "unverified"; do not present guesses as facts
|
|
27
|
+
- Use bullet points; aim for information completeness, not a hard word limit
|
|
28
|
+
|
|
29
|
+
Exploration log:
|
|
30
|
+
`
|
|
31
|
+
|
|
32
|
+
/** tool_calls name across both stored shapes ({function:{name}} and flat {name}). */
|
|
33
|
+
function toolCallName(tc) {
|
|
34
|
+
return tc?.function?.name ?? tc?.name ?? ""
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Tool that produced a tool-result message (falls back to its owner assistant's tool_call). */
|
|
38
|
+
function toolResultName(msg, ownerToolCalls) {
|
|
39
|
+
if (typeof msg?.name === "string" && msg.name) return msg.name
|
|
40
|
+
const owner = (ownerToolCalls ?? []).find((tc) => tc.id === msg?.tool_call_id)
|
|
41
|
+
return owner ? toolCallName(owner) : ""
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Find the pure-exploration "assistant(tool_calls)→tool…" pair blocks added since `start`.
|
|
46
|
+
* A block is explorable only when EVERY tool call AND every tool result in it is an exploration
|
|
47
|
+
* tool — mixed blocks (read + edit in one turn) stay untouched, or we'd orphan the edit pairing.
|
|
48
|
+
*/
|
|
49
|
+
function findExplorationBlocks(history, start) {
|
|
50
|
+
const blocks = []
|
|
51
|
+
let i = start
|
|
52
|
+
while (i < history.length) {
|
|
53
|
+
const m = history[i]
|
|
54
|
+
if (m?.role === "assistant" && Array.isArray(m.tool_calls) && m.tool_calls.length > 0) {
|
|
55
|
+
let j = i + 1
|
|
56
|
+
while (j < history.length && history[j]?.role === "tool") j++
|
|
57
|
+
const toolMsgs = history.slice(i + 1, j)
|
|
58
|
+
const allCallsExplore = m.tool_calls.every((tc) => EXPLORE_TOOLS.has(toolCallName(tc)))
|
|
59
|
+
const allResultsExplore = toolMsgs.length > 0 && toolMsgs.every((t) => EXPLORE_TOOLS.has(toolResultName(t, m.tool_calls)))
|
|
60
|
+
if (allCallsExplore && allResultsExplore) {
|
|
61
|
+
blocks.push({ start: i, end: j, messages: history.slice(i, j), toolCount: toolMsgs.length })
|
|
62
|
+
}
|
|
63
|
+
i = j
|
|
64
|
+
} else {
|
|
65
|
+
i++
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return blocks
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Serialize a batch of exploration messages for the summary LLM (same shape as compaction serialization). */
|
|
72
|
+
function serializeExplorationMessages(messages) {
|
|
73
|
+
const cap = 8000 // exploration results ARE the signal to distill — generous cap (quality-first, N1)
|
|
74
|
+
return messages
|
|
75
|
+
.map((m) => {
|
|
76
|
+
const toolNote = m.tool_calls ? ` [called tools: ${m.tool_calls.map(toolCallName).join(", ")}]` : ""
|
|
77
|
+
let text = ""
|
|
78
|
+
if (typeof m.content === "string") text = m.content
|
|
79
|
+
else if (Array.isArray(m.content)) text = m.content.filter((p) => p?.type === "text").map((p) => p.text ?? "").join(" ")
|
|
80
|
+
return `[${m.role}]${toolNote} ${text.slice(0, cap)}`
|
|
81
|
+
})
|
|
82
|
+
.join("\n")
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Core (shared) distillation: replace this run's pure-exploration pair blocks with a single
|
|
87
|
+
* "[Exploration summary]" note placed where the first block was. Returns a NEW history array,
|
|
88
|
+
* or null when there is nothing to shrink (<3 exploration results / LLM failure). Pairing-safe:
|
|
89
|
+
* whole assistant→tool blocks are removed, so no orphan tool_calls/tool can survive.
|
|
90
|
+
*/
|
|
91
|
+
async function distillExplorations(history, start, provider, signal, agent, depth) {
|
|
92
|
+
if (!Array.isArray(history) || history.length - start < 2) return null
|
|
93
|
+
const blocks = findExplorationBlocks(history, start)
|
|
94
|
+
const resultCount = blocks.reduce((n, b) => n + b.toolCount, 0)
|
|
95
|
+
if (resultCount < 3) return null
|
|
96
|
+
|
|
97
|
+
const serialized = blocks.map((b) => serializeExplorationMessages(b.messages)).join("\n")
|
|
98
|
+
|
|
99
|
+
let summary
|
|
100
|
+
try {
|
|
101
|
+
// Silent by design (D11): thinking:null and no onToken/onReasoning — this internal
|
|
102
|
+
// distillation must not stream to the frontend. signal propagates user cancellation.
|
|
103
|
+
const resp = await chat({ ...provider, thinking: null, reasoningEffort: null }, {
|
|
104
|
+
messages: [{ role: "user", content: EXPLORE_SUMMARY_PROMPT + serialized }],
|
|
105
|
+
signal,
|
|
106
|
+
// §18.6 D-TR4:轨迹元数据增补——kind=distill(探索蒸馏面——agent 元数据透出;
|
|
107
|
+
// depth 经 summarizeRunExplorations 参数透传——agent.mjs 主作用域传入)
|
|
108
|
+
logCtx: {
|
|
109
|
+
stage: "distill", child: agent?._logId ?? null, kind: "distill",
|
|
110
|
+
role: agent?._role ?? null, depth: depth ?? null,
|
|
111
|
+
session: agent?._sessionStart ?? null, cwd: agent?.cwd ?? process.cwd(),
|
|
112
|
+
traces: agent?.config?.traces?.enabled !== false,
|
|
113
|
+
},
|
|
114
|
+
})
|
|
115
|
+
summary = resp?.content
|
|
116
|
+
} catch {
|
|
117
|
+
return null // N3: never block the run's return or lose history — original results stay
|
|
118
|
+
}
|
|
119
|
+
if (!summary) return null
|
|
120
|
+
|
|
121
|
+
const drop = new Set()
|
|
122
|
+
for (const b of blocks) for (let k = b.start; k < b.end; k++) drop.add(k)
|
|
123
|
+
const note = { role: "user", content: "[Exploration summary]\n" + summary }
|
|
124
|
+
const next = []
|
|
125
|
+
let inserted = false
|
|
126
|
+
for (let k = 0; k < history.length; k++) {
|
|
127
|
+
if (drop.has(k)) {
|
|
128
|
+
if (!inserted) { next.push(note); inserted = true }
|
|
129
|
+
continue
|
|
130
|
+
}
|
|
131
|
+
next.push(history[k])
|
|
132
|
+
}
|
|
133
|
+
return next
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* End-of-run exploration distillation (runAgent's final return). Shrinks the MACHINE line
|
|
138
|
+
* (agent.history) only; agent._fullHistory is never touched. Triggers when this run added ≥3
|
|
139
|
+
* exploration tool results; on LLM failure it silently keeps the original history (N3).
|
|
140
|
+
* The distillation itself is silent and never streams (D11); `callbacks.onDistilled` fires
|
|
141
|
+
* ONLY after the replacement actually lands (never on no-op/failure) — callers persist the
|
|
142
|
+
* compressed session (SEND-STALL-DISTILL §2.3).
|
|
143
|
+
*/
|
|
144
|
+
export async function summarizeRunExplorations(agent, callbacks, signal, depth = 0) {
|
|
145
|
+
const next = await distillExplorations(agent.history, agent._runStartHistoryLen ?? 0, agent.provider, signal, agent, depth)
|
|
146
|
+
if (!next) return
|
|
147
|
+
agent.history = next
|
|
148
|
+
// The machine line changed shape — the measured token baseline was for the pre-shrink context.
|
|
149
|
+
// Invalidate so the next compaction check re-estimates instead of over-counting stale history.
|
|
150
|
+
agent._lastPromptTokens = null
|
|
151
|
+
agent._usageAtLen = null
|
|
152
|
+
// The compressed machine line must reach the disk: the run's own save already happened,
|
|
153
|
+
// so without this hook the async distill would leave the session un-compressed on exit.
|
|
154
|
+
callbacks.onDistilled?.()
|
|
155
|
+
}
|
package/src/generate-title.mjs
CHANGED
|
@@ -46,7 +46,7 @@ export async function generateTitle(userContent, provider) {
|
|
|
46
46
|
const url = `${provider.baseURL.replace(/\/+$/, "")}${chatPath}`
|
|
47
47
|
const opts = {
|
|
48
48
|
method: "POST",
|
|
49
|
-
headers: { "Content-Type": "application/json", Authorization: `Bearer ${provider.apiKey}` },
|
|
49
|
+
headers: { ...(provider.headers ?? {}), "Content-Type": "application/json", Authorization: `Bearer ${provider.apiKey}` }, // 定制头展开(PROVIDER.md §21)——定制头在前、内置头在后:内置头胜出
|
|
50
50
|
body,
|
|
51
51
|
signal: AbortSignal.timeout(10000),
|
|
52
52
|
}
|
package/src/hooks.mjs
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* PreToolUse — before each tool call (can block execution)
|
|
9
9
|
* PostToolUse — after successful tool execution
|
|
10
10
|
* PostToolUseFailure — after failed tool execution
|
|
11
|
-
*
|
|
11
|
+
* Stop — main-session run end (no tool name; block is meaningless)
|
|
12
12
|
*
|
|
13
13
|
* Each hook: { matcher?, command, args?, timeout?, action? }
|
|
14
14
|
* matcher: regex against tool name (default: match all)
|
|
@@ -29,9 +29,11 @@ export async function runHooks(event, ctx) {
|
|
|
29
29
|
if (!hooks?.length) return true
|
|
30
30
|
|
|
31
31
|
for (const hook of hooks) {
|
|
32
|
-
|
|
32
|
+
// Matcher = tool-name regex: tool events always carry a toolName; no-tool-name
|
|
33
|
+
// events (Stop) ignore it — a configured matcher must not silently never fire.
|
|
34
|
+
if (hook.matcher && ctx.toolName != null) {
|
|
33
35
|
try {
|
|
34
|
-
if (!new RegExp(hook.matcher).test(ctx.toolName
|
|
36
|
+
if (!new RegExp(hook.matcher).test(ctx.toolName)) continue
|
|
35
37
|
} catch { /* invalid regex → skip */ }
|
|
36
38
|
}
|
|
37
39
|
|
|
@@ -48,6 +50,8 @@ async function runOneHook(event, hook, ctx) {
|
|
|
48
50
|
toolArgs: ctx.toolArgs ?? null,
|
|
49
51
|
result: ctx.result ?? null,
|
|
50
52
|
error: ctx.error?.message ?? null,
|
|
53
|
+
// Event-specific fields (Stop: turn / reason) — existing callers pass no extra ⇒ payload unchanged.
|
|
54
|
+
...(ctx.extra ?? {}),
|
|
51
55
|
timestamp: new Date().toISOString(),
|
|
52
56
|
})
|
|
53
57
|
|
|
@@ -8,14 +8,20 @@ import { segmentCJK, CODE_EXTS, DOC_EXTS, SKIP_DIRS, BIG_FILE_LINES } from "./sc
|
|
|
8
8
|
export function detectLanguage(filename) {
|
|
9
9
|
const ext = filename.slice(filename.lastIndexOf(".")).toLowerCase()
|
|
10
10
|
const map = {
|
|
11
|
-
".mjs": "javascript", ".js": "javascript", ".
|
|
11
|
+
".mjs": "javascript", ".js": "javascript", ".cjs": "javascript", ".jsx": "jsx",
|
|
12
|
+
".ts": "typescript", ".tsx": "tsx", ".mts": "typescript", ".cts": "typescript",
|
|
12
13
|
".py": "python", ".rs": "rust", ".go": "go", ".java": "java",
|
|
13
14
|
".c": "c", ".h": "c", ".cpp": "cpp", ".hpp": "cpp",
|
|
14
|
-
".rb": "ruby", ".swift": "swift", ".kt": "kotlin",
|
|
15
|
+
".rb": "ruby", ".swift": "swift", ".kt": "kotlin", ".dart": "dart", ".lua": "lua",
|
|
16
|
+
".cs": "csharp", ".fs": "fsharp", ".fsx": "fsharp",
|
|
17
|
+
".clj": "clojure", ".cljs": "clojure", ".ex": "elixir", ".exs": "elixir",
|
|
18
|
+
".erl": "erlang", ".hrl": "erlang", ".scala": "scala", ".groovy": "groovy",
|
|
19
|
+
".pl": "perl", ".pm": "perl", ".r": "r", ".jl": "julia", ".zig": "zig",
|
|
20
|
+
".ps1": "powershell", ".proto": "protobuf", ".graphql": "graphql", ".tf": "terraform", ".hcl": "hcl",
|
|
15
21
|
".sh": "bash", ".bash": "bash", ".sql": "sql",
|
|
16
22
|
".yaml": "yaml", ".yml": "yaml", ".toml": "toml", ".json": "json",
|
|
17
23
|
".css": "css", ".html": "html", ".vue": "vue", ".svelte": "svelte",
|
|
18
|
-
".md": "markdown", ".mdc": "markdown",
|
|
24
|
+
".md": "markdown", ".mdc": "markdown", ".mdx": "markdown", ".org": "org", ".wiki": "wiki", ".tex": "tex",
|
|
19
25
|
}
|
|
20
26
|
return map[ext] ?? ext.slice(1)
|
|
21
27
|
}
|
package/src/memory/code-sync.mjs
CHANGED
|
@@ -4,13 +4,30 @@
|
|
|
4
4
|
import { readFile, stat } from "node:fs/promises"
|
|
5
5
|
import { join, relative } from "node:path"
|
|
6
6
|
import { embed, cosine, toBlob, fromBlob } from "../embedding.mjs"
|
|
7
|
-
import { CODE_EXTS, DOC_EXTS,
|
|
7
|
+
import { CODE_EXTS, DOC_EXTS, MAX_CODE_FILE_BYTES, MAX_DOC_FILE_BYTES } from "./schema.mjs"
|
|
8
8
|
import { buildFtsQuery, ensureEmbeddings, EMBED_TEXT_MAX_LEN } from "./core.mjs"
|
|
9
9
|
import { detectLanguage, _upsertCodeFile, _upsertDocFile, yieldTick } from "./code-index.mjs"
|
|
10
|
+
import { walkProjectFiles, isSkippedRelPath, extensionOf, createUnlistedTally, MAX_WALK_FILES } from "./file-walk.mjs"
|
|
11
|
+
import { loadConventions } from "../conventions.mjs"
|
|
12
|
+
import { logEvent } from "../log.mjs"
|
|
10
13
|
|
|
11
14
|
const DIFF_FULL_SYNC_THRESHOLD = 200
|
|
12
15
|
const CODE_EMBED_BATCH = 64
|
|
13
16
|
|
|
17
|
+
/**
|
|
18
|
+
* Index extension sets = built-in tables ∪ project declaration
|
|
19
|
+
* (`.thincoder/conventions.json` → index.codeExtensions / index.docExtensions;
|
|
20
|
+
* PORTABILITY PO-9/§3.7 — a project may declare extensions this product does not
|
|
21
|
+
* ship a default for). Declaration only ADDS (union), never removes.
|
|
22
|
+
*/
|
|
23
|
+
export function indexExtensions(dir) {
|
|
24
|
+
const conv = loadConventions(dir)
|
|
25
|
+
return {
|
|
26
|
+
code: new Set([...CODE_EXTS, ...conv.index.codeExtensions]),
|
|
27
|
+
doc: new Set([...DOC_EXTS, ...conv.index.docExtensions]),
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
14
31
|
/**
|
|
15
32
|
* git-driven incremental indexing: use git diff to find files changed since
|
|
16
33
|
* the last index, and only rebuild FTS5 chunks for those files (vectors are untouched).
|
|
@@ -19,6 +36,7 @@ const CODE_EMBED_BATCH = 64
|
|
|
19
36
|
*/
|
|
20
37
|
export async function gitSync(memory, dir, { onProgress } = {}) {
|
|
21
38
|
const { execFile: _execFile } = await import("node:child_process")
|
|
39
|
+
const { code: codeExts, doc: docExts } = indexExtensions(dir)
|
|
22
40
|
const gitRun = (args) => new Promise((resolve, reject) => {
|
|
23
41
|
_execFile("git", args, { cwd: dir, encoding: "utf8", timeout: 10000, windowsHide: true }, (err, stdout) => {
|
|
24
42
|
if (err) reject(err); else resolve(stdout)
|
|
@@ -55,17 +73,16 @@ export async function gitSync(memory, dir, { onProgress } = {}) {
|
|
|
55
73
|
for (let i = 0; i < diffOut.length; i++) {
|
|
56
74
|
const rel = diffOut[i].replaceAll("\\", "/")
|
|
57
75
|
const abs = join(dir, rel)
|
|
58
|
-
const ext =
|
|
76
|
+
const ext = extensionOf(rel)
|
|
59
77
|
|
|
60
|
-
|
|
61
|
-
if (pathDirs.some((d) => SKIP_DIRS.has(d) || d.startsWith("."))) continue
|
|
78
|
+
if (isSkippedRelPath(rel)) continue
|
|
62
79
|
|
|
63
|
-
if (!
|
|
80
|
+
if (!codeExts.has(ext) && !docExts.has(ext)) { skipped++; continue }
|
|
64
81
|
|
|
65
82
|
try {
|
|
66
83
|
const text = await readFile(abs, "utf8")
|
|
67
84
|
const lines = text.split("\n")
|
|
68
|
-
if (
|
|
85
|
+
if (codeExts.has(ext)) {
|
|
69
86
|
const lang = detectLanguage(abs)
|
|
70
87
|
let mtimeMs = 0
|
|
71
88
|
try { mtimeMs = Math.floor((await stat(abs)).mtimeMs) } catch { /* new file */ }
|
|
@@ -79,7 +96,7 @@ export async function gitSync(memory, dir, { onProgress } = {}) {
|
|
|
79
96
|
} catch (e) {
|
|
80
97
|
const isDeleted = e.code === "ENOENT"
|
|
81
98
|
if (isDeleted) {
|
|
82
|
-
if (
|
|
99
|
+
if (codeExts.has(ext)) memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(dir, rel)
|
|
83
100
|
else memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(dir, rel)
|
|
84
101
|
removed++
|
|
85
102
|
} else {
|
|
@@ -104,16 +121,27 @@ export async function gitSync(memory, dir, { onProgress } = {}) {
|
|
|
104
121
|
|
|
105
122
|
/**
|
|
106
123
|
* List project files matching the given extensions.
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
* (
|
|
110
|
-
*
|
|
124
|
+
* Preferred source = git (`git ls-files --cached --others --exclude-standard`:
|
|
125
|
+
* tracked + untracked-not-ignored, .gitignore respected). Projects WITHOUT git
|
|
126
|
+
* fall back to a filesystem walk (PORTABILITY FR15/P8) — before, the index was
|
|
127
|
+
* silently empty there. The walk cannot honour .gitignore (no git → no such
|
|
128
|
+
* concept) and skips the shared SKIP_DIRS/dot-directory set instead.
|
|
129
|
+
* Returns { entries, unlisted }: `entries` = { abs, rel } pairs (rel relative to
|
|
130
|
+
* dir); `unlisted` = files whose extension is in NO index list (count + sample),
|
|
131
|
+
* the visible signal behind "my .xyz files are not searchable" (PORTABILITY PO-9).
|
|
132
|
+
* `truncated` = the non-git walk hit its file cap (capped trees are logged, never
|
|
133
|
+
* silently indexed-partial). opts.maxFiles = the walk cap (test seam).
|
|
111
134
|
*/
|
|
112
|
-
export async function listProjectFiles(dir, exts) {
|
|
135
|
+
export async function listProjectFiles(dir, exts, { maxFiles } = {}) {
|
|
113
136
|
const { execFile: _execFile } = await import("node:child_process")
|
|
114
137
|
const { join: joinPath } = await import("node:path")
|
|
138
|
+
const { code, doc } = indexExtensions(dir)
|
|
139
|
+
const tally = createUnlistedTally(new Set([...code, ...doc]))
|
|
115
140
|
|
|
116
|
-
|
|
141
|
+
const files = []
|
|
142
|
+
const finish = (truncated = false) => ({ entries: files, unlisted: tally.result(), truncated })
|
|
143
|
+
|
|
144
|
+
// Git listing when available; a non-git project falls through to the walk.
|
|
117
145
|
let gitTop
|
|
118
146
|
try {
|
|
119
147
|
gitTop = (await new Promise((resolve, reject) => {
|
|
@@ -121,10 +149,17 @@ export async function listProjectFiles(dir, exts) {
|
|
|
121
149
|
(err, stdout) => { if (err) reject(err); else resolve(stdout.trim()) })
|
|
122
150
|
})).replace(/\\/g, "/")
|
|
123
151
|
} catch {
|
|
124
|
-
|
|
152
|
+
// Not a git repo → filesystem walk (FR15: the index must still be usable).
|
|
153
|
+
const walked = await walkProjectFiles(dir, exts, { knownExts: new Set([...code, ...doc]), ...(maxFiles !== undefined ? { maxFiles } : {}) })
|
|
154
|
+
files.push(...walked.files)
|
|
155
|
+
// The walk's cap must stay visible end-to-end (file-walk.mjs: "never silently
|
|
156
|
+
// dropped") — a capped listing says so in the log, not only in a discarded flag.
|
|
157
|
+
if (walked.truncated) {
|
|
158
|
+
logEvent("index:truncated", { dir, files: walked.files.length, maxFiles: maxFiles ?? MAX_WALK_FILES })
|
|
159
|
+
}
|
|
160
|
+
return { entries: files, unlisted: walked.unlisted, truncated: walked.truncated }
|
|
125
161
|
}
|
|
126
162
|
|
|
127
|
-
const files = []
|
|
128
163
|
try {
|
|
129
164
|
const raw = await new Promise((resolve, reject) => {
|
|
130
165
|
_execFile("git", ["ls-files", "--cached", "--others", "--exclude-standard"],
|
|
@@ -134,16 +169,15 @@ export async function listProjectFiles(dir, exts) {
|
|
|
134
169
|
for (const line of raw.trim().split("\n")) {
|
|
135
170
|
const p = line.trim()
|
|
136
171
|
if (!p) continue
|
|
137
|
-
const ext = p.slice(p.lastIndexOf(".")).toLowerCase()
|
|
138
|
-
if (!exts.has(ext)) continue
|
|
139
|
-
const abs = joinPath(dir, p)
|
|
140
172
|
const rel = p.replace(/\\/g, "/")
|
|
141
|
-
if (rel
|
|
142
|
-
|
|
173
|
+
if (isSkippedRelPath(rel)) continue
|
|
174
|
+
const ext = extensionOf(rel)
|
|
175
|
+
if (!ext || !exts.has(ext)) { tally.note(rel); continue }
|
|
176
|
+
files.push({ abs: joinPath(dir, rel), rel })
|
|
143
177
|
}
|
|
144
178
|
} catch { /* ls-files failed */ }
|
|
145
179
|
|
|
146
|
-
return
|
|
180
|
+
return finish()
|
|
147
181
|
}
|
|
148
182
|
|
|
149
183
|
|
|
@@ -152,7 +186,8 @@ export async function listProjectFiles(dir, exts) {
|
|
|
152
186
|
* Incremental by mtime — only rebuilds chunks for files that have changed.
|
|
153
187
|
*/
|
|
154
188
|
export async function codeSync(memory, dir, { onProgress } = {}) {
|
|
155
|
-
const
|
|
189
|
+
const { code: exts } = indexExtensions(dir)
|
|
190
|
+
const { entries, unlisted } = await listProjectFiles(dir, exts)
|
|
156
191
|
const files = [] // { abs, rel, mtimeMs }
|
|
157
192
|
let overSizeSkipped = 0
|
|
158
193
|
for (const { abs, rel } of entries) {
|
|
@@ -206,7 +241,10 @@ export async function codeSync(memory, dir, { onProgress } = {}) {
|
|
|
206
241
|
|
|
207
242
|
onProgress?.({ phase: "done", total: files.length, updated, removed, skipped, failed, overSizeSkipped })
|
|
208
243
|
markIndexedCommit(memory, dir)
|
|
209
|
-
|
|
244
|
+
if (unlisted.count > 0) {
|
|
245
|
+
logEvent("index:unlisted", { dir, kind: "code", count: unlisted.count, exts: unlisted.exts.map((e) => e.ext) })
|
|
246
|
+
}
|
|
247
|
+
return { updated, removed, skipped, failed, errors, total: files.length, overSizeSkipped, unlistedExts: unlisted }
|
|
210
248
|
}
|
|
211
249
|
|
|
212
250
|
/** Record current HEAD as the index anchor (gitSync incremental diff baseline); silently skip non-git repos */
|
|
@@ -313,7 +351,8 @@ export function codeSearchTool(memory) {
|
|
|
313
351
|
return {
|
|
314
352
|
name: "code_search",
|
|
315
353
|
description:
|
|
316
|
-
"Search the project's source code for relevant code. Use this to find functions, classes, or code patterns across the codebase. Supports natural language queries and code snippets. Returns matching code chunks with file paths and line numbers. Prefer doc_search for the intended design (design docs, conventions); code_search for the implementation as written."
|
|
354
|
+
"Search the project's source code for relevant code. Use this to find functions, classes, or code patterns across the codebase. Supports natural language queries and code snippets. Returns matching code chunks with file paths and line numbers. Prefer doc_search for the intended design (design docs, conventions); code_search for the implementation as written. " +
|
|
355
|
+
"For what was said in a session (conversation/chat history), use read_history.",
|
|
317
356
|
parameters: {
|
|
318
357
|
type: "object",
|
|
319
358
|
properties: {
|
|
@@ -337,32 +376,33 @@ export function codeSearchTool(memory) {
|
|
|
337
376
|
* Single-file incremental reindex: called after write/edit/delete, only rebuilds this one path.
|
|
338
377
|
*/
|
|
339
378
|
export async function reindexFile(memory, cwd, absPath) {
|
|
340
|
-
const ext =
|
|
379
|
+
const ext = extensionOf(absPath)
|
|
341
380
|
const rel = relative(cwd, absPath).replaceAll("\\", "/")
|
|
342
381
|
if (rel === ".." || rel.startsWith("../")) return
|
|
343
|
-
|
|
344
|
-
|
|
382
|
+
if (isSkippedRelPath(rel)) return
|
|
383
|
+
// Declared extensions count for the single-file path too (union with built-ins).
|
|
384
|
+
const { code: codeExts, doc: docExts } = indexExtensions(cwd)
|
|
345
385
|
|
|
346
386
|
// skip oversized files (minified bundles, test fixtures, generated code)
|
|
347
|
-
const maxBytes =
|
|
387
|
+
const maxBytes = codeExts.has(ext) ? MAX_CODE_FILE_BYTES : docExts.has(ext) ? MAX_DOC_FILE_BYTES : 0
|
|
348
388
|
if (maxBytes > 0) {
|
|
349
389
|
try { const st = await stat(absPath); if (st.size > maxBytes) return } catch { /* can't stat, proceed */ }
|
|
350
390
|
}
|
|
351
391
|
|
|
352
392
|
let text
|
|
353
393
|
try { text = await readFile(absPath, "utf8") } catch {
|
|
354
|
-
if (
|
|
355
|
-
else if (
|
|
394
|
+
if (codeExts.has(ext)) memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(cwd, rel)
|
|
395
|
+
else if (docExts.has(ext)) memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(cwd, rel)
|
|
356
396
|
return
|
|
357
397
|
}
|
|
358
398
|
const lines = text.split("\n")
|
|
359
399
|
|
|
360
|
-
if (
|
|
400
|
+
if (codeExts.has(ext)) {
|
|
361
401
|
const lang = detectLanguage(absPath)
|
|
362
402
|
let mtimeMs = 0
|
|
363
403
|
try { mtimeMs = Math.floor((await stat(absPath)).mtimeMs) } catch { /* new file */ }
|
|
364
404
|
_upsertCodeFile(memory, cwd, rel, lines, lang, mtimeMs)
|
|
365
|
-
} else if (
|
|
405
|
+
} else if (docExts.has(ext)) {
|
|
366
406
|
let mtimeMs = 0
|
|
367
407
|
try { mtimeMs = Math.floor((await stat(absPath)).mtimeMs) } catch { /* new file */ }
|
|
368
408
|
_upsertDocFile(memory, cwd, rel, lines, mtimeMs)
|