thincoder 0.12.59 → 0.12.61

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/CHANGELOG.md +62 -4
  2. package/README.md +10 -8
  3. package/bin/thincoder.mjs +99 -133
  4. package/package.json +6 -4
  5. package/src/abort-provenance.mjs +116 -0
  6. package/src/acp/bridge.mjs +45 -21
  7. package/src/acp.mjs +6 -1
  8. package/src/advisor/citations.mjs +83 -21
  9. package/src/advisor/compaction.mjs +174 -0
  10. package/src/advisor/loop.mjs +293 -0
  11. package/src/advisor/messages.mjs +59 -137
  12. package/src/advisor/project-context.mjs +194 -0
  13. package/src/advisor/repos.mjs +17 -40
  14. package/src/advisor/run.mjs +156 -359
  15. package/src/advisor/truncate.mjs +57 -0
  16. package/src/advisor.mjs +27 -7
  17. package/src/agent/completion.mjs +17 -11
  18. package/src/agent/dispatch.mjs +145 -27
  19. package/src/agent/helpers.mjs +107 -13
  20. package/src/agent/record-results.mjs +55 -11
  21. package/src/agent/relay-prefix.mjs +39 -0
  22. package/src/agent/run-stages.mjs +242 -0
  23. package/src/agent/setup-reminders.mjs +69 -0
  24. package/src/agent/setup.mjs +107 -127
  25. package/src/agent/spawn-child.mjs +55 -13
  26. package/src/agent-tools/advisor-async.mjs +346 -0
  27. package/src/agent-tools/advisor-settle.mjs +231 -0
  28. package/src/agent-tools/advisor.mjs +167 -116
  29. package/src/agent-tools/async-settle.mjs +191 -0
  30. package/src/agent-tools/batch-segment.mjs +195 -0
  31. package/src/agent-tools/consult.mjs +139 -107
  32. package/src/agent-tools/design-token.mjs +117 -0
  33. package/src/agent-tools/digest-budget.mjs +76 -0
  34. package/src/agent-tools/eng.mjs +24 -29
  35. package/src/agent-tools/escalate-async.mjs +289 -0
  36. package/src/agent-tools/read-history.mjs +166 -32
  37. package/src/agent-tools/recent-changes.mjs +2 -1
  38. package/src/agent-tools/review-streak.mjs +93 -0
  39. package/src/agent-tools/settings.mjs +137 -34
  40. package/src/agent-tools/subagent-actions.mjs +180 -133
  41. package/src/agent-tools/subagent-async.mjs +184 -177
  42. package/src/agent-tools/subagent-panel.mjs +160 -0
  43. package/src/agent-tools/subagent-run.mjs +205 -0
  44. package/src/agent-tools/subagent-scheduler.mjs +100 -27
  45. package/src/agent-tools/subagent-spawn.mjs +453 -0
  46. package/src/agent-tools/subagent.mjs +256 -578
  47. package/src/agent-tools/verify.mjs +119 -292
  48. package/src/agent-tools.mjs +1 -0
  49. package/src/agent.mjs +89 -205
  50. package/src/cli/distill-command.mjs +12 -6
  51. package/src/cli/make-agent.mjs +26 -8
  52. package/src/cli/memory-command.mjs +4 -3
  53. package/src/cli/permission.mjs +2 -2
  54. package/src/cli/setup-wizard.mjs +42 -17
  55. package/src/completions.mjs +114 -0
  56. package/src/config-migrate.mjs +70 -0
  57. package/src/config.mjs +180 -63
  58. package/src/context.mjs +5 -147
  59. package/src/conventions.mjs +223 -0
  60. package/src/crash-reports.mjs +128 -0
  61. package/src/distill.mjs +11 -11
  62. package/src/expand-home.mjs +16 -0
  63. package/src/explore-distill.mjs +155 -0
  64. package/src/generate-title.mjs +1 -1
  65. package/src/hooks.mjs +7 -3
  66. package/src/memory/code-index.mjs +9 -3
  67. package/src/memory/code-sync.mjs +72 -32
  68. package/src/memory/core.mjs +6 -193
  69. package/src/memory/delete.mjs +236 -0
  70. package/src/memory/docs.mjs +68 -54
  71. package/src/memory/file-walk.mjs +109 -0
  72. package/src/memory/schema.mjs +15 -3
  73. package/src/memory.mjs +3 -1
  74. package/src/model-ref.mjs +66 -0
  75. package/src/model-specs.mjs +42 -8
  76. package/src/peer-domains.mjs +265 -0
  77. package/src/peer-instances.mjs +231 -0
  78. package/src/prompt-overlays.mjs +82 -0
  79. package/src/prompts/advisor-design.md +18 -75
  80. package/src/prompts/advisor-round1.md +14 -67
  81. package/src/prompts/advisor-round2.md +15 -51
  82. package/src/prompts/advisor-round3.md +15 -51
  83. package/src/prompts/common.md +115 -0
  84. package/src/prompts/consult-base.md +5 -23
  85. package/src/prompts/discipline-engineering.md +217 -0
  86. package/src/prompts/discipline-normal.md +179 -0
  87. package/src/prompts/persona-coder.md +21 -0
  88. package/src/prompts/persona-eng-coder.md +37 -0
  89. package/src/prompts/persona-eng-designer.md +55 -0
  90. package/src/prompts/persona-engineering.md +54 -0
  91. package/src/prompts/persona-explore.md +15 -0
  92. package/src/prompts/persona-normal.md +27 -0
  93. package/src/prompts/persona-plan.md +26 -0
  94. package/src/provider/anthropic.mjs +4 -4
  95. package/src/provider/core.mjs +18 -98
  96. package/src/provider/errors.mjs +101 -0
  97. package/src/provider/google.mjs +5 -6
  98. package/src/provider/index.mjs +2 -1
  99. package/src/provider/list-models.mjs +93 -0
  100. package/src/provider/rate.mjs +2 -1
  101. package/src/provider/responses.mjs +5 -3
  102. package/src/provider/retry.mjs +8 -45
  103. package/src/provider/sse.mjs +3 -4
  104. package/src/proxy.mjs +9 -14
  105. package/src/session-gc.mjs +214 -0
  106. package/src/session-guard.mjs +47 -0
  107. package/src/session-rename.mjs +38 -0
  108. package/src/session-slots.mjs +188 -60
  109. package/src/session.mjs +104 -124
  110. package/src/token-ttl.mjs +274 -0
  111. package/src/tools/{system.mjs → bash.mjs} +19 -221
  112. package/src/tools/checklist-sync.mjs +181 -0
  113. package/src/tools/checklist.mjs +52 -39
  114. package/src/tools/edit-batch.mjs +109 -10
  115. package/src/tools/edit-diff.mjs +110 -27
  116. package/src/tools/edit.md +17 -12
  117. package/src/tools/execute.mjs +31 -4
  118. package/src/tools/file.mjs +41 -16
  119. package/src/tools/git.md +1 -1
  120. package/src/tools/git.mjs +23 -34
  121. package/src/tools/glob-dialect.mjs +130 -0
  122. package/src/tools/glob.md +3 -3
  123. package/src/tools/grep.md +1 -1
  124. package/src/tools/index.mjs +9 -8
  125. package/src/tools/ops.mjs +188 -3
  126. package/src/tools/patch.mjs +3 -3
  127. package/src/tools/question.md +4 -0
  128. package/src/tools/question.mjs +26 -0
  129. package/src/tools/read.md +1 -2
  130. package/src/tools/read_image.md +1 -1
  131. package/src/tools/search.mjs +236 -0
  132. package/src/tools/shared.mjs +14 -13
  133. package/src/tools/wait_for.md +22 -0
  134. package/src/tui/agent-turn.mjs +36 -228
  135. package/src/tui/ansi.mjs +2 -0
  136. package/src/tui/clipboard.mjs +7 -1
  137. package/src/tui/cmd-advisor.mjs +3 -2
  138. package/src/tui/cmd-config.mjs +142 -30
  139. package/src/tui/cmd-eng.mjs +28 -40
  140. package/src/tui/cmd-exit.mjs +6 -8
  141. package/src/tui/cmd-mcp.mjs +8 -2
  142. package/src/tui/cmd-model.mjs +14 -12
  143. package/src/tui/cmd-new.mjs +3 -2
  144. package/src/tui/cmd-reindex.mjs +7 -0
  145. package/src/tui/cmd-session.mjs +19 -4
  146. package/src/tui/cmd-submodel.mjs +8 -5
  147. package/src/tui/cmd-think.mjs +10 -10
  148. package/src/tui/cmd-undo.mjs +4 -3
  149. package/src/tui/cmd-upgrade.mjs +19 -4
  150. package/src/tui/config-helpers.mjs +28 -16
  151. package/src/tui/distill-cmd.mjs +1 -1
  152. package/src/tui/index.mjs +40 -38
  153. package/src/tui/interaction.mjs +3 -3
  154. package/src/tui/key-handler.mjs +61 -17
  155. package/src/tui/key-modes.mjs +86 -8
  156. package/src/tui/layout.mjs +18 -10
  157. package/src/tui/model-catalog.mjs +89 -0
  158. package/src/tui/model-picker.mjs +498 -0
  159. package/src/tui/mouse.mjs +52 -9
  160. package/src/tui/pickers.mjs +28 -392
  161. package/src/tui/render-frame.mjs +32 -16
  162. package/src/tui/render-loop.mjs +2 -0
  163. package/src/tui/render-segments.mjs +12 -9
  164. package/src/tui/render.mjs +37 -5
  165. package/src/tui/slash-commands.mjs +2 -2
  166. package/src/tui/startup.mjs +4 -0
  167. package/src/tui/subagent-blocks.mjs +106 -295
  168. package/src/tui/subagent-children.mjs +162 -0
  169. package/src/tui/subagent-freeze.mjs +169 -0
  170. package/src/tui/subagent-panel.mjs +24 -31
  171. package/src/tui/suspension-drive.mjs +297 -0
  172. package/src/tui/tool-args.mjs +7 -5
  173. package/src/tui/tool-display.mjs +143 -0
  174. package/src/tui/tool-events.mjs +56 -185
  175. package/src/tui/tui-lifecycle.mjs +46 -4
  176. package/src/tui/update-notice.mjs +4 -0
  177. package/src/tui/wizard.mjs +61 -21
  178. package/src/tui/wrapped-spawn.mjs +38 -0
  179. package/src/prompts/coder.md +0 -56
  180. package/src/prompts/discipline.md +0 -102
  181. package/src/prompts/eng-coder.md +0 -44
  182. package/src/prompts/engineering-sub.md +0 -34
  183. package/src/prompts/engineering.md +0 -340
  184. package/src/prompts/explore.md +0 -21
  185. package/src/prompts/main.md +0 -56
  186. package/src/prompts/methodology-template.md +0 -58
  187. package/src/prompts/plan.md +0 -16
  188. package/src/prompts/system.md +0 -63
  189. package/src/tools/pdf-parse-text.mjs +0 -497
  190. package/src/tools/pdf-parse-xref.mjs +0 -499
  191. package/src/tools/pdf.mjs +0 -155
  192. package/src/tools/read_pdf.md +0 -21
@@ -0,0 +1,155 @@
1
+ /**
2
+ * explore-distill.mjs — End-of-run exploration distillation (AGENT-LOOP §13 +
3
+ * CONTEXT-COMPACTION §5, 2026-08-23). 2026-09-05 module-split: moved verbatim out of
4
+ * context.mjs (524 > 500 hard limit). The main agent's machine line is flooded by inline
5
+ * step-by-step exploration (read/grep/...). At run end we distill THIS run's exploration
6
+ * tool-results into one semantic summary note that replaces them in the machine line,
7
+ * while agent._fullHistory (the human line) stays untouched. VS Code compact.mjs
8
+ * distillation mirrors this module (same-name file in thincoder-vscode/src, cross-repo
9
+ * parity anchors point here).
10
+ */
11
+
12
+ import { chat } from "./provider/index.mjs"
13
+
14
+ /** Read-only knowledge tools counted as "exploration" (execute writes files → never exploration). */
15
+ export const EXPLORE_TOOLS = new Set([
16
+ "read", "grep", "glob", "ls", "code_search", "doc_search", "repo_outline",
17
+ ])
18
+
19
+ /** Summary prompt for turning a burst of exploration results into a semantic summary. */
20
+ export const EXPLORE_SUMMARY_PROMPT = `You are distilling exploration tool results. Summarize the following read-only codebase exploration into a compact semantic summary for the main agent's own context.
21
+
22
+ Requirements:
23
+ - Capture WHAT was discovered, WHERE (which files / directories / symbols), and the KEY CONCLUSIONS — do not list tool calls mechanically
24
+ - Keep actionable facts the main agent needs to continue: code locations, function names, file paths, structure, and open questions the exploration raised
25
+ - Drop raw tool-output noise, repeated lines, and verbatim file dumps — keep only what must be remembered
26
+ - Be honest: mark anything not actually verified as "unverified"; do not present guesses as facts
27
+ - Use bullet points; aim for information completeness, not a hard word limit
28
+
29
+ Exploration log:
30
+ `
31
+
32
+ /** tool_calls name across both stored shapes ({function:{name}} and flat {name}). */
33
+ function toolCallName(tc) {
34
+ return tc?.function?.name ?? tc?.name ?? ""
35
+ }
36
+
37
+ /** Tool that produced a tool-result message (falls back to its owner assistant's tool_call). */
38
+ function toolResultName(msg, ownerToolCalls) {
39
+ if (typeof msg?.name === "string" && msg.name) return msg.name
40
+ const owner = (ownerToolCalls ?? []).find((tc) => tc.id === msg?.tool_call_id)
41
+ return owner ? toolCallName(owner) : ""
42
+ }
43
+
44
+ /**
45
+ * Find the pure-exploration "assistant(tool_calls)→tool…" pair blocks added since `start`.
46
+ * A block is explorable only when EVERY tool call AND every tool result in it is an exploration
47
+ * tool — mixed blocks (read + edit in one turn) stay untouched, or we'd orphan the edit pairing.
48
+ */
49
+ function findExplorationBlocks(history, start) {
50
+ const blocks = []
51
+ let i = start
52
+ while (i < history.length) {
53
+ const m = history[i]
54
+ if (m?.role === "assistant" && Array.isArray(m.tool_calls) && m.tool_calls.length > 0) {
55
+ let j = i + 1
56
+ while (j < history.length && history[j]?.role === "tool") j++
57
+ const toolMsgs = history.slice(i + 1, j)
58
+ const allCallsExplore = m.tool_calls.every((tc) => EXPLORE_TOOLS.has(toolCallName(tc)))
59
+ const allResultsExplore = toolMsgs.length > 0 && toolMsgs.every((t) => EXPLORE_TOOLS.has(toolResultName(t, m.tool_calls)))
60
+ if (allCallsExplore && allResultsExplore) {
61
+ blocks.push({ start: i, end: j, messages: history.slice(i, j), toolCount: toolMsgs.length })
62
+ }
63
+ i = j
64
+ } else {
65
+ i++
66
+ }
67
+ }
68
+ return blocks
69
+ }
70
+
71
+ /** Serialize a batch of exploration messages for the summary LLM (same shape as compaction serialization). */
72
+ function serializeExplorationMessages(messages) {
73
+ const cap = 8000 // exploration results ARE the signal to distill — generous cap (quality-first, N1)
74
+ return messages
75
+ .map((m) => {
76
+ const toolNote = m.tool_calls ? ` [called tools: ${m.tool_calls.map(toolCallName).join(", ")}]` : ""
77
+ let text = ""
78
+ if (typeof m.content === "string") text = m.content
79
+ else if (Array.isArray(m.content)) text = m.content.filter((p) => p?.type === "text").map((p) => p.text ?? "").join(" ")
80
+ return `[${m.role}]${toolNote} ${text.slice(0, cap)}`
81
+ })
82
+ .join("\n")
83
+ }
84
+
85
+ /**
86
+ * Core (shared) distillation: replace this run's pure-exploration pair blocks with a single
87
+ * "[Exploration summary]" note placed where the first block was. Returns a NEW history array,
88
+ * or null when there is nothing to shrink (<3 exploration results / LLM failure). Pairing-safe:
89
+ * whole assistant→tool blocks are removed, so no orphan tool_calls/tool can survive.
90
+ */
91
+ async function distillExplorations(history, start, provider, signal, agent, depth) {
92
+ if (!Array.isArray(history) || history.length - start < 2) return null
93
+ const blocks = findExplorationBlocks(history, start)
94
+ const resultCount = blocks.reduce((n, b) => n + b.toolCount, 0)
95
+ if (resultCount < 3) return null
96
+
97
+ const serialized = blocks.map((b) => serializeExplorationMessages(b.messages)).join("\n")
98
+
99
+ let summary
100
+ try {
101
+ // Silent by design (D11): thinking:null and no onToken/onReasoning — this internal
102
+ // distillation must not stream to the frontend. signal propagates user cancellation.
103
+ const resp = await chat({ ...provider, thinking: null, reasoningEffort: null }, {
104
+ messages: [{ role: "user", content: EXPLORE_SUMMARY_PROMPT + serialized }],
105
+ signal,
106
+ // §18.6 D-TR4:轨迹元数据增补——kind=distill(探索蒸馏面——agent 元数据透出;
107
+ // depth 经 summarizeRunExplorations 参数透传——agent.mjs 主作用域传入)
108
+ logCtx: {
109
+ stage: "distill", child: agent?._logId ?? null, kind: "distill",
110
+ role: agent?._role ?? null, depth: depth ?? null,
111
+ session: agent?._sessionStart ?? null, cwd: agent?.cwd ?? process.cwd(),
112
+ traces: agent?.config?.traces?.enabled !== false,
113
+ },
114
+ })
115
+ summary = resp?.content
116
+ } catch {
117
+ return null // N3: never block the run's return or lose history — original results stay
118
+ }
119
+ if (!summary) return null
120
+
121
+ const drop = new Set()
122
+ for (const b of blocks) for (let k = b.start; k < b.end; k++) drop.add(k)
123
+ const note = { role: "user", content: "[Exploration summary]\n" + summary }
124
+ const next = []
125
+ let inserted = false
126
+ for (let k = 0; k < history.length; k++) {
127
+ if (drop.has(k)) {
128
+ if (!inserted) { next.push(note); inserted = true }
129
+ continue
130
+ }
131
+ next.push(history[k])
132
+ }
133
+ return next
134
+ }
135
+
136
+ /**
137
+ * End-of-run exploration distillation (runAgent's final return). Shrinks the MACHINE line
138
+ * (agent.history) only; agent._fullHistory is never touched. Triggers when this run added ≥3
139
+ * exploration tool results; on LLM failure it silently keeps the original history (N3).
140
+ * The distillation itself is silent and never streams (D11); `callbacks.onDistilled` fires
141
+ * ONLY after the replacement actually lands (never on no-op/failure) — callers persist the
142
+ * compressed session (SEND-STALL-DISTILL §2.3).
143
+ */
144
+ export async function summarizeRunExplorations(agent, callbacks, signal, depth = 0) {
145
+ const next = await distillExplorations(agent.history, agent._runStartHistoryLen ?? 0, agent.provider, signal, agent, depth)
146
+ if (!next) return
147
+ agent.history = next
148
+ // The machine line changed shape — the measured token baseline was for the pre-shrink context.
149
+ // Invalidate so the next compaction check re-estimates instead of over-counting stale history.
150
+ agent._lastPromptTokens = null
151
+ agent._usageAtLen = null
152
+ // The compressed machine line must reach the disk: the run's own save already happened,
153
+ // so without this hook the async distill would leave the session un-compressed on exit.
154
+ callbacks.onDistilled?.()
155
+ }
@@ -46,7 +46,7 @@ export async function generateTitle(userContent, provider) {
46
46
  const url = `${provider.baseURL.replace(/\/+$/, "")}${chatPath}`
47
47
  const opts = {
48
48
  method: "POST",
49
- headers: { "Content-Type": "application/json", Authorization: `Bearer ${provider.apiKey}` },
49
+ headers: { ...(provider.headers ?? {}), "Content-Type": "application/json", Authorization: `Bearer ${provider.apiKey}` }, // 定制头展开(PROVIDER.md §21)——定制头在前、内置头在后:内置头胜出
50
50
  body,
51
51
  signal: AbortSignal.timeout(10000),
52
52
  }
package/src/hooks.mjs CHANGED
@@ -8,7 +8,7 @@
8
8
  * PreToolUse — before each tool call (can block execution)
9
9
  * PostToolUse — after successful tool execution
10
10
  * PostToolUseFailure — after failed tool execution
11
- * Notification generic notification (triggered by agent)
11
+ * Stop main-session run end (no tool name; block is meaningless)
12
12
  *
13
13
  * Each hook: { matcher?, command, args?, timeout?, action? }
14
14
  * matcher: regex against tool name (default: match all)
@@ -29,9 +29,11 @@ export async function runHooks(event, ctx) {
29
29
  if (!hooks?.length) return true
30
30
 
31
31
  for (const hook of hooks) {
32
- if (hook.matcher) {
32
+ // Matcher = tool-name regex: tool events always carry a toolName; no-tool-name
33
+ // events (Stop) ignore it — a configured matcher must not silently never fire.
34
+ if (hook.matcher && ctx.toolName != null) {
33
35
  try {
34
- if (!new RegExp(hook.matcher).test(ctx.toolName ?? "")) continue
36
+ if (!new RegExp(hook.matcher).test(ctx.toolName)) continue
35
37
  } catch { /* invalid regex → skip */ }
36
38
  }
37
39
 
@@ -48,6 +50,8 @@ async function runOneHook(event, hook, ctx) {
48
50
  toolArgs: ctx.toolArgs ?? null,
49
51
  result: ctx.result ?? null,
50
52
  error: ctx.error?.message ?? null,
53
+ // Event-specific fields (Stop: turn / reason) — existing callers pass no extra ⇒ payload unchanged.
54
+ ...(ctx.extra ?? {}),
51
55
  timestamp: new Date().toISOString(),
52
56
  })
53
57
 
@@ -8,14 +8,20 @@ import { segmentCJK, CODE_EXTS, DOC_EXTS, SKIP_DIRS, BIG_FILE_LINES } from "./sc
8
8
  export function detectLanguage(filename) {
9
9
  const ext = filename.slice(filename.lastIndexOf(".")).toLowerCase()
10
10
  const map = {
11
- ".mjs": "javascript", ".js": "javascript", ".jsx": "jsx", ".ts": "typescript", ".tsx": "tsx",
11
+ ".mjs": "javascript", ".js": "javascript", ".cjs": "javascript", ".jsx": "jsx",
12
+ ".ts": "typescript", ".tsx": "tsx", ".mts": "typescript", ".cts": "typescript",
12
13
  ".py": "python", ".rs": "rust", ".go": "go", ".java": "java",
13
14
  ".c": "c", ".h": "c", ".cpp": "cpp", ".hpp": "cpp",
14
- ".rb": "ruby", ".swift": "swift", ".kt": "kotlin",
15
+ ".rb": "ruby", ".swift": "swift", ".kt": "kotlin", ".dart": "dart", ".lua": "lua",
16
+ ".cs": "csharp", ".fs": "fsharp", ".fsx": "fsharp",
17
+ ".clj": "clojure", ".cljs": "clojure", ".ex": "elixir", ".exs": "elixir",
18
+ ".erl": "erlang", ".hrl": "erlang", ".scala": "scala", ".groovy": "groovy",
19
+ ".pl": "perl", ".pm": "perl", ".r": "r", ".jl": "julia", ".zig": "zig",
20
+ ".ps1": "powershell", ".proto": "protobuf", ".graphql": "graphql", ".tf": "terraform", ".hcl": "hcl",
15
21
  ".sh": "bash", ".bash": "bash", ".sql": "sql",
16
22
  ".yaml": "yaml", ".yml": "yaml", ".toml": "toml", ".json": "json",
17
23
  ".css": "css", ".html": "html", ".vue": "vue", ".svelte": "svelte",
18
- ".md": "markdown", ".mdc": "markdown",
24
+ ".md": "markdown", ".mdc": "markdown", ".mdx": "markdown", ".org": "org", ".wiki": "wiki", ".tex": "tex",
19
25
  }
20
26
  return map[ext] ?? ext.slice(1)
21
27
  }
@@ -4,13 +4,30 @@
4
4
  import { readFile, stat } from "node:fs/promises"
5
5
  import { join, relative } from "node:path"
6
6
  import { embed, cosine, toBlob, fromBlob } from "../embedding.mjs"
7
- import { CODE_EXTS, DOC_EXTS, SKIP_DIRS, MAX_CODE_FILE_BYTES, MAX_DOC_FILE_BYTES } from "./schema.mjs"
7
+ import { CODE_EXTS, DOC_EXTS, MAX_CODE_FILE_BYTES, MAX_DOC_FILE_BYTES } from "./schema.mjs"
8
8
  import { buildFtsQuery, ensureEmbeddings, EMBED_TEXT_MAX_LEN } from "./core.mjs"
9
9
  import { detectLanguage, _upsertCodeFile, _upsertDocFile, yieldTick } from "./code-index.mjs"
10
+ import { walkProjectFiles, isSkippedRelPath, extensionOf, createUnlistedTally, MAX_WALK_FILES } from "./file-walk.mjs"
11
+ import { loadConventions } from "../conventions.mjs"
12
+ import { logEvent } from "../log.mjs"
10
13
 
11
14
  const DIFF_FULL_SYNC_THRESHOLD = 200
12
15
  const CODE_EMBED_BATCH = 64
13
16
 
17
+ /**
18
+ * Index extension sets = built-in tables ∪ project declaration
19
+ * (`.thincoder/conventions.json` → index.codeExtensions / index.docExtensions;
20
+ * PORTABILITY PO-9/§3.7 — a project may declare extensions this product does not
21
+ * ship a default for). Declaration only ADDS (union), never removes.
22
+ */
23
+ export function indexExtensions(dir) {
24
+ const conv = loadConventions(dir)
25
+ return {
26
+ code: new Set([...CODE_EXTS, ...conv.index.codeExtensions]),
27
+ doc: new Set([...DOC_EXTS, ...conv.index.docExtensions]),
28
+ }
29
+ }
30
+
14
31
  /**
15
32
  * git-driven incremental indexing: use git diff to find files changed since
16
33
  * the last index, and only rebuild FTS5 chunks for those files (vectors are untouched).
@@ -19,6 +36,7 @@ const CODE_EMBED_BATCH = 64
19
36
  */
20
37
  export async function gitSync(memory, dir, { onProgress } = {}) {
21
38
  const { execFile: _execFile } = await import("node:child_process")
39
+ const { code: codeExts, doc: docExts } = indexExtensions(dir)
22
40
  const gitRun = (args) => new Promise((resolve, reject) => {
23
41
  _execFile("git", args, { cwd: dir, encoding: "utf8", timeout: 10000, windowsHide: true }, (err, stdout) => {
24
42
  if (err) reject(err); else resolve(stdout)
@@ -55,17 +73,16 @@ export async function gitSync(memory, dir, { onProgress } = {}) {
55
73
  for (let i = 0; i < diffOut.length; i++) {
56
74
  const rel = diffOut[i].replaceAll("\\", "/")
57
75
  const abs = join(dir, rel)
58
- const ext = rel.slice(rel.lastIndexOf(".")).toLowerCase()
76
+ const ext = extensionOf(rel)
59
77
 
60
- const pathDirs = rel.split("/")
61
- if (pathDirs.some((d) => SKIP_DIRS.has(d) || d.startsWith("."))) continue
78
+ if (isSkippedRelPath(rel)) continue
62
79
 
63
- if (!CODE_EXTS.has(ext) && !DOC_EXTS.has(ext)) { skipped++; continue }
80
+ if (!codeExts.has(ext) && !docExts.has(ext)) { skipped++; continue }
64
81
 
65
82
  try {
66
83
  const text = await readFile(abs, "utf8")
67
84
  const lines = text.split("\n")
68
- if (CODE_EXTS.has(ext)) {
85
+ if (codeExts.has(ext)) {
69
86
  const lang = detectLanguage(abs)
70
87
  let mtimeMs = 0
71
88
  try { mtimeMs = Math.floor((await stat(abs)).mtimeMs) } catch { /* new file */ }
@@ -79,7 +96,7 @@ export async function gitSync(memory, dir, { onProgress } = {}) {
79
96
  } catch (e) {
80
97
  const isDeleted = e.code === "ENOENT"
81
98
  if (isDeleted) {
82
- if (CODE_EXTS.has(ext)) memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(dir, rel)
99
+ if (codeExts.has(ext)) memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(dir, rel)
83
100
  else memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(dir, rel)
84
101
  removed++
85
102
  } else {
@@ -104,16 +121,27 @@ export async function gitSync(memory, dir, { onProgress } = {}) {
104
121
 
105
122
  /**
106
123
  * List project files matching the given extensions.
107
- * Only indexes git repos if dir isn't inside a git worktree, returns [].
108
- * Uses `git ls-files --cached --others --exclude-standard` to get the file list
109
- * (tracked + untracked-not-ignored, respecting .gitignore).
110
- * Returns an array of { abs, rel } pairs (abs = full path, rel = path relative to dir).
124
+ * Preferred source = git (`git ls-files --cached --others --exclude-standard`:
125
+ * tracked + untracked-not-ignored, .gitignore respected). Projects WITHOUT git
126
+ * fall back to a filesystem walk (PORTABILITY FR15/P8) — before, the index was
127
+ * silently empty there. The walk cannot honour .gitignore (no git no such
128
+ * concept) and skips the shared SKIP_DIRS/dot-directory set instead.
129
+ * Returns { entries, unlisted }: `entries` = { abs, rel } pairs (rel relative to
130
+ * dir); `unlisted` = files whose extension is in NO index list (count + sample),
131
+ * the visible signal behind "my .xyz files are not searchable" (PORTABILITY PO-9).
132
+ * `truncated` = the non-git walk hit its file cap (capped trees are logged, never
133
+ * silently indexed-partial). opts.maxFiles = the walk cap (test seam).
111
134
  */
112
- export async function listProjectFiles(dir, exts) {
135
+ export async function listProjectFiles(dir, exts, { maxFiles } = {}) {
113
136
  const { execFile: _execFile } = await import("node:child_process")
114
137
  const { join: joinPath } = await import("node:path")
138
+ const { code, doc } = indexExtensions(dir)
139
+ const tally = createUnlistedTally(new Set([...code, ...doc]))
115
140
 
116
- // Only index git repos — if this isn't one, return empty
141
+ const files = []
142
+ const finish = (truncated = false) => ({ entries: files, unlisted: tally.result(), truncated })
143
+
144
+ // Git listing when available; a non-git project falls through to the walk.
117
145
  let gitTop
118
146
  try {
119
147
  gitTop = (await new Promise((resolve, reject) => {
@@ -121,10 +149,17 @@ export async function listProjectFiles(dir, exts) {
121
149
  (err, stdout) => { if (err) reject(err); else resolve(stdout.trim()) })
122
150
  })).replace(/\\/g, "/")
123
151
  } catch {
124
- return [] // not a git repo → nothing to index
152
+ // Not a git repo → filesystem walk (FR15: the index must still be usable).
153
+ const walked = await walkProjectFiles(dir, exts, { knownExts: new Set([...code, ...doc]), ...(maxFiles !== undefined ? { maxFiles } : {}) })
154
+ files.push(...walked.files)
155
+ // The walk's cap must stay visible end-to-end (file-walk.mjs: "never silently
156
+ // dropped") — a capped listing says so in the log, not only in a discarded flag.
157
+ if (walked.truncated) {
158
+ logEvent("index:truncated", { dir, files: walked.files.length, maxFiles: maxFiles ?? MAX_WALK_FILES })
159
+ }
160
+ return { entries: files, unlisted: walked.unlisted, truncated: walked.truncated }
125
161
  }
126
162
 
127
- const files = []
128
163
  try {
129
164
  const raw = await new Promise((resolve, reject) => {
130
165
  _execFile("git", ["ls-files", "--cached", "--others", "--exclude-standard"],
@@ -134,16 +169,15 @@ export async function listProjectFiles(dir, exts) {
134
169
  for (const line of raw.trim().split("\n")) {
135
170
  const p = line.trim()
136
171
  if (!p) continue
137
- const ext = p.slice(p.lastIndexOf(".")).toLowerCase()
138
- if (!exts.has(ext)) continue
139
- const abs = joinPath(dir, p)
140
172
  const rel = p.replace(/\\/g, "/")
141
- if (rel.split("/").some((seg) => SKIP_DIRS.has(seg) || seg.startsWith("."))) continue
142
- files.push({ abs, rel })
173
+ if (isSkippedRelPath(rel)) continue
174
+ const ext = extensionOf(rel)
175
+ if (!ext || !exts.has(ext)) { tally.note(rel); continue }
176
+ files.push({ abs: joinPath(dir, rel), rel })
143
177
  }
144
178
  } catch { /* ls-files failed */ }
145
179
 
146
- return files
180
+ return finish()
147
181
  }
148
182
 
149
183
 
@@ -152,7 +186,8 @@ export async function listProjectFiles(dir, exts) {
152
186
  * Incremental by mtime — only rebuilds chunks for files that have changed.
153
187
  */
154
188
  export async function codeSync(memory, dir, { onProgress } = {}) {
155
- const entries = await listProjectFiles(dir, CODE_EXTS)
189
+ const { code: exts } = indexExtensions(dir)
190
+ const { entries, unlisted } = await listProjectFiles(dir, exts)
156
191
  const files = [] // { abs, rel, mtimeMs }
157
192
  let overSizeSkipped = 0
158
193
  for (const { abs, rel } of entries) {
@@ -206,7 +241,10 @@ export async function codeSync(memory, dir, { onProgress } = {}) {
206
241
 
207
242
  onProgress?.({ phase: "done", total: files.length, updated, removed, skipped, failed, overSizeSkipped })
208
243
  markIndexedCommit(memory, dir)
209
- return { updated, removed, skipped, failed, errors, total: files.length, overSizeSkipped }
244
+ if (unlisted.count > 0) {
245
+ logEvent("index:unlisted", { dir, kind: "code", count: unlisted.count, exts: unlisted.exts.map((e) => e.ext) })
246
+ }
247
+ return { updated, removed, skipped, failed, errors, total: files.length, overSizeSkipped, unlistedExts: unlisted }
210
248
  }
211
249
 
212
250
  /** Record current HEAD as the index anchor (gitSync incremental diff baseline); silently skip non-git repos */
@@ -313,7 +351,8 @@ export function codeSearchTool(memory) {
313
351
  return {
314
352
  name: "code_search",
315
353
  description:
316
- "Search the project's source code for relevant code. Use this to find functions, classes, or code patterns across the codebase. Supports natural language queries and code snippets. Returns matching code chunks with file paths and line numbers. Prefer doc_search for the intended design (design docs, conventions); code_search for the implementation as written.",
354
+ "Search the project's source code for relevant code. Use this to find functions, classes, or code patterns across the codebase. Supports natural language queries and code snippets. Returns matching code chunks with file paths and line numbers. Prefer doc_search for the intended design (design docs, conventions); code_search for the implementation as written. " +
355
+ "For what was said in a session (conversation/chat history), use read_history.",
317
356
  parameters: {
318
357
  type: "object",
319
358
  properties: {
@@ -337,32 +376,33 @@ export function codeSearchTool(memory) {
337
376
  * Single-file incremental reindex: called after write/edit/delete, only rebuilds this one path.
338
377
  */
339
378
  export async function reindexFile(memory, cwd, absPath) {
340
- const ext = absPath.slice(absPath.lastIndexOf(".")).toLowerCase()
379
+ const ext = extensionOf(absPath)
341
380
  const rel = relative(cwd, absPath).replaceAll("\\", "/")
342
381
  if (rel === ".." || rel.startsWith("../")) return
343
- const dirs = rel.split("/").slice(0, -1)
344
- if (dirs.some((d) => SKIP_DIRS.has(d) || d.startsWith("."))) return
382
+ if (isSkippedRelPath(rel)) return
383
+ // Declared extensions count for the single-file path too (union with built-ins).
384
+ const { code: codeExts, doc: docExts } = indexExtensions(cwd)
345
385
 
346
386
  // skip oversized files (minified bundles, test fixtures, generated code)
347
- const maxBytes = CODE_EXTS.has(ext) ? MAX_CODE_FILE_BYTES : DOC_EXTS.has(ext) ? MAX_DOC_FILE_BYTES : 0
387
+ const maxBytes = codeExts.has(ext) ? MAX_CODE_FILE_BYTES : docExts.has(ext) ? MAX_DOC_FILE_BYTES : 0
348
388
  if (maxBytes > 0) {
349
389
  try { const st = await stat(absPath); if (st.size > maxBytes) return } catch { /* can't stat, proceed */ }
350
390
  }
351
391
 
352
392
  let text
353
393
  try { text = await readFile(absPath, "utf8") } catch {
354
- if (CODE_EXTS.has(ext)) memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(cwd, rel)
355
- else if (DOC_EXTS.has(ext)) memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(cwd, rel)
394
+ if (codeExts.has(ext)) memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(cwd, rel)
395
+ else if (docExts.has(ext)) memory.db.prepare(`DELETE FROM doc_chunks WHERE origin = ? AND path = ?`).run(cwd, rel)
356
396
  return
357
397
  }
358
398
  const lines = text.split("\n")
359
399
 
360
- if (CODE_EXTS.has(ext)) {
400
+ if (codeExts.has(ext)) {
361
401
  const lang = detectLanguage(absPath)
362
402
  let mtimeMs = 0
363
403
  try { mtimeMs = Math.floor((await stat(absPath)).mtimeMs) } catch { /* new file */ }
364
404
  _upsertCodeFile(memory, cwd, rel, lines, lang, mtimeMs)
365
- } else if (DOC_EXTS.has(ext)) {
405
+ } else if (docExts.has(ext)) {
366
406
  let mtimeMs = 0
367
407
  try { mtimeMs = Math.floor((await stat(absPath)).mtimeMs) } catch { /* new file */ }
368
408
  _upsertDocFile(memory, cwd, rel, lines, mtimeMs)