thincoder 0.8.2 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/bin/thincoder.mjs +25 -25
- package/package.json +1 -1
- package/src/agent/dispatch.mjs +17 -12
- package/src/agent/helpers.mjs +22 -11
- package/src/agent/setup.mjs +32 -10
- package/src/agent-tools/goal.mjs +12 -11
- package/src/agent-tools/plan.mjs +5 -6
- package/src/agent-tools/recent-changes.mjs +3 -3
- package/src/agent-tools/skill.mjs +6 -6
- package/src/agent-tools/subagent.mjs +19 -18
- package/src/agent-tools/task.mjs +7 -26
- package/src/agent-tools/verify.mjs +21 -19
- package/src/agent-tools.mjs +3 -3
- package/src/agent.mjs +82 -92
- package/src/cli/distill-command.mjs +2 -2
- package/src/cli/make-agent.mjs +9 -9
- package/src/cli/memory-command.mjs +1 -1
- package/src/cli/permission.mjs +3 -3
- package/src/cli/setup-wizard.mjs +15 -15
- package/src/config.mjs +56 -53
- package/src/context.mjs +59 -61
- package/src/distill.mjs +35 -35
- package/src/embedding.mjs +17 -17
- package/src/git/checkpoint.mjs +211 -38
- package/src/git/gitmem.mjs +21 -20
- package/src/markdown.mjs +13 -13
- package/src/mcp/helpers.mjs +6 -1
- package/src/mcp/transport-http.mjs +2 -1
- package/src/mcp/transport-stdio.mjs +3 -2
- package/src/mcp/transport-ws.mjs +3 -2
- package/src/mcp.mjs +5 -2
- package/src/memory/code-index.mjs +16 -14
- package/src/memory/code-sync.mjs +36 -26
- package/src/memory/core.mjs +42 -35
- package/src/memory/docs.mjs +24 -15
- package/src/memory/schema.mjs +28 -27
- package/src/memory.mjs +2 -2
- package/src/prompts/coder.md +0 -6
- package/src/prompts/discipline.md +9 -0
- package/src/prompts/main.md +24 -24
- package/src/prompts/system.md +21 -16
- package/src/provider/core.mjs +11 -6
- package/src/provider/index.mjs +2 -2
- package/src/provider/rate.mjs +11 -11
- package/src/session.mjs +73 -42
- package/src/skills.mjs +17 -17
- package/src/tools/checkpoint.md +6 -2
- package/src/tools/file.mjs +20 -14
- package/src/tools/git.mjs +83 -12
- package/src/tools/index.mjs +1 -1
- package/src/tools/ls.md +1 -1
- package/src/tools/patch.mjs +20 -18
- package/src/tools/repomap-parse.mjs +17 -17
- package/src/tools/repomap.mjs +29 -29
- package/src/tools/shared.mjs +55 -28
- package/src/tools/system.mjs +163 -118
- package/src/tools/web.mjs +6 -6
- package/src/tui/agent-turn.mjs +82 -26
- package/src/tui/ansi.mjs +5 -3
- package/src/tui/clipboard.mjs +2 -2
- package/src/tui/cmd-auto.mjs +3 -12
- package/src/tui/cmd-clear.mjs +1 -1
- package/src/tui/cmd-config.mjs +108 -23
- package/src/tui/cmd-exit.mjs +2 -2
- package/src/tui/cmd-extract.mjs +11 -3
- package/src/tui/cmd-goal.mjs +3 -12
- package/src/tui/cmd-help.mjs +2 -2
- package/src/tui/cmd-init.mjs +3 -3
- package/src/tui/cmd-mcp.mjs +38 -25
- package/src/tui/cmd-model.mjs +1 -1
- package/src/tui/cmd-new.mjs +33 -13
- package/src/tui/cmd-plan.mjs +3 -12
- package/src/tui/cmd-reindex.mjs +2 -2
- package/src/tui/cmd-restore.mjs +1 -1
- package/src/tui/cmd-session.mjs +1 -1
- package/src/tui/cmd-skills.mjs +1 -1
- package/src/tui/cmd-think.mjs +4 -11
- package/src/tui/config-helpers.mjs +6 -6
- package/src/tui/distill-cmd.mjs +4 -3
- package/src/tui/index.mjs +131 -72
- package/src/tui/interaction.mjs +13 -12
- package/src/tui/key-handler.mjs +34 -26
- package/src/tui/layout.mjs +22 -17
- package/src/tui/pickers.mjs +19 -19
- package/src/tui/render-frame.mjs +37 -11
- package/src/tui/render.mjs +25 -24
- package/src/tui/slash-commands.mjs +22 -22
- package/src/tui/startup.mjs +15 -14
- package/src/tui/wizard.mjs +11 -11
- package/src/tui.mjs +2 -2
- package/bin/thincoder.js +0 -4
- package/src/tools/bash.mjs +0 -144
- package/src/tools/glob.mjs +0 -51
- package/src/tools/grep.mjs +0 -100
- package/src/tools/ls.mjs +0 -36
- package/src/tools.mjs +0 -2
- package/src/tui-render.mjs +0 -4
package/src/context.mjs
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* context.mjs —
|
|
3
|
-
* token
|
|
4
|
-
*
|
|
5
|
-
*
|
|
2
|
+
* context.mjs — Context management and compaction
|
|
3
|
+
* When no measured token count is available, use estimation as fallback (ASCII/4 + non-ASCII/1, no tokenizer dependency).
|
|
4
|
+
* When a measured value exists (response usage.prompt_tokens), trust it — estimation underestimates CJK by 3-4x and relying solely on it may never trigger compaction.
|
|
5
|
+
* Compaction strategy: keep earliest 2 + latest N messages, summarize the middle into one via LLM (inspired by kimi-code, simplified).
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import { chat } from "./provider/index.mjs"
|
|
9
9
|
import { estimateText } from "./provider/rate.mjs"
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
const IMAGE_TOKEN_ESTIMATE = 256 // rough estimate for image placeholder tokens
|
|
12
|
+
|
|
13
|
+
/** Rough token count for a list of messages (body + reasoning + tool_calls params) */
|
|
12
14
|
export function estimateTokens(messages) {
|
|
13
15
|
let tokens = 0
|
|
14
16
|
for (const m of messages) {
|
|
@@ -16,7 +18,7 @@ export function estimateTokens(messages) {
|
|
|
16
18
|
else if (Array.isArray(m.content)) {
|
|
17
19
|
for (const part of m.content) {
|
|
18
20
|
if (part.type === "text") tokens += estimateText(part.text)
|
|
19
|
-
else if (part.type === "image_url") tokens +=
|
|
21
|
+
else if (part.type === "image_url") tokens += IMAGE_TOKEN_ESTIMATE
|
|
20
22
|
}
|
|
21
23
|
}
|
|
22
24
|
if (typeof m.reasoning_content === "string") tokens += estimateText(m.reasoning_content)
|
|
@@ -27,59 +29,58 @@ export function estimateTokens(messages) {
|
|
|
27
29
|
return tokens
|
|
28
30
|
}
|
|
29
31
|
|
|
30
|
-
const KEEP_HEAD = 2 //
|
|
31
|
-
const KEEP_TAIL = 10 //
|
|
32
|
+
const KEEP_HEAD = 2 // Keep the earliest user intent — must not lose it
|
|
33
|
+
const KEEP_TAIL = 10 // Keep the most recent work context — must not lose it
|
|
32
34
|
|
|
33
|
-
const SUMMARIZE_PROMPT =
|
|
34
|
-
|
|
35
|
-
-
|
|
36
|
-
-
|
|
37
|
-
-
|
|
38
|
-
-
|
|
39
|
-
-
|
|
40
|
-
-
|
|
35
|
+
const SUMMARIZE_PROMPT = `You are a conversation compressor. Summarize the following agent work log into a compact summary for use as context in the ongoing conversation.
|
|
36
|
+
Requirements:
|
|
37
|
+
- Write in first person, present tense — these are "my" handover notes, continuing my own train of thought
|
|
38
|
+
- Most important: preserve design decisions and their reasons — architecture choices, API contracts, naming conventions, trade-off rationale. These are the anchors the subsequent code must not deviate from
|
|
39
|
+
- Keep: the user's original request, files modified and why, unresolved issues, next steps
|
|
40
|
+
- Drop: pleasantries, repetition, fine-grained tool output details
|
|
41
|
+
- Honestly mark uncertain items: anything not actually verified must say "unverified"; do not present guesses as facts
|
|
42
|
+
- Use bullet-point output; aim for information completeness, not a hard word limit (old 500-char cap is deprecated; in a 1M-context era, err on the long side)
|
|
41
43
|
|
|
42
|
-
|
|
44
|
+
Work log:
|
|
43
45
|
`
|
|
44
46
|
|
|
45
|
-
/**
|
|
47
|
+
/** Context prefix after compaction, informing the agent what happened */
|
|
46
48
|
const COMPACTION_PREFIX =
|
|
47
49
|
"[Context was automatically compacted. Below is a summary of earlier work. " +
|
|
48
50
|
"Treat it as notes, not proof — trust its conclusions (don't redo what it reports as done) " +
|
|
49
|
-
"but re-verify transient state
|
|
50
|
-
"Design decisions made earlier may be summarized — if you recall a decision that is missing from the summary, check memory_search or re-examine the code.]\n\n"
|
|
51
|
+
"but re-verify transient state with tools. Check memory_search for any missing decisions.]\n\n"
|
|
51
52
|
|
|
52
|
-
/**
|
|
53
|
+
/** After this many consecutive compaction summary failures, degrade to deterministic truncation (losing info is better than task-killing 400 errors) */
|
|
53
54
|
export const COMPRESS_FAILURE_LIMIT = 3
|
|
54
55
|
|
|
55
|
-
/**
|
|
56
|
+
/** Task re-injection reminder prefix (after compaction, clear old versions from history first for a single source of truth) */
|
|
56
57
|
const TASK_REINJECT_PREFIX = "[System reminder: your current task list after compaction:"
|
|
57
58
|
|
|
58
|
-
/**
|
|
59
|
+
/** Truncation fallback note (used when the summary LLM fails repeatedly; no LLM call) */
|
|
59
60
|
const FALLBACK_NOTE =
|
|
60
61
|
"[Context was truncated after repeated summarization failures. " +
|
|
61
62
|
"The middle portion of earlier work was dropped WITHOUT a summary. " +
|
|
62
63
|
"Re-verify any state you need with tools before relying on it.]\n\n"
|
|
63
64
|
|
|
64
65
|
/**
|
|
65
|
-
*
|
|
66
|
-
* head
|
|
67
|
-
*
|
|
68
|
-
* tail
|
|
69
|
-
*
|
|
66
|
+
* Split history into head / middle (to be summarized) / tail; return null if no middle to compress.
|
|
67
|
+
* The head boundary must avoid orphan tool_calls: when an assistant message has tool_calls, all its tool responses must stay in head,
|
|
68
|
+
* otherwise compressing them to plain text violates the protocol (tool_calls must be followed by tool messages).
|
|
69
|
+
* The tail boundary must include any assistant whose tool results are in the tail — if the assistant is in the middle,
|
|
70
|
+
* the summary swallows it, leaving orphan tool results → protocol 400.
|
|
70
71
|
*/
|
|
71
72
|
function splitHistory(history) {
|
|
72
73
|
if (history.length <= KEEP_HEAD + KEEP_TAIL + 1) return null
|
|
73
74
|
let headEnd = KEEP_HEAD
|
|
74
|
-
// head
|
|
75
|
-
//
|
|
75
|
+
// head must not end with dangling tool_calls: when assistant declares tool_calls, all its tool results must stay in head.
|
|
76
|
+
// Parallel calls: one assistant followed by multiple tool messages — accepting only one still causes 400, must collect all
|
|
76
77
|
if (history[headEnd - 1]?.role === "assistant" && history[headEnd - 1].tool_calls?.length) {
|
|
77
78
|
while (headEnd < history.length && history[headEnd].role === "tool") headEnd++
|
|
78
79
|
}
|
|
79
80
|
let tailStart = history.length - KEEP_TAIL
|
|
80
81
|
|
|
81
|
-
// tail
|
|
82
|
-
//
|
|
82
|
+
// Tool messages in the tail region whose assistant tool_calls are in the middle: the summary would swallow the assistant,
|
|
83
|
+
// leaving orphan tool results → protocol 400. Collect tool_call_ids from the tail, find their owner assistants and pull them into tail
|
|
83
84
|
const tailToolIds = new Set()
|
|
84
85
|
for (let i = tailStart; i < history.length; i++) {
|
|
85
86
|
if (history[i].role === "tool") tailToolIds.add(history[i].tool_call_id)
|
|
@@ -100,7 +101,7 @@ function splitHistory(history) {
|
|
|
100
101
|
return { headEnd, tailStart }
|
|
101
102
|
}
|
|
102
103
|
|
|
103
|
-
/**
|
|
104
|
+
/** Replace middle with a note, then re-inject task/plan state (shared by LLM summary and truncation fallback) */
|
|
104
105
|
function applyCompression(agent, headEnd, tailStart, note) {
|
|
105
106
|
const head = agent.history.slice(0, headEnd)
|
|
106
107
|
const tail = agent.history.slice(tailStart)
|
|
@@ -110,13 +111,13 @@ function applyCompression(agent, headEnd, tailStart, note) {
|
|
|
110
111
|
{ role: "assistant", content: "Understood. I'll continue from these notes, re-verifying anything transient." },
|
|
111
112
|
...tail,
|
|
112
113
|
]
|
|
113
|
-
//
|
|
114
|
+
// Measured token baseline is invalidated along with old history (prompt_tokens were for pre-compaction context), fall back to estimation until next response
|
|
114
115
|
agent._lastPromptTokens = null
|
|
115
116
|
agent._usageAtLen = null
|
|
116
117
|
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
//
|
|
118
|
+
// After compaction, re-inject the task list (the agent needs to know what it was doing).
|
|
119
|
+
// Single source of truth: first remove any stale re-injections from the tail, then inject the latest version —
|
|
120
|
+
// no longer embedded in the summary body (would duplicate and grow stale)
|
|
120
121
|
agent.history = agent.history.filter(
|
|
121
122
|
(m) => !(m.role === "user" && typeof m.content === "string" && m.content.startsWith(TASK_REINJECT_PREFIX))
|
|
122
123
|
)
|
|
@@ -128,11 +129,7 @@ function applyCompression(agent, headEnd, tailStart, note) {
|
|
|
128
129
|
})
|
|
129
130
|
}
|
|
130
131
|
|
|
131
|
-
//
|
|
132
|
-
agent._turnsSinceTaskUpdate = 0
|
|
133
|
-
agent._turnsInPlanMode = 0
|
|
134
|
-
|
|
135
|
-
// plan mode 中压缩:重新注入 plan 模式引导
|
|
132
|
+
// Plan mode compaction: re-inject plan mode guidance
|
|
136
133
|
if (agent.planMode) {
|
|
137
134
|
agent.history.push({
|
|
138
135
|
role: "user",
|
|
@@ -142,14 +139,14 @@ function applyCompression(agent, headEnd, tailStart, note) {
|
|
|
142
139
|
}
|
|
143
140
|
|
|
144
141
|
/**
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
142
|
+
* If history exceeds threshold, compact it. Returns whether compaction happened.
|
|
143
|
+
* Only called at safe points in the loop (history ends with user or tool message — a complete exchange boundary).
|
|
144
|
+
* Automatically re-injects task list state after compaction.
|
|
148
145
|
*/
|
|
149
146
|
export async function compressIfNeeded(agent, threshold) {
|
|
150
147
|
const history = agent.history
|
|
151
|
-
//
|
|
152
|
-
//
|
|
148
|
+
// Prefer the real baseline: the last response's prompt_tokens is the measured value for the full context (system+tools+history).
|
|
149
|
+
// Subsequent appended messages use estimation as increment; when no measured value exists (first turn / after restore / right after compaction), fall back to pure estimation
|
|
153
150
|
const tokens =
|
|
154
151
|
agent._lastPromptTokens != null
|
|
155
152
|
? agent._lastPromptTokens + estimateTokens(history.slice(agent._usageAtLen ?? history.length))
|
|
@@ -158,23 +155,24 @@ export async function compressIfNeeded(agent, threshold) {
|
|
|
158
155
|
|
|
159
156
|
const split = splitHistory(history)
|
|
160
157
|
if (!split) {
|
|
161
|
-
//
|
|
162
|
-
//
|
|
158
|
+
// History is too short (≤13 messages) to find a middle section, but tokens exceed threshold — typically a single giant message
|
|
159
|
+
// (large paste / huge injection). When summarization has no room, degrade to deterministic shrinking to ensure context always reduces
|
|
163
160
|
return shrinkOversized(agent)
|
|
164
161
|
}
|
|
165
162
|
|
|
166
163
|
const middle = history.slice(split.headEnd, split.tailStart)
|
|
167
164
|
const serialized = middle
|
|
168
165
|
.map((m) => {
|
|
169
|
-
const toolNote = m.tool_calls ? ` [
|
|
170
|
-
// user
|
|
166
|
+
const toolNote = m.tool_calls ? ` [called tools: ${m.tool_calls.map((t) => t.function.name).join(", ")}]` : ""
|
|
167
|
+
// user messages get a wider cap (8000): cutting off a long user-pasted requirement loses original intent; tool/assistant capped at 2000 is enough
|
|
171
168
|
const cap = m.role === "user" ? 8000 : 2000
|
|
172
169
|
const content = typeof m.content === "string" ? m.content.slice(0, cap) : ""
|
|
173
170
|
return `[${m.role}]${toolNote} ${content}`
|
|
174
171
|
})
|
|
175
172
|
.join("\n")
|
|
176
173
|
|
|
177
|
-
|
|
174
|
+
// The summary is a plain-text task, no reasoning needed — passing thinking to the compaction provider wastes tokens
|
|
175
|
+
const summary = await chat({ ...agent.provider, thinking: null, reasoningEffort: null }, {
|
|
178
176
|
messages: [{ role: "user", content: SUMMARIZE_PROMPT + serialized }],
|
|
179
177
|
})
|
|
180
178
|
|
|
@@ -183,8 +181,8 @@ export async function compressIfNeeded(agent, threshold) {
|
|
|
183
181
|
}
|
|
184
182
|
|
|
185
183
|
/**
|
|
186
|
-
*
|
|
187
|
-
*
|
|
184
|
+
* Deterministic truncation fallback: called when the summary LLM fails repeatedly, no network call.
|
|
185
|
+
* Drops the middle so the task can continue. Returns whether truncation happened.
|
|
188
186
|
*/
|
|
189
187
|
export function compressFallback(agent) {
|
|
190
188
|
const split = splitHistory(agent.history)
|
|
@@ -193,21 +191,21 @@ export function compressFallback(agent) {
|
|
|
193
191
|
return true
|
|
194
192
|
}
|
|
195
193
|
|
|
196
|
-
/**
|
|
194
|
+
/** Hard truncation limit for a single message body: when exceeded and the splitter can't find a middle section, truncate to a stub (prevents one giant message from blocking compaction) */
|
|
197
195
|
const OVERSIZE_CONTENT_LIMIT = 8_000
|
|
198
196
|
|
|
199
197
|
/**
|
|
200
|
-
*
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
*
|
|
198
|
+
* Deterministic shrinking: last resort when splitHistory can't find a middle section (history too short) but threshold is exceeded. No LLM call.
|
|
199
|
+
* Truncates user/tool message bodies exceeding OVERSIZE_CONTENT_LIMIT to a stub (keeps head + tail);
|
|
200
|
+
* does not touch reasoning_content (DeepSeek/Kimi echo protocol) or tool_calls pairing structure — no protocol 400 risk.
|
|
201
|
+
* Only called after compressIfNeeded determines threshold is exceeded. Returns whether any message was truncated.
|
|
204
202
|
*/
|
|
205
203
|
export function shrinkOversized(agent, limit = OVERSIZE_CONTENT_LIMIT) {
|
|
206
204
|
let shrunk = false
|
|
207
205
|
for (const m of agent.history) {
|
|
208
206
|
if ((m.role !== "user" && m.role !== "tool") || typeof m.content !== "string") continue
|
|
209
207
|
if (m.content.length <= limit) continue
|
|
210
|
-
//
|
|
208
|
+
// Truncate keeping head + tail, insert stub in between; keepHead/keepTail proportional but not exceeding 50%/25% of limit
|
|
211
209
|
const keepHead = Math.min(Math.floor(limit * 0.5), 4000)
|
|
212
210
|
const keepTail = Math.min(Math.floor(limit * 0.25), 2000)
|
|
213
211
|
m.content =
|
|
@@ -217,7 +215,7 @@ export function shrinkOversized(agent, limit = OVERSIZE_CONTENT_LIMIT) {
|
|
|
217
215
|
shrunk = true
|
|
218
216
|
}
|
|
219
217
|
if (shrunk) {
|
|
220
|
-
//
|
|
218
|
+
// Same as compaction: measured token baseline is invalidated by the changed history, fall back to estimation until next response
|
|
221
219
|
agent._lastPromptTokens = null
|
|
222
220
|
agent._usageAtLen = null
|
|
223
221
|
}
|
package/src/distill.mjs
CHANGED
|
@@ -1,53 +1,53 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* distill.mjs —
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* distill.mjs — extract knowledge candidates from sessions (the "automatic track" of the dual-track system)
|
|
3
|
+
* Principle (settled): manually triggered, LLM produces candidates, human confirms each one before writing.
|
|
4
|
+
* Absolutely no automatic storage at session end.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import { chat } from "./provider/index.mjs"
|
|
8
8
|
import { put, putMarkdown } from "./memory.mjs"
|
|
9
9
|
import { commitAndPush } from "./git/gitmem.mjs"
|
|
10
10
|
|
|
11
|
-
const DISTILL_PROMPT =
|
|
11
|
+
const DISTILL_PROMPT = `You are a knowledge extractor. Read the following agent work session log and extract knowledge worth remembering across sessions.
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
Output a JSON array (nothing else):
|
|
14
14
|
[
|
|
15
15
|
{
|
|
16
16
|
"type": "rule | knowledge | decision | pattern",
|
|
17
|
-
"title": "
|
|
18
|
-
"content": "
|
|
17
|
+
"title": "Short title",
|
|
18
|
+
"content": "Full content, self-contained — understandable without session context",
|
|
19
19
|
"tags": ["tag1", "tag2"],
|
|
20
20
|
"scope": "personal | project"
|
|
21
21
|
}
|
|
22
22
|
]
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
- knowledge
|
|
26
|
-
- decision
|
|
27
|
-
- pattern
|
|
28
|
-
- rule
|
|
29
|
-
- scope
|
|
24
|
+
Extraction criteria:
|
|
25
|
+
- knowledge: factual project knowledge (architecture, deployment, conventions)
|
|
26
|
+
- decision: technical decisions made in the session and their rationale
|
|
27
|
+
- pattern: debugging experiences, solutions, reusable workflows
|
|
28
|
+
- rule: coding standards (caution! rules are usually best written manually; only extract rules explicitly established in the session)
|
|
29
|
+
- scope: use "project" for project-specific knowledge; use "personal" for general or personal preferences
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
-
|
|
33
|
-
-
|
|
34
|
-
-
|
|
31
|
+
Do NOT extract:
|
|
32
|
+
- one-off task details ("changed line X in file Y today")
|
|
33
|
+
- transient state mentioned in the session (current bugs, work-in-progress)
|
|
34
|
+
- pleasantries and obvious facts
|
|
35
35
|
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
If nothing is worth extracting, output []
|
|
37
|
+
If the session is long, prioritize conclusions that appeared last and are still in effect.
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
Session log:
|
|
40
40
|
`
|
|
41
41
|
|
|
42
42
|
/**
|
|
43
|
-
*
|
|
44
|
-
*
|
|
43
|
+
* Extract candidates from a session transcript. transcript: plain-text session record.
|
|
44
|
+
* Returns [{ type, title, content, tags, scope }], or [] on parse failure.
|
|
45
45
|
*/
|
|
46
46
|
export async function extractCandidates(provider, transcript) {
|
|
47
47
|
const res = await chat(provider, {
|
|
48
48
|
messages: [{ role: "user", content: DISTILL_PROMPT + transcript }],
|
|
49
49
|
})
|
|
50
|
-
//
|
|
50
|
+
// Non-greedy match first JSON array (greedy [\s\S]* would eat across multiple arrays including interstitial text)
|
|
51
51
|
const match = res.content.match(/\[[\s\S]*?\]/)
|
|
52
52
|
if (!match) return []
|
|
53
53
|
try {
|
|
@@ -60,37 +60,37 @@ export async function extractCandidates(provider, transcript) {
|
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
/**
|
|
63
|
-
*
|
|
63
|
+
* Convert agent's OpenAI-format history to readable session transcript text.
|
|
64
64
|
*/
|
|
65
65
|
export function historyToTranscript(history, { maxChars = 30_000 } = {}) {
|
|
66
66
|
const lines = []
|
|
67
67
|
for (const m of history) {
|
|
68
68
|
if (m.role === "tool") {
|
|
69
|
-
lines.push(`[
|
|
69
|
+
lines.push(`[tool result] ${(m.content ?? "").slice(0, 500)}`)
|
|
70
70
|
} else if (m.tool_calls?.length) {
|
|
71
71
|
const calls = m.tool_calls.map((tc) => `${tc.function?.name ?? "?"}(${tc.function?.arguments?.slice(0, 200) ?? ""})`).join(", ")
|
|
72
|
-
lines.push(`[assistant] ${m.content ?? ""}\n[
|
|
72
|
+
lines.push(`[assistant] ${m.content ?? ""}\n[called tools] ${calls}`)
|
|
73
73
|
} else {
|
|
74
74
|
lines.push(`[${m.role}] ${m.content ?? ""}`)
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
77
|
const text = lines.join("\n\n")
|
|
78
|
-
//
|
|
78
|
+
// Overlong: keep head and tail (earliest requirements + latest conclusions are most important)
|
|
79
79
|
if (text.length <= maxChars) return text
|
|
80
80
|
const half = Math.floor(maxChars / 2)
|
|
81
|
-
return text.slice(0, half) + "\n\n...[
|
|
81
|
+
return text.slice(0, half) + "\n\n...[... middle portion omitted ...]...\n\n" + text.slice(-half)
|
|
82
82
|
}
|
|
83
83
|
|
|
84
84
|
/**
|
|
85
|
-
*
|
|
85
|
+
* Write confirmed candidates to the specified layer.
|
|
86
86
|
* opts: { projectDir, team: { dir } | null, author }
|
|
87
|
-
* scope=team
|
|
88
|
-
*
|
|
87
|
+
* scope=team requires opts.team; project requires opts.projectDir.
|
|
88
|
+
* Returns write result description.
|
|
89
89
|
*/
|
|
90
90
|
export async function saveCandidate(memory, candidate, opts = {}) {
|
|
91
91
|
const scope = candidate.scope ?? "personal"
|
|
92
|
-
// tags
|
|
93
|
-
//
|
|
92
|
+
// tags come from LLM output (untrusted): if not an array, stringify then split by comma/whitespace —
|
|
93
|
+
// calling .split on a non-string would crash, and models often produce "a, b" comma strings
|
|
94
94
|
const tags = Array.isArray(candidate.tags)
|
|
95
95
|
? candidate.tags.map((t) => String(t)).filter(Boolean)
|
|
96
96
|
: String(candidate.tags ?? "").split(/[\s,]+/).filter(Boolean)
|
|
@@ -100,7 +100,7 @@ export async function saveCandidate(memory, candidate, opts = {}) {
|
|
|
100
100
|
return `personal#${id}`
|
|
101
101
|
}
|
|
102
102
|
if (scope === "project") {
|
|
103
|
-
if (!opts.projectDir) throw new Error("project scope unavailable")
|
|
103
|
+
if (!opts.projectDir) throw new Error("project scope unavailable — no project directory configured (set memory.projectDir in ~/.thincoder/config.json)")
|
|
104
104
|
const filename = await putMarkdown(memory, {
|
|
105
105
|
layer: "project", dir: opts.projectDir,
|
|
106
106
|
type: candidate.type, title: candidate.title, content: candidate.content,
|
|
@@ -109,7 +109,7 @@ export async function saveCandidate(memory, candidate, opts = {}) {
|
|
|
109
109
|
return `project:${filename}`
|
|
110
110
|
}
|
|
111
111
|
if (scope === "team") {
|
|
112
|
-
if (!opts.team?.dir) throw new Error("team scope not configured")
|
|
112
|
+
if (!opts.team?.dir) throw new Error("team scope not configured — configure memory.team in ~/.thincoder/config.json")
|
|
113
113
|
const filename = await putMarkdown(memory, {
|
|
114
114
|
layer: "team", dir: opts.team.dir,
|
|
115
115
|
type: candidate.type, title: candidate.title, content: candidate.content,
|
package/src/embedding.mjs
CHANGED
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* embedding.mjs —
|
|
3
|
-
* OpenAI
|
|
4
|
-
*
|
|
5
|
-
*
|
|
2
|
+
* embedding.mjs — vector embeddings
|
|
3
|
+
* OpenAI-compatible /v1/embeddings (SiliconFlow bge-m3 / Ollama / OpenAI all supported),
|
|
4
|
+
* reuses provider.mjs fetch + retry pattern, zero dependencies.
|
|
5
|
+
* Vectors are normalized before storage; dot product then equals cosine similarity.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import { RETRYABLE_STATUS } from "./provider/index.mjs"
|
|
9
9
|
const MAX_RETRIES = 3
|
|
10
|
-
const BATCH_SIZE = 32 //
|
|
10
|
+
const BATCH_SIZE = 32 // max texts per request (within SiliconFlow limits)
|
|
11
11
|
|
|
12
|
-
/**
|
|
12
|
+
/** Create an embedder. config: { baseURL, apiKey, model } */
|
|
13
13
|
export function createEmbedder(config) {
|
|
14
|
-
if (!config?.baseURL) throw new Error("embedding config: baseURL is required")
|
|
15
|
-
if (!config?.apiKey) throw new Error("embedding config: apiKey is required
|
|
16
|
-
if (!config?.model) throw new Error("embedding config: model is required")
|
|
14
|
+
if (!config?.baseURL) throw new Error("embedding config: baseURL is required — configure embedding.baseURL in ~/.thincoder/config.json")
|
|
15
|
+
if (!config?.apiKey) throw new Error("embedding config: apiKey is required — set SILICONFLOW_API_KEY env or configure embedding.apiKey in ~/.thincoder/config.json")
|
|
16
|
+
if (!config?.model) throw new Error("embedding config: model is required — configure embedding.model in ~/.thincoder/config.json")
|
|
17
17
|
return {
|
|
18
18
|
baseURL: config.baseURL.replace(/\/+$/, ""),
|
|
19
19
|
apiKey: config.apiKey,
|
|
@@ -22,8 +22,8 @@ export function createEmbedder(config) {
|
|
|
22
22
|
}
|
|
23
23
|
|
|
24
24
|
/**
|
|
25
|
-
*
|
|
26
|
-
*
|
|
25
|
+
* Batch embedding. texts: string[] → Float32Array[] (normalized)
|
|
26
|
+
* Auto-batches, retries on failure (exponential backoff).
|
|
27
27
|
*/
|
|
28
28
|
export async function embed(embedder, texts, { signal } = {}) {
|
|
29
29
|
if (texts.length === 0) return []
|
|
@@ -31,11 +31,11 @@ export async function embed(embedder, texts, { signal } = {}) {
|
|
|
31
31
|
for (let i = 0; i < texts.length; i += BATCH_SIZE) {
|
|
32
32
|
const batch = texts.slice(i, i + BATCH_SIZE)
|
|
33
33
|
const data = await requestWithRetry(embedder, batch, signal)
|
|
34
|
-
//
|
|
34
|
+
// Mismatched count is a hard error — silently accepting would misalign vectors with texts, poisoning the entire index
|
|
35
35
|
if (!Array.isArray(data.data) || data.data.length !== batch.length) {
|
|
36
36
|
throw new Error(`Embedding API returned ${data.data?.length ?? 0} vectors for ${batch.length} inputs`)
|
|
37
37
|
}
|
|
38
|
-
//
|
|
38
|
+
// Spec says data[] order matches input, but sort by index field if present — don't bet on server implementation
|
|
39
39
|
const items = data.data.every((d) => typeof d.index === "number")
|
|
40
40
|
? [...data.data].sort((a, b) => a.index - b.index)
|
|
41
41
|
: data.data
|
|
@@ -46,7 +46,7 @@ export async function embed(embedder, texts, { signal } = {}) {
|
|
|
46
46
|
return vectors
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
-
/**
|
|
49
|
+
/** Cosine similarity (inputs are normalized, dot product equals cosine) */
|
|
50
50
|
export function cosine(a, b) {
|
|
51
51
|
if (a.length !== b.length) return 0
|
|
52
52
|
let sum = 0
|
|
@@ -55,20 +55,20 @@ export function cosine(a, b) {
|
|
|
55
55
|
return sum
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
-
/** Float32Array →
|
|
58
|
+
/** Float32Array → Buffer suitable for sqlite BLOB storage */
|
|
59
59
|
export function toBlob(vec) {
|
|
60
60
|
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength)
|
|
61
61
|
}
|
|
62
62
|
|
|
63
63
|
/** sqlite BLOB → Float32Array */
|
|
64
64
|
export function fromBlob(buf) {
|
|
65
|
-
// BLOB
|
|
65
|
+
// BLOB may come from Buffer pool where byteOffset isn't 4-aligned; creating a view directly would RangeError — copy to align first
|
|
66
66
|
if (buf.byteOffset % 4 !== 0) buf = new Uint8Array(buf)
|
|
67
67
|
if (buf.byteLength % 4 !== 0) return new Float32Array(0)
|
|
68
68
|
return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4)
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
-
// ----------------------------------------------------------------
|
|
71
|
+
// ---------------------------------------------------------------- internal
|
|
72
72
|
|
|
73
73
|
async function requestWithRetry(embedder, input, signal) {
|
|
74
74
|
let lastError
|