smart-context-mcp 1.18.1 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -13
- package/package.json +5 -2
- package/server.json +2 -2
- package/src/embeddings/embedder.js +28 -0
- package/src/embeddings/hashing.js +77 -0
- package/src/embeddings/index.js +91 -0
- package/src/embeddings/tokenize.js +46 -0
- package/src/global-memory/scrub.js +46 -0
- package/src/global-memory/store.js +375 -0
- package/src/index-watcher.js +224 -0
- package/src/index.js +91 -9
- package/src/orchestration/base-orchestrator.js +37 -1
- package/src/parsers/registry.js +26 -0
- package/src/playbooks/builtin/debug-flake.yaml +17 -0
- package/src/playbooks/builtin/doc-sync.yaml +16 -0
- package/src/playbooks/builtin/preflight-merge.yaml +20 -0
- package/src/playbooks/builtin/ramp-up.yaml +14 -0
- package/src/playbooks/builtin/refactor-safe.yaml +18 -0
- package/src/playbooks/loader.js +123 -0
- package/src/playbooks/runner.js +182 -0
- package/src/playbooks/yaml-mini.js +162 -0
- package/src/server.js +108 -13
- package/src/storage/sqlite.js +75 -1
- package/src/task-runner.js +4 -0
- package/src/tools/global-memory.js +110 -0
- package/src/tools/smart-context.js +18 -4
- package/src/tools/smart-playbook.js +63 -0
- package/src/tools/smart-read-batch.js +26 -3
- package/src/tools/smart-read.js +128 -15
- package/src/tools/smart-search.js +692 -55
- package/src/tools/smart-status.js +13 -0
- package/src/tools/smart-turn.js +88 -4
- package/src/turn/next-actions.js +4 -1
- package/src/utils/task-budget.js +116 -0
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
const COMMENT_RE = /(^|\s)#.*$/;
|
|
2
|
+
|
|
3
|
+
const stripComment = (line) => {
|
|
4
|
+
let inSingle = false;
|
|
5
|
+
let inDouble = false;
|
|
6
|
+
for (let i = 0; i < line.length; i += 1) {
|
|
7
|
+
const ch = line[i];
|
|
8
|
+
if (ch === '\\' && (inSingle || inDouble)) { i += 1; continue; }
|
|
9
|
+
if (ch === '"' && !inSingle) inDouble = !inDouble;
|
|
10
|
+
else if (ch === '\'' && !inDouble) inSingle = !inSingle;
|
|
11
|
+
else if (ch === '#' && !inSingle && !inDouble) return line.slice(0, i).trimEnd();
|
|
12
|
+
}
|
|
13
|
+
return line.replace(COMMENT_RE, '$1').trimEnd();
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
const unquoteScalar = (raw) => {
|
|
17
|
+
const trimmed = raw.trim();
|
|
18
|
+
if (trimmed === '') return null;
|
|
19
|
+
if ((trimmed.startsWith('"') && trimmed.endsWith('"')) ||
|
|
20
|
+
(trimmed.startsWith('\'') && trimmed.endsWith('\''))) {
|
|
21
|
+
return trimmed.slice(1, -1).replace(/\\(.)/g, '$1');
|
|
22
|
+
}
|
|
23
|
+
if (/^-?\d+$/.test(trimmed)) return parseInt(trimmed, 10);
|
|
24
|
+
if (/^-?\d+\.\d+$/.test(trimmed)) return parseFloat(trimmed);
|
|
25
|
+
if (trimmed === 'true') return true;
|
|
26
|
+
if (trimmed === 'false') return false;
|
|
27
|
+
if (trimmed === 'null' || trimmed === '~') return null;
|
|
28
|
+
return trimmed;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
const indentOf = (line) => {
|
|
32
|
+
let i = 0;
|
|
33
|
+
while (i < line.length && line[i] === ' ') i += 1;
|
|
34
|
+
return i;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
const parseFlowInline = (rest) => {
|
|
38
|
+
if (rest === '' || rest === undefined) return undefined;
|
|
39
|
+
const trimmed = rest.trim();
|
|
40
|
+
if (trimmed === '{}') return {};
|
|
41
|
+
if (trimmed === '[]') return [];
|
|
42
|
+
return unquoteScalar(rest);
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
class Reader {
|
|
46
|
+
constructor(lines) {
|
|
47
|
+
this.lines = lines;
|
|
48
|
+
this.idx = 0;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
peek() {
|
|
52
|
+
while (this.idx < this.lines.length) {
|
|
53
|
+
const raw = stripComment(this.lines[this.idx]);
|
|
54
|
+
if (raw.trim() === '') { this.idx += 1; continue; }
|
|
55
|
+
return { raw, indent: indentOf(raw), content: raw.slice(indentOf(raw)) };
|
|
56
|
+
}
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
consume() {
|
|
61
|
+
const next = this.peek();
|
|
62
|
+
if (next) this.idx += 1;
|
|
63
|
+
return next;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const parseValue = (reader, baseIndent) => {
|
|
68
|
+
const first = reader.peek();
|
|
69
|
+
if (!first || first.indent < baseIndent) return null;
|
|
70
|
+
|
|
71
|
+
if (first.content.startsWith('- ') || first.content === '-') {
|
|
72
|
+
return parseSequence(reader, first.indent);
|
|
73
|
+
}
|
|
74
|
+
return parseMapping(reader, first.indent);
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
const parseMapping = (reader, baseIndent) => {
|
|
78
|
+
const obj = {};
|
|
79
|
+
while (true) {
|
|
80
|
+
const next = reader.peek();
|
|
81
|
+
if (!next || next.indent < baseIndent) break;
|
|
82
|
+
if (next.indent > baseIndent) break;
|
|
83
|
+
if (next.content.startsWith('-')) break;
|
|
84
|
+
|
|
85
|
+
const colon = next.content.indexOf(':');
|
|
86
|
+
if (colon === -1) throw new Error(`YAML parse error: missing ':' near "${next.content}"`);
|
|
87
|
+
|
|
88
|
+
const key = next.content.slice(0, colon).trim();
|
|
89
|
+
const rest = next.content.slice(colon + 1).trim();
|
|
90
|
+
reader.consume();
|
|
91
|
+
|
|
92
|
+
if (rest === '' || rest === undefined) {
|
|
93
|
+
const nestedPeek = reader.peek();
|
|
94
|
+
if (nestedPeek && nestedPeek.indent > baseIndent) {
|
|
95
|
+
obj[key] = parseValue(reader, nestedPeek.indent);
|
|
96
|
+
} else {
|
|
97
|
+
obj[key] = null;
|
|
98
|
+
}
|
|
99
|
+
} else {
|
|
100
|
+
obj[key] = parseFlowInline(rest);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return obj;
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
const parseSequence = (reader, baseIndent) => {
|
|
107
|
+
const arr = [];
|
|
108
|
+
while (true) {
|
|
109
|
+
const next = reader.peek();
|
|
110
|
+
if (!next || next.indent !== baseIndent) break;
|
|
111
|
+
if (!next.content.startsWith('-')) break;
|
|
112
|
+
|
|
113
|
+
const tail = next.content.slice(1).trimStart();
|
|
114
|
+
reader.consume();
|
|
115
|
+
|
|
116
|
+
if (tail === '') {
|
|
117
|
+
const nestedPeek = reader.peek();
|
|
118
|
+
if (nestedPeek && nestedPeek.indent > baseIndent) {
|
|
119
|
+
arr.push(parseValue(reader, nestedPeek.indent));
|
|
120
|
+
} else {
|
|
121
|
+
arr.push(null);
|
|
122
|
+
}
|
|
123
|
+
} else if (tail.includes(':') && !/^["']/.test(tail)) {
|
|
124
|
+
const colon = tail.indexOf(':');
|
|
125
|
+
const key = tail.slice(0, colon).trim();
|
|
126
|
+
const rest = tail.slice(colon + 1).trim();
|
|
127
|
+
const item = {};
|
|
128
|
+
if (rest === '') {
|
|
129
|
+
const nestedPeek = reader.peek();
|
|
130
|
+
if (nestedPeek && nestedPeek.indent > baseIndent) {
|
|
131
|
+
const nested = parseMapping(reader, nestedPeek.indent);
|
|
132
|
+
item[key] = nested;
|
|
133
|
+
} else {
|
|
134
|
+
item[key] = null;
|
|
135
|
+
}
|
|
136
|
+
} else {
|
|
137
|
+
item[key] = parseFlowInline(rest);
|
|
138
|
+
}
|
|
139
|
+
const nestedPeek = reader.peek();
|
|
140
|
+
const itemIndent = baseIndent + 2;
|
|
141
|
+
if (nestedPeek && nestedPeek.indent >= itemIndent && !nestedPeek.content.startsWith('-')) {
|
|
142
|
+
const rest2 = parseMapping(reader, nestedPeek.indent);
|
|
143
|
+
Object.assign(item, rest2);
|
|
144
|
+
}
|
|
145
|
+
arr.push(item);
|
|
146
|
+
} else {
|
|
147
|
+
arr.push(parseFlowInline(tail));
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return arr;
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
export const parseYamlMini = (input) => {
|
|
154
|
+
if (typeof input !== 'string') throw new Error('YAML parse: input must be a string');
|
|
155
|
+
const lines = input.replace(/\r\n?/g, '\n').split('\n');
|
|
156
|
+
const reader = new Reader(lines);
|
|
157
|
+
const first = reader.peek();
|
|
158
|
+
if (!first) return null;
|
|
159
|
+
const baseIndent = first.indent;
|
|
160
|
+
if (first.content.startsWith('-')) return parseSequence(reader, baseIndent);
|
|
161
|
+
return parseMapping(reader, baseIndent);
|
|
162
|
+
};
|
package/src/server.js
CHANGED
|
@@ -10,6 +10,9 @@ import { smartReadBatch } from './tools/smart-read-batch.js';
|
|
|
10
10
|
import { smartShell } from './tools/smart-shell.js';
|
|
11
11
|
import { smartTest } from './tools/smart-test.js';
|
|
12
12
|
import { smartReview } from './tools/smart-review.js';
|
|
13
|
+
import { smartPlaybook } from './tools/smart-playbook.js';
|
|
14
|
+
import { globalMemory } from './tools/global-memory.js';
|
|
15
|
+
import { startIndexWatcher, isWatchEnabled, setActiveWatcher } from './index-watcher.js';
|
|
13
16
|
import { smartSummary } from './tools/smart-summary.js';
|
|
14
17
|
import { smartStatus } from './tools/smart-status.js';
|
|
15
18
|
import { smartDoctor } from './tools/smart-doctor.js';
|
|
@@ -126,7 +129,7 @@ export const createDevctxServer = () => {
|
|
|
126
129
|
|
|
127
130
|
server.tool(
|
|
128
131
|
'smart_read',
|
|
129
|
-
'Read a file with token-efficient modes. ALWAYS prefer outline/signatures/symbol/explain over full. Reading cascade: outline → signatures → symbol → explain → range → full (last resort). Mode guide: outline (~90% savings): file structure, exports, top-level symbols — use first for orientation. signatures (~85% savings): function signatures with parameters and return types — use when you need the API surface. symbol: extract specific functions/classes by name (string or array) — use when you know what to read; add context=true for callers, tests, and dependencies. explain (~95% savings): one-shot compact summary of a symbol (signature, docstring, first body line, side effects, caller count). Cached in SQLite by content hash — second call is free. Requires symbol. range: specific line range — use only when you need exact lines. full: raw content, no savings —
|
|
132
|
+
'Read a file with token-efficient modes. ALWAYS prefer outline/signatures/symbol/explain over full. Reading cascade: outline → signatures → symbol → explain → range → full (last resort). Mode guide: outline (~90% savings): file structure, exports, top-level symbols — use first for orientation. signatures (~85% savings): function signatures with parameters and return types — use when you need the API surface. symbol: extract specific functions/classes by name (string or array) — use when you know what to read; add context=true for callers, tests, and dependencies. explain (~95% savings): one-shot compact summary of a symbol (signature, docstring, first body line, side effects, caller count). Cached in SQLite by content hash — second call is free. Requires symbol. range: specific line range — use only when you need exact lines. full: raw content, no savings — explicit last resort; with a token budget it degrades to lighter modes first and reports `fullMode` metadata explaining whether full was actually used. maxTokens: token budget — auto-degrades to lighter modes before truncation; when the budget changes the result, `budgetDetails` reports the final mode, truncation actions, and marks `scope="content"`. Supports JS/TS, Python, Go, Rust, Java, C#, Kotlin, PHP, Swift, shell, Terraform, Dockerfile, SQL, JSON, TOML, YAML.',
|
|
130
133
|
{
|
|
131
134
|
filePath: z.string(),
|
|
132
135
|
mode: z.enum(['full', 'outline', 'signatures', 'range', 'symbol', 'explain']).optional(),
|
|
@@ -134,15 +137,23 @@ export const createDevctxServer = () => {
|
|
|
134
137
|
endLine: z.number().optional(),
|
|
135
138
|
symbol: z.union([z.string(), z.array(z.string())]).optional(),
|
|
136
139
|
maxTokens: z.number().int().min(1).optional(),
|
|
140
|
+
tokenBudget: z.union([
|
|
141
|
+
z.number().int().min(1),
|
|
142
|
+
z.object({
|
|
143
|
+
id: z.string().optional(),
|
|
144
|
+
maxTokens: z.number().int().min(1),
|
|
145
|
+
shared: z.boolean().optional(),
|
|
146
|
+
}),
|
|
147
|
+
]).optional(),
|
|
137
148
|
context: z.boolean().optional(),
|
|
138
149
|
},
|
|
139
|
-
async ({ filePath, mode = 'outline', startLine, endLine, symbol, maxTokens, context }) =>
|
|
140
|
-
asTextResult(await smartRead({ filePath, mode, startLine, endLine, symbol, maxTokens, context })),
|
|
150
|
+
async ({ filePath, mode = 'outline', startLine, endLine, symbol, maxTokens, tokenBudget, context }) =>
|
|
151
|
+
asTextResult(await smartRead({ filePath, mode, startLine, endLine, symbol, maxTokens, tokenBudget, context })),
|
|
141
152
|
);
|
|
142
153
|
|
|
143
154
|
server.tool(
|
|
144
155
|
'smart_read_batch',
|
|
145
|
-
'Read multiple files in one call. Each item accepts path, mode (prefer outline/signatures/symbol/explain — full saves 0 tokens), symbol, startLine, endLine, maxTokens (per-file budget). Optional global maxTokens budget with early stop when exceeded
|
|
156
|
+
'Read multiple files in one call. Each item accepts path, mode (prefer outline/signatures/symbol/explain — full saves 0 tokens), symbol, startLine, endLine, maxTokens (per-file budget). Optional global maxTokens budget with early stop when exceeded; when that happens, `budgetDetails` reports the batch-level stop point, marks `scope="batch"`, and includes `actions`. Max 20 files per call.',
|
|
146
157
|
{
|
|
147
158
|
files: z.array(z.object({
|
|
148
159
|
path: z.string(),
|
|
@@ -153,22 +164,35 @@ export const createDevctxServer = () => {
|
|
|
153
164
|
maxTokens: z.number().int().min(1).optional(),
|
|
154
165
|
})).min(1).max(20),
|
|
155
166
|
maxTokens: z.number().int().min(1).optional(),
|
|
167
|
+
tokenBudget: z.union([
|
|
168
|
+
z.number().int().min(1),
|
|
169
|
+
z.object({
|
|
170
|
+
id: z.string().optional(),
|
|
171
|
+
maxTokens: z.number().int().min(1),
|
|
172
|
+
shared: z.boolean().optional(),
|
|
173
|
+
}),
|
|
174
|
+
]).optional(),
|
|
156
175
|
},
|
|
157
|
-
async ({ files, maxTokens }) =>
|
|
158
|
-
asTextResult(await smartReadBatch({ files, maxTokens })),
|
|
176
|
+
async ({ files, maxTokens, tokenBudget }) =>
|
|
177
|
+
asTextResult(await smartReadBatch({ files, maxTokens, tokenBudget })),
|
|
159
178
|
);
|
|
160
179
|
|
|
161
180
|
server.tool(
|
|
162
181
|
'smart_search',
|
|
163
|
-
'Search code with ranked, deduplicated results and index boosting. Best for: finding where a symbol is defined/used, understanding call chains, locating implementations. NOT ideal for: exact string matching (use Grep), finding files by name (use Glob), broad multi-word queries (generates noise). Optional intent adjusts ranking. maxFiles caps the number of files returned (default
|
|
182
|
+
'Search code with ranked, deduplicated results and index boosting. Best for: finding where a symbol is defined/used, understanding call chains, locating implementations. NOT ideal for: exact string matching (use Grep), finding files by name (use Glob), broad multi-word queries (generates noise). Optional intent adjusts ranking. `mode` controls search strategy: `needle` = exact literal only (no regex or term expansion), `balanced` = exact + regex + term expansion (default), `semantic` = exact-first plus a local semantic block only when exact signal is weak. maxFiles caps the number of files returned (default 5). maxTokens caps the overall response payload: `matches` is truncated first, then optional diagnostics and semantic blocks are compacted or omitted if needed. When budgeting happens, `budgetDetails` reports `actions`, which sections were compacted, and marks `scope="response"`. kinds filters results by symbol kind from the index — e.g. ["adr","adr-section"] returns only architecture decision docs; ["class","function"] returns only those declarations; use to scope a query to a domain. `semantic=true` remains supported as a legacy alias for `mode="semantic"`. semanticLimit caps the semantic block (default 8). Top ranked files include `matchedBy`, `boostSource`, `scoreBreakdown`, and `whyRanked` so ranking decisions are inspectable. Semantic block adds zero deps and runs in <5ms even on large indexes. When more files exist beyond the initial window, the response includes `hasMore`, `totalFiles`, and `nextSuggestedMaxFiles` to support expansion on demand. When the search is too broad or returns nothing useful, the response also includes actionable `suggestions` for refining the query, mode, or kinds.',
|
|
164
183
|
{
|
|
165
184
|
query: z.string(),
|
|
166
185
|
cwd: z.string().optional(),
|
|
167
186
|
intent: z.enum(['implementation', 'debug', 'tests', 'config', 'docs', 'explore']).optional(),
|
|
168
187
|
maxFiles: z.number().int().min(1).max(50).optional(),
|
|
188
|
+
maxTokens: z.number().int().min(1).optional(),
|
|
169
189
|
kinds: z.array(z.string()).optional(),
|
|
190
|
+
mode: z.enum(['needle', 'balanced', 'semantic']).optional(),
|
|
191
|
+
semantic: z.boolean().optional(),
|
|
192
|
+
semanticLimit: z.number().int().min(1).max(50).optional(),
|
|
170
193
|
},
|
|
171
|
-
async ({ query, cwd = '.', intent, maxFiles,
|
|
194
|
+
async ({ query, cwd = '.', intent, maxFiles, maxTokens, kinds, mode, semantic, semanticLimit }) =>
|
|
195
|
+
asTextResult(await smartSearch({ query, cwd, intent, maxFiles, maxTokens, kinds, mode, semantic, semanticLimit })),
|
|
172
196
|
);
|
|
173
197
|
|
|
174
198
|
server.tool(
|
|
@@ -178,6 +202,14 @@ export const createDevctxServer = () => {
|
|
|
178
202
|
task: z.string().optional(),
|
|
179
203
|
intent: z.enum(['implementation', 'debug', 'tests', 'config', 'docs', 'explore']).optional(),
|
|
180
204
|
maxTokens: z.number().optional(),
|
|
205
|
+
tokenBudget: z.union([
|
|
206
|
+
z.number().int().min(1),
|
|
207
|
+
z.object({
|
|
208
|
+
id: z.string().optional(),
|
|
209
|
+
maxTokens: z.number().int().min(1),
|
|
210
|
+
shared: z.boolean().optional(),
|
|
211
|
+
}),
|
|
212
|
+
]).optional(),
|
|
181
213
|
entryFile: z.string().optional(),
|
|
182
214
|
diff: z.union([z.boolean(), z.string()]).optional(),
|
|
183
215
|
detail: z.enum(['minimal', 'balanced', 'deep']).optional(),
|
|
@@ -190,8 +222,8 @@ export const createDevctxServer = () => {
|
|
|
190
222
|
pathMaxHops: z.number().int().min(1).max(10).optional(),
|
|
191
223
|
pathDirected: z.boolean().optional(),
|
|
192
224
|
},
|
|
193
|
-
async ({ task, intent, maxTokens, entryFile, diff, detail, include, prefetch, paths, pathMaxHops, pathDirected }) =>
|
|
194
|
-
asTextResult(await smartContext({ task, intent, maxTokens, entryFile, diff, detail, include, prefetch, paths, pathMaxHops, pathDirected })),
|
|
225
|
+
async ({ task, intent, maxTokens, tokenBudget, entryFile, diff, detail, include, prefetch, paths, pathMaxHops, pathDirected }) =>
|
|
226
|
+
asTextResult(await smartContext({ task, intent, maxTokens, tokenBudget, entryFile, diff, detail, include, prefetch, paths, pathMaxHops, pathDirected })),
|
|
195
227
|
);
|
|
196
228
|
|
|
197
229
|
server.tool(
|
|
@@ -234,6 +266,37 @@ export const createDevctxServer = () => {
|
|
|
234
266
|
asTextResult(await smartReview({ ref, maxFiles, maxCallers, maxTests, includeBlame })),
|
|
235
267
|
);
|
|
236
268
|
|
|
269
|
+
server.tool(
|
|
270
|
+
'smart_playbook',
|
|
271
|
+
'Run a declarative workflow (playbook) that composes other smart_* tools in one call. Built-in playbooks: preflight-merge (smart_review + smart_test affected + checkpoint), debug-flake (last_failure + curated debug context + affected tests), refactor-safe (curated context + affected tests + checkpoint), doc-sync (ADR search + docs context), ramp-up (status + doctor + ADR overview). Override or add your own in .devctx/playbooks/*.yaml (or *.json). Pass list=true to enumerate available playbooks. Pass dryRun=true to resolve and validate steps without executing them. Args support {{args.X}} interpolation against defaults + caller args. Only smart_* tools are allowed inside playbooks; shell access stays gated by smart_shell allowlist.',
|
|
272
|
+
{
|
|
273
|
+
name: z.string().optional(),
|
|
274
|
+
args: z.record(z.string(), z.any()).optional(),
|
|
275
|
+
list: z.boolean().optional(),
|
|
276
|
+
dryRun: z.boolean().optional(),
|
|
277
|
+
stopOnFail: z.boolean().optional(),
|
|
278
|
+
},
|
|
279
|
+
async ({ name, args, list, dryRun, stopOnFail }) =>
|
|
280
|
+
asTextResult(await smartPlaybook({ name, args, list, dryRun, stopOnFail })),
|
|
281
|
+
);
|
|
282
|
+
|
|
283
|
+
server.tool(
|
|
284
|
+
'global_memory',
|
|
285
|
+
'Opt-in cross-project memory persisted to ~/.devctx/global.db (override with DEVCTX_GLOBAL_DB). Enable via DEVCTX_GLOBAL_MEMORY=true. Stores canonical decisions, recurring patterns, playbook drafts, notes, and repo-local noise hints so an agent can carry insights between repos without re-deriving them. Content is scrubbed for likely secrets/JWTs/API keys/emails/home paths before being persisted. Project paths are stored hashed (FNV-1a) instead of raw. Actions: save (kind+content+tags?), recall (kind?+query?+limit? — uses local hashing/TF-IDF embedder for ranking, zero deps), list (counts per kind), delete (id), mark_used (id), stats (db size + per-kind totals), noise_stats (inspect repo noise hints), noise_reset (reset repo noise hints or one hint via query). Valid kinds: decision, pattern, playbook, note. projectScope=true (default) hashes the current project so recall can be filtered per-project; set false for repo-agnostic access.',
|
|
286
|
+
{
|
|
287
|
+
action: z.enum(['save', 'recall', 'list', 'delete', 'stats', 'mark_used', 'noise_stats', 'noise_reset']),
|
|
288
|
+
kind: z.enum(['decision', 'pattern', 'playbook', 'note']).optional(),
|
|
289
|
+
content: z.string().optional(),
|
|
290
|
+
tags: z.array(z.string()).optional(),
|
|
291
|
+
query: z.string().optional(),
|
|
292
|
+
limit: z.number().int().min(1).max(100).optional(),
|
|
293
|
+
id: z.number().int().optional(),
|
|
294
|
+
projectScope: z.boolean().optional(),
|
|
295
|
+
},
|
|
296
|
+
async ({ action, kind, content, tags, query, limit, id, projectScope }) =>
|
|
297
|
+
asTextResult(await globalMemory({ action, kind, content, tags, query, limit, id, projectScope })),
|
|
298
|
+
);
|
|
299
|
+
|
|
237
300
|
server.tool(
|
|
238
301
|
'build_index',
|
|
239
302
|
'Build a lightweight symbol index for the project. Speeds up smart_search ranking and smart_read symbol lookups. Pass incremental=true to only reindex files with changed mtime (much faster for large repos). Pass warmCache=true to preload frequently accessed files after indexing. Without incremental, rebuilds from scratch. Sends progress notifications during indexing for large projects.',
|
|
@@ -545,13 +608,21 @@ export const createDevctxServer = () => {
|
|
|
545
608
|
event: z.enum(['manual', 'milestone', 'decision', 'blocker', 'status_change', 'file_change', 'task_switch', 'task_complete', 'session_end', 'read_only', 'heartbeat']).optional(),
|
|
546
609
|
force: z.boolean().optional(),
|
|
547
610
|
maxTokens: z.number().int().min(100).max(2000).optional(),
|
|
611
|
+
tokenBudget: z.union([
|
|
612
|
+
z.number().int().min(1),
|
|
613
|
+
z.object({
|
|
614
|
+
id: z.string().optional(),
|
|
615
|
+
maxTokens: z.number().int().min(1),
|
|
616
|
+
shared: z.boolean().optional(),
|
|
617
|
+
}),
|
|
618
|
+
]).optional(),
|
|
548
619
|
ensureSession: z.boolean().optional(),
|
|
549
620
|
includeMetrics: z.boolean().optional(),
|
|
550
621
|
metricsWindow: z.enum(['24h', '7d', '30d', 'all']).optional(),
|
|
551
622
|
latestMetrics: z.number().int().min(1).max(20).optional(),
|
|
552
623
|
verbosity: z.enum(['minimal', 'standard', 'full']).optional().describe('Default "minimal" — returns compact recommendedPath/continuity/task. Use "standard" or "full" only when you need long instructions, candidates, or full checkpoint diagnostics.'),
|
|
553
624
|
},
|
|
554
|
-
async ({ phase, sessionId, prompt, update, event, force, maxTokens, ensureSession, includeMetrics, metricsWindow, latestMetrics, verbosity }) =>
|
|
625
|
+
async ({ phase, sessionId, prompt, update, event, force, maxTokens, tokenBudget, ensureSession, includeMetrics, metricsWindow, latestMetrics, verbosity }) =>
|
|
555
626
|
asTextResult(await smartTurn({
|
|
556
627
|
phase,
|
|
557
628
|
sessionId,
|
|
@@ -560,6 +631,7 @@ export const createDevctxServer = () => {
|
|
|
560
631
|
event,
|
|
561
632
|
force,
|
|
562
633
|
maxTokens,
|
|
634
|
+
tokenBudget,
|
|
563
635
|
ensureSession,
|
|
564
636
|
includeMetrics,
|
|
565
637
|
metricsWindow,
|
|
@@ -576,15 +648,24 @@ export const createDevctxServer = () => {
|
|
|
576
648
|
sessionId: z.string().optional(),
|
|
577
649
|
taskId: z.string().optional(),
|
|
578
650
|
maxTokens: z.number().int().min(100).max(2000).optional(),
|
|
651
|
+
tokenBudget: z.union([
|
|
652
|
+
z.number().int().min(1),
|
|
653
|
+
z.object({
|
|
654
|
+
id: z.string().optional(),
|
|
655
|
+
maxTokens: z.number().int().min(1),
|
|
656
|
+
shared: z.boolean().optional(),
|
|
657
|
+
}),
|
|
658
|
+
]).optional(),
|
|
579
659
|
verbosity: z.enum(['minimal', 'standard', 'full']).optional(),
|
|
580
660
|
},
|
|
581
|
-
async ({ prompt, sessionId, taskId, maxTokens, verbosity }) =>
|
|
661
|
+
async ({ prompt, sessionId, taskId, maxTokens, tokenBudget, verbosity }) =>
|
|
582
662
|
asTextResult(await smartTurn({
|
|
583
663
|
phase: 'start',
|
|
584
664
|
prompt,
|
|
585
665
|
sessionId,
|
|
586
666
|
taskId,
|
|
587
667
|
maxTokens,
|
|
668
|
+
tokenBudget,
|
|
588
669
|
ensureSession: true,
|
|
589
670
|
verbosity: verbosity ?? 'minimal',
|
|
590
671
|
})),
|
|
@@ -616,8 +697,22 @@ export const runDevctxServer = async () => {
|
|
|
616
697
|
const transport = new StdioServerTransport();
|
|
617
698
|
await server.connect(transport);
|
|
618
699
|
|
|
700
|
+
const watcher = isWatchEnabled() ? startIndexWatcher() : null;
|
|
701
|
+
if (watcher) {
|
|
702
|
+
setActiveWatcher(watcher);
|
|
703
|
+
if (process.env.DEVCTX_DEBUG === '1') {
|
|
704
|
+
process.stderr.write('[devctx] reactive index watcher: ENABLED\n');
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
|
|
619
708
|
const shutdown = () => {
|
|
620
|
-
|
|
709
|
+
const finalize = async () => {
|
|
710
|
+
try { if (watcher) await watcher.stop(); } catch { /* noop */ }
|
|
711
|
+
try { setActiveWatcher(null); } catch { /* noop */ }
|
|
712
|
+
try { await transport.close(); } catch { /* noop */ }
|
|
713
|
+
process.exit(0);
|
|
714
|
+
};
|
|
715
|
+
finalize();
|
|
621
716
|
};
|
|
622
717
|
|
|
623
718
|
process.on('SIGINT', shutdown);
|
package/src/storage/sqlite.js
CHANGED
|
@@ -6,7 +6,7 @@ import { setTimeout as delay } from 'node:timers/promises';
|
|
|
6
6
|
import { projectRoot } from '../utils/runtime-config.js';
|
|
7
7
|
|
|
8
8
|
export const STATE_DB_FILENAME = 'state.sqlite';
|
|
9
|
-
export const SQLITE_SCHEMA_VERSION =
|
|
9
|
+
export const SQLITE_SCHEMA_VERSION = 8;
|
|
10
10
|
export const ACTIVE_SESSION_SCOPE = 'project';
|
|
11
11
|
export const STATE_DB_SOFT_MAX_BYTES = 32 * 1024 * 1024;
|
|
12
12
|
const STATE_DB_BUSY_TIMEOUT_MS = 1000;
|
|
@@ -20,6 +20,7 @@ export const EXPECTED_TABLES = [
|
|
|
20
20
|
'hook_turn_state',
|
|
21
21
|
'meta',
|
|
22
22
|
'metrics_events',
|
|
23
|
+
'read_cache',
|
|
23
24
|
'session_events',
|
|
24
25
|
'sessions',
|
|
25
26
|
'summary_cache',
|
|
@@ -266,6 +267,26 @@ const MIGRATIONS = [
|
|
|
266
267
|
ON explain_cache(updated_at DESC)`,
|
|
267
268
|
],
|
|
268
269
|
},
|
|
270
|
+
{
|
|
271
|
+
version: 8,
|
|
272
|
+
statements: [
|
|
273
|
+
`CREATE TABLE IF NOT EXISTS read_cache (
|
|
274
|
+
cache_key TEXT PRIMARY KEY,
|
|
275
|
+
file_path TEXT NOT NULL,
|
|
276
|
+
mode TEXT NOT NULL,
|
|
277
|
+
selector TEXT NOT NULL DEFAULT '',
|
|
278
|
+
content_hash TEXT NOT NULL,
|
|
279
|
+
payload_json TEXT NOT NULL,
|
|
280
|
+
tokens INTEGER NOT NULL DEFAULT 0,
|
|
281
|
+
created_at TEXT NOT NULL,
|
|
282
|
+
updated_at TEXT NOT NULL
|
|
283
|
+
)`,
|
|
284
|
+
`CREATE INDEX IF NOT EXISTS idx_read_cache_file_mode
|
|
285
|
+
ON read_cache(file_path, mode, updated_at DESC)`,
|
|
286
|
+
`CREATE INDEX IF NOT EXISTS idx_read_cache_updated
|
|
287
|
+
ON read_cache(updated_at DESC)`,
|
|
288
|
+
],
|
|
289
|
+
},
|
|
269
290
|
];
|
|
270
291
|
|
|
271
292
|
let sqliteModulePromise = null;
|
|
@@ -1512,6 +1533,7 @@ export const runStorageMaintenance = async ({
|
|
|
1512
1533
|
workflowMetrics: removeOlder('DELETE FROM workflow_metrics WHERE created_at < ?'),
|
|
1513
1534
|
contextAccess: removeOlder('DELETE FROM context_access WHERE timestamp < ?'),
|
|
1514
1535
|
explainCache: removeOlder('DELETE FROM explain_cache WHERE updated_at < ?'),
|
|
1536
|
+
readCache: removeOlder('DELETE FROM read_cache WHERE updated_at < ?'),
|
|
1515
1537
|
};
|
|
1516
1538
|
|
|
1517
1539
|
setMeta(db, STORAGE_GC_META_KEY, String(now));
|
|
@@ -1955,6 +1977,9 @@ export const cleanupLegacyState = async ({
|
|
|
1955
1977
|
const buildExplainCacheKey = ({ filePath, symbol, contentHash }) =>
|
|
1956
1978
|
createHash('sha256').update(`${filePath}\u241F${symbol}\u241F${contentHash}`).digest('hex');
|
|
1957
1979
|
|
|
1980
|
+
const buildReadCacheKey = ({ filePath, mode, selector = '', contentHash }) =>
|
|
1981
|
+
createHash('sha256').update(`${filePath}\u241F${mode}\u241F${selector}\u241F${contentHash}`).digest('hex');
|
|
1982
|
+
|
|
1958
1983
|
export const getExplainCache = async ({
|
|
1959
1984
|
filePath: dbPath = getStateDbPath(),
|
|
1960
1985
|
relPath,
|
|
@@ -2005,6 +2030,55 @@ export const clearExplainCache = async ({ filePath = getStateDbPath() } = {}) =>
|
|
|
2005
2030
|
return db.prepare('DELETE FROM explain_cache').run().changes;
|
|
2006
2031
|
}, { filePath });
|
|
2007
2032
|
|
|
2033
|
+
export const getReadCache = async ({
|
|
2034
|
+
filePath: dbPath = getStateDbPath(),
|
|
2035
|
+
relPath,
|
|
2036
|
+
mode,
|
|
2037
|
+
selector = '',
|
|
2038
|
+
contentHash,
|
|
2039
|
+
} = {}) => withStateDb((db) => {
|
|
2040
|
+
if (!relPath || !mode || !contentHash) return null;
|
|
2041
|
+
const cacheKey = buildReadCacheKey({ filePath: relPath, mode, selector, contentHash });
|
|
2042
|
+
const row = db.prepare(`
|
|
2043
|
+
SELECT payload_json, tokens, updated_at
|
|
2044
|
+
FROM read_cache
|
|
2045
|
+
WHERE cache_key = ?
|
|
2046
|
+
`).get(cacheKey);
|
|
2047
|
+
if (!row) return null;
|
|
2048
|
+
return {
|
|
2049
|
+
payload: parseJsonText(row.payload_json, null),
|
|
2050
|
+
tokens: row.tokens,
|
|
2051
|
+
updatedAt: row.updated_at,
|
|
2052
|
+
};
|
|
2053
|
+
}, { filePath: dbPath });
|
|
2054
|
+
|
|
2055
|
+
export const setReadCache = async ({
|
|
2056
|
+
filePath: dbPath = getStateDbPath(),
|
|
2057
|
+
relPath,
|
|
2058
|
+
mode,
|
|
2059
|
+
selector = '',
|
|
2060
|
+
contentHash,
|
|
2061
|
+
payload,
|
|
2062
|
+
tokens = 0,
|
|
2063
|
+
} = {}) => withStateDb((db) => {
|
|
2064
|
+
if (!relPath || !mode || !contentHash || !payload) return null;
|
|
2065
|
+
const cacheKey = buildReadCacheKey({ filePath: relPath, mode, selector, contentHash });
|
|
2066
|
+
const now = new Date().toISOString();
|
|
2067
|
+
db.prepare(`
|
|
2068
|
+
INSERT INTO read_cache(cache_key, file_path, mode, selector, content_hash, payload_json, tokens, created_at, updated_at)
|
|
2069
|
+
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2070
|
+
ON CONFLICT(cache_key) DO UPDATE SET
|
|
2071
|
+
payload_json = excluded.payload_json,
|
|
2072
|
+
tokens = excluded.tokens,
|
|
2073
|
+
updated_at = excluded.updated_at
|
|
2074
|
+
`).run(cacheKey, relPath, mode, selector, contentHash, toJsonText(payload), tokens, now, now);
|
|
2075
|
+
return { cacheKey, updatedAt: now };
|
|
2076
|
+
}, { filePath: dbPath });
|
|
2077
|
+
|
|
2078
|
+
export const clearReadCachePersistent = async ({ filePath = getStateDbPath() } = {}) => withStateDb((db) => {
|
|
2079
|
+
return db.prepare('DELETE FROM read_cache').run().changes;
|
|
2080
|
+
}, { filePath });
|
|
2081
|
+
|
|
2008
2082
|
const LAST_TEST_FAILURE_META_KEY = 'last_test_failure';
|
|
2009
2083
|
|
|
2010
2084
|
export const setLastTestFailure = async ({
|
package/src/task-runner.js
CHANGED
|
@@ -129,6 +129,7 @@ const runWorkflowCommand = async ({
|
|
|
129
129
|
client,
|
|
130
130
|
prompt,
|
|
131
131
|
sessionId,
|
|
132
|
+
tokenBudget,
|
|
132
133
|
event,
|
|
133
134
|
stdinPrompt = false,
|
|
134
135
|
dryRun = false,
|
|
@@ -147,6 +148,7 @@ const runWorkflowCommand = async ({
|
|
|
147
148
|
const startResolution = await withRunnerLockRetry(() => resolveManagedStart({
|
|
148
149
|
prompt: requestedPrompt,
|
|
149
150
|
sessionId,
|
|
151
|
+
tokenBudget,
|
|
150
152
|
ensureSession: true,
|
|
151
153
|
allowIsolation: false,
|
|
152
154
|
startMaxTokens: DEFAULT_START_MAX_TOKENS,
|
|
@@ -425,6 +427,7 @@ export const runTaskRunner = async ({
|
|
|
425
427
|
client = null,
|
|
426
428
|
prompt = '',
|
|
427
429
|
sessionId,
|
|
430
|
+
tokenBudget,
|
|
428
431
|
event,
|
|
429
432
|
stdinPrompt = false,
|
|
430
433
|
dryRun = false,
|
|
@@ -456,6 +459,7 @@ export const runTaskRunner = async ({
|
|
|
456
459
|
client: resolvedClient,
|
|
457
460
|
prompt,
|
|
458
461
|
sessionId,
|
|
462
|
+
tokenBudget,
|
|
459
463
|
event,
|
|
460
464
|
stdinPrompt,
|
|
461
465
|
dryRun,
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import {
|
|
2
|
+
saveEntry,
|
|
3
|
+
recallEntries,
|
|
4
|
+
markEntryUsed,
|
|
5
|
+
deleteEntry,
|
|
6
|
+
listKinds,
|
|
7
|
+
getStats,
|
|
8
|
+
recordNoiseHint,
|
|
9
|
+
getNoiseHints,
|
|
10
|
+
resetNoiseHints,
|
|
11
|
+
isGlobalMemoryEnabled,
|
|
12
|
+
VALID_GLOBAL_KINDS,
|
|
13
|
+
} from '../global-memory/store.js';
|
|
14
|
+
import { containsLikelySecret } from '../global-memory/scrub.js';
|
|
15
|
+
import { projectRoot } from '../utils/paths.js';
|
|
16
|
+
import { recordDevctxOperation } from '../missed-opportunities.js';
|
|
17
|
+
import { recordDecision, DECISION_REASONS, EXPECTED_BENEFITS } from '../decision-explainer.js';
|
|
18
|
+
|
|
19
|
+
const VALID_ACTIONS = new Set(['save', 'recall', 'list', 'delete', 'stats', 'mark_used', 'noise_stats', 'noise_reset']);
|
|
20
|
+
|
|
21
|
+
export const globalMemory = async ({
|
|
22
|
+
action = 'stats',
|
|
23
|
+
kind,
|
|
24
|
+
content,
|
|
25
|
+
tags,
|
|
26
|
+
query,
|
|
27
|
+
limit,
|
|
28
|
+
id,
|
|
29
|
+
projectScope = true,
|
|
30
|
+
} = {}) => {
|
|
31
|
+
if (!VALID_ACTIONS.has(action)) {
|
|
32
|
+
return { success: false, error: `Invalid action: ${action}. Must be one of: ${[...VALID_ACTIONS].join(', ')}` };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (!isGlobalMemoryEnabled()) {
|
|
36
|
+
return {
|
|
37
|
+
success: false,
|
|
38
|
+
disabled: true,
|
|
39
|
+
message: 'Global memory is opt-in. Set DEVCTX_GLOBAL_MEMORY=true to enable.',
|
|
40
|
+
hint: 'Stored content is automatically scrubbed for secrets/emails/paths before persistence.',
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
recordDevctxOperation();
|
|
45
|
+
|
|
46
|
+
try {
|
|
47
|
+
switch (action) {
|
|
48
|
+
case 'save': {
|
|
49
|
+
if (!kind || !content) return { success: false, error: 'save requires kind and content' };
|
|
50
|
+
if (!VALID_GLOBAL_KINDS.has(kind)) {
|
|
51
|
+
return { success: false, error: `Invalid kind. Must be one of: ${[...VALID_GLOBAL_KINDS].join(', ')}` };
|
|
52
|
+
}
|
|
53
|
+
const secretFlag = containsLikelySecret(content);
|
|
54
|
+
const result = await saveEntry({
|
|
55
|
+
kind,
|
|
56
|
+
content,
|
|
57
|
+
tags,
|
|
58
|
+
projectPath: projectScope ? projectRoot : null,
|
|
59
|
+
});
|
|
60
|
+
recordDecision({
|
|
61
|
+
tool: 'global_memory',
|
|
62
|
+
action: `save ${kind}`,
|
|
63
|
+
reason: DECISION_REASONS.RELATED_FILES ?? 'cross-project memory',
|
|
64
|
+
alternative: 'Re-derive context next session manually',
|
|
65
|
+
expectedBenefit: EXPECTED_BENEFITS.TOKEN_SAVINGS(content.length / 4),
|
|
66
|
+
context: secretFlag ? 'Content contained likely secrets; scrubbed before persistence' : 'Saved without secret hits',
|
|
67
|
+
});
|
|
68
|
+
return { success: true, ...result, scrubbedFromSecrets: secretFlag };
|
|
69
|
+
}
|
|
70
|
+
case 'recall': {
|
|
71
|
+
const result = await recallEntries({
|
|
72
|
+
kind,
|
|
73
|
+
query,
|
|
74
|
+
limit: limit ?? 10,
|
|
75
|
+
projectPath: projectScope ? projectRoot : null,
|
|
76
|
+
});
|
|
77
|
+
return { success: true, action, ...result };
|
|
78
|
+
}
|
|
79
|
+
case 'list': {
|
|
80
|
+
const result = await listKinds();
|
|
81
|
+
return { success: true, action, ...result };
|
|
82
|
+
}
|
|
83
|
+
case 'delete': {
|
|
84
|
+
if (!id) return { success: false, error: 'delete requires id' };
|
|
85
|
+
const result = await deleteEntry({ id });
|
|
86
|
+
return { success: true, action, ...result };
|
|
87
|
+
}
|
|
88
|
+
case 'mark_used': {
|
|
89
|
+
if (!id) return { success: false, error: 'mark_used requires id' };
|
|
90
|
+
const result = await markEntryUsed({ id });
|
|
91
|
+
return { success: true, action, ...result };
|
|
92
|
+
}
|
|
93
|
+
case 'stats':
|
|
94
|
+
default: {
|
|
95
|
+
const stats = await getStats();
|
|
96
|
+
return { success: true, action: 'stats', ...stats };
|
|
97
|
+
}
|
|
98
|
+
case 'noise_stats': {
|
|
99
|
+
const result = await getNoiseHints({ projectPath: projectScope ? projectRoot : null, limit: limit ?? 50 });
|
|
100
|
+
return { success: true, action, ...result };
|
|
101
|
+
}
|
|
102
|
+
case 'noise_reset': {
|
|
103
|
+
const result = await resetNoiseHints({ projectPath: projectScope ? projectRoot : null, hintKey: query });
|
|
104
|
+
return { success: true, action, ...result };
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
} catch (err) {
|
|
108
|
+
return { success: false, error: err?.message ?? String(err) };
|
|
109
|
+
}
|
|
110
|
+
};
|