specmem-hardwicksoftware 3.7.35 → 3.7.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/README.md +11 -15
  3. package/bin/specmem-console.cjs +839 -51
  4. package/claude-hooks/agent-chooser-hook.js +6 -6
  5. package/claude-hooks/agent-loading-hook.cjs +16 -16
  6. package/claude-hooks/agent-loading-hook.js +18 -18
  7. package/claude-hooks/agent-type-matcher.js +1 -1
  8. package/claude-hooks/background-completion-silencer.js +1 -1
  9. package/claude-hooks/file-claim-enforcer.cjs +37 -36
  10. package/claude-hooks/output-cleaner.cjs +1 -1
  11. package/claude-hooks/settings.json +27 -3
  12. package/claude-hooks/specmem-search-enforcer.cjs +2 -11
  13. package/claude-hooks/specmem-team-member-inject.js +1 -1
  14. package/claude-hooks/specmem-unified-hook.py +1 -1
  15. package/claude-hooks/subagent-loading-hook.cjs +1 -1
  16. package/claude-hooks/task-progress-hook.cjs +7 -7
  17. package/claude-hooks/task-progress-hook.js +3 -3
  18. package/claude-hooks/team-comms-enforcer.cjs +49 -47
  19. package/dist/claude-sessions/sessionParser.js +5 -0
  20. package/dist/codebase/codebaseIndexer.js +48 -17
  21. package/dist/codebase/exclusions.js +3 -4
  22. package/dist/codebase/index.js +4 -0
  23. package/dist/codebase/pdfExtractor.js +298 -0
  24. package/dist/dashboard/api/taskTeamMembers.js +2 -2
  25. package/dist/db/bigBrainMigrations.js +29 -0
  26. package/dist/hooks/hookManager.js +4 -4
  27. package/dist/hooks/teamFramingCli.js +1 -1
  28. package/dist/hooks/teamMemberPrepromptHook.js +5 -5
  29. package/dist/init/claudeConfigInjector.js +2 -2
  30. package/dist/mcp/compactionProxy.js +834 -186
  31. package/dist/mcp/compactionProxyDaemon.js +112 -37
  32. package/dist/mcp/contextVault.js +439 -0
  33. package/dist/mcp/embeddingServerManager.js +61 -1
  34. package/dist/mcp/mcpProtocolHandler.js +6 -1
  35. package/dist/mcp/miniCOTServerManager.js +82 -8
  36. package/dist/mcp/specMemServer.js +45 -10
  37. package/dist/mcp/toolRegistry.js +6 -0
  38. package/dist/startup/startupIndexing.js +14 -0
  39. package/dist/team-members/taskOrchestrator.js +3 -3
  40. package/dist/team-members/taskTeamMemberLogger.js +2 -2
  41. package/dist/tools/goofy/deployTeamMember.js +3 -3
  42. package/dist/tools/goofy/digInTheVault.js +81 -0
  43. package/dist/tools/goofy/stashTheGoods.js +56 -0
  44. package/dist/tools/teamMemberDeployer.js +2 -2
  45. package/dist/watcher/changeHandler.js +65 -8
  46. package/dist/watcher/changeQueue.js +20 -1
  47. package/embedding-sandbox/mini-cot-service.py +11 -13
  48. package/embedding-sandbox/pdf-text-extract.py +208 -0
  49. package/package.json +1 -1
  50. package/scripts/deploy-hooks.cjs +2 -2
  51. package/scripts/global-postinstall.cjs +2 -2
  52. package/scripts/specmem-init.cjs +130 -36
  53. package/specmem/model-config.json +6 -6
  54. package/specmem/supervisord.conf +1 -1
  55. package/svg-sections/readme-token-compaction.svg +246 -0
@@ -3,7 +3,7 @@
3
3
  * Compaction Proxy Daemon — Standalone Persistent Process
4
4
  * ========================================================
5
5
  *
6
- * Survives MCP server restarts. Self-terminates when Claude is gone.
6
+ * Survives MCP server restarts. Self-terminates when parent is gone.
7
7
  *
8
8
  * Spawned by startCompactionProxy() in compactionProxy.js.
9
9
  * Imports the core handleRequest and all compression logic — zero duplication.
@@ -12,7 +12,8 @@
12
12
  * 1. MCP starts → startCompactionProxy() spawns this daemon (detached)
13
13
  * 2. MCP dies/restarts → daemon stays alive, port file persists
14
14
  * 3. Next MCP start → detects running daemon via PID file, skips spawn
15
- * 4. Claude exits → KYS watchdog self-terminates after 2 min idle
15
+ * 4. Parent exits → orphan watchdog detects dead parent PID, starts idle countdown
16
+ * 5. No requests for ORPHAN_IDLE_TIMEOUT → graceful self-termination
16
17
  *
17
18
  * @author hardwicksoftwareservices
18
19
  * @website https://justcalljon.pro
@@ -20,7 +21,6 @@
20
21
 
21
22
  import { createServer } from 'http';
22
23
  import { writeFileSync, unlinkSync, existsSync, mkdirSync } from 'fs';
23
- import { execSync } from 'child_process';
24
24
  import { createConnection } from 'net';
25
25
  import {
26
26
  handleRequest,
@@ -34,6 +34,7 @@ import {
34
34
  PID_FILE,
35
35
  DISABLED_FILE,
36
36
  CLAUDE_DIR,
37
+ getLastRequestTime,
37
38
  } from './compactionProxy.js';
38
39
 
39
40
  // ============================================================================
@@ -68,28 +69,66 @@ function cleanup() {
68
69
  }
69
70
 
70
71
  // ============================================================================
71
- // KYS Watchdog self-terminate when Claude is gone
72
+ // Orphan Detection & KYS Watchdog
72
73
  // ============================================================================
74
+ //
75
+ // Three-layer orphan detection:
76
+ // 1. Parent PID monitoring — process.kill(ppid, 0) every 30s
77
+ // 2. Socket activity timeout — no requests for N min after parent death
78
+ // 3. Live project registry — hasLiveProjects() as secondary signal
79
+ //
80
+ // Parent alive + idle = fine (Claude may just be thinking).
81
+ // Parent dead + active requests = keep serving (another MCP may have connected).
82
+ // Parent dead + idle + no projects = self-terminate after grace period.
73
83
 
74
- const KYS_INTERVAL = 30000; // Check every 30s
75
- const KYS_IDLE_THRESHOLD = 120000; // 2 minutes without Claude → exit
76
- const KYS_GRACE_PERIOD = 60000; // 60s grace after startup
84
+ const KYS_INTERVAL = 30000; // Check every 30s
85
+ const ORPHAN_IDLE_TIMEOUT = 10 * 60 * 1000; // 10 min idle after parent death → exit
86
+ const ORPHAN_GRACE_PERIOD = 60000; // 60s grace no kill within first 60s of orphan
87
+ const STARTUP_GRACE_PERIOD = 60000; // 60s grace after daemon startup
77
88
 
78
- function isClaudeAlive() {
79
- // Check registry first — much cheaper than pgrep
80
- if (hasLiveProjects()) return true;
81
- // Fallback to pgrep for cases where MCP didn't register (old versions)
89
+ // Capture parent PID at module load — this is the spawner (MCP server / bootstrap)
90
+ const _parentPid = process.ppid;
91
+
92
+ let _orphanDetectedAt = 0; // 0 = not orphaned
93
+
94
+ /**
95
+ * Check if parent process (MCP server / bootstrap) is still alive.
96
+ * Uses process.kill(pid, 0) — throws ESRCH if process doesn't exist.
97
+ * Much more reliable than pgrep, and doesn't match unrelated "claude" processes.
98
+ */
99
+ function isParentAlive() {
100
+ if (!_parentPid || _parentPid <= 0) return false;
101
+ // If we've been reparented to init (PID 1), parent is definitely dead
102
+ if (process.ppid === 1 && _parentPid !== 1) return false;
82
103
  try {
83
- const result = execSync(
84
- 'pgrep -f "claude" 2>/dev/null',
85
- { encoding: 'utf8', timeout: 5000 }
86
- ).trim();
87
- return !!result;
88
- } catch {
89
- return false;
104
+ process.kill(_parentPid, 0);
105
+ return true;
106
+ } catch (e) {
107
+ // ESRCH = no such process → dead
108
+ // EPERM = process exists but no permission to signal → still alive
109
+ return e.code === 'EPERM';
110
+ }
111
+ }
112
+
113
+ /**
114
+ * Touch activity timestamp. Called on every incoming request.
115
+ */
116
+ function touchActivity() {
117
+ // If we got a request while orphaned, someone is still using us — reset countdown
118
+ if (_orphanDetectedAt > 0) {
119
+ log('info', `ORPHAN-WATCHDOG: request received while orphaned — resetting idle countdown`);
120
+ _orphanDetectedAt = 0;
90
121
  }
91
122
  }
92
123
 
124
+ /**
125
+ * Wrapped handleRequest that tracks activity for orphan detection.
126
+ */
127
+ function wrappedHandleRequest(req, res) {
128
+ touchActivity();
129
+ return handleRequest(req, res);
130
+ }
131
+
93
132
  // ============================================================================
94
133
  // Main
95
134
  // ============================================================================
@@ -107,12 +146,13 @@ async function main() {
107
146
  mkdirSync(CLAUDE_DIR, { recursive: true });
108
147
  }
109
148
 
110
- // Write PID file: pid:timestamp
111
- const pidData = `${process.pid}:${Date.now()}`;
149
+ // Write PID file: pid:timestamp:parentPid
150
+ const pidData = `${process.pid}:${Date.now()}:${_parentPid}`;
112
151
  writeFileSync(PID_FILE, pidData, { mode: 0o644 });
152
+ log('info', `Daemon: wrote PID file (pid=${process.pid}, parent=${_parentPid})`);
113
153
 
114
- // Create HTTP server using the shared handleRequest from compactionProxy.js
115
- const server = createServer(handleRequest);
154
+ // Create HTTP server using wrapped handleRequest for activity tracking
155
+ const server = createServer(wrappedHandleRequest);
116
156
  server.keepAliveTimeout = 300000;
117
157
  server.headersTimeout = 310000;
118
158
 
@@ -131,9 +171,9 @@ async function main() {
131
171
  // If disabled flag existed at startup, start in passthrough mode
132
172
  if (startPaused) {
133
173
  setPaused(true);
134
- log('info', `Daemon started in PASSTHROUGH mode (disabled flag): PID=${process.pid} port=${PROXY_PORT}`);
174
+ log('info', `Daemon started in PASSTHROUGH mode (disabled flag): PID=${process.pid} port=${PROXY_PORT} parent=${_parentPid}`);
135
175
  } else {
136
- log('info', `Daemon started: PID=${process.pid} port=${PROXY_PORT}`);
176
+ log('info', `Daemon started: PID=${process.pid} port=${PROXY_PORT} parent=${_parentPid}`);
137
177
  }
138
178
  // Start the stale project reaper
139
179
  startReaper();
@@ -162,27 +202,62 @@ async function main() {
162
202
  log('info', `Daemon: SIGUSR1 → proxy ${paused ? 'PAUSED' : 'RESUMED'}`);
163
203
  });
164
204
 
165
- // --- KYS Watchdog ---
205
+ // --- Orphan Detection & KYS Watchdog ---
166
206
  const startedAt = Date.now();
167
- let lastClaudeSeen = Date.now();
207
+
208
+ function gracefulShutdown(reason) {
209
+ log('warn', `ORPHAN-WATCHDOG: ${reason}`);
210
+ cleanup();
211
+ server.close(() => process.exit(0));
212
+ // Force exit after 5s if server.close() hangs
213
+ setTimeout(() => process.exit(0), 5000).unref();
214
+ }
168
215
 
169
216
  const kysTimer = setInterval(() => {
170
- // Grace period — don't kill right after startup
171
- if (Date.now() - startedAt < KYS_GRACE_PERIOD) return;
217
+ // Startup grace period — don't kill right after spawn
218
+ if (Date.now() - startedAt < STARTUP_GRACE_PERIOD) return;
219
+
220
+ const parentAlive = isParentAlive();
221
+ const idleMs = Date.now() - getLastRequestTime();
222
+ const hasProjects = hasLiveProjects();
172
223
 
173
- if (isClaudeAlive()) {
174
- lastClaudeSeen = Date.now();
224
+ if (parentAlive) {
225
+ // Parent alive — everything is fine, reset orphan state if needed
226
+ if (_orphanDetectedAt > 0) {
227
+ log('info', 'ORPHAN-WATCHDOG: parent alive again (re-adopted?) — clearing orphan state');
228
+ _orphanDetectedAt = 0;
229
+ }
175
230
  return;
176
231
  }
177
232
 
178
- const idleMs = Date.now() - lastClaudeSeen;
179
- if (idleMs >= KYS_IDLE_THRESHOLD) {
180
- log('info', `Daemon: No Claude for ${(idleMs / 1000).toFixed(0)}s — self-terminating`);
181
- cleanup();
182
- server.close(() => process.exit(0));
183
- // Force exit after 5s if server.close() hangs
184
- setTimeout(() => process.exit(0), 5000).unref();
233
+ // --- Parent is dead ---
234
+
235
+ if (_orphanDetectedAt === 0) {
236
+ // First detection of orphan state
237
+ _orphanDetectedAt = Date.now();
238
+ log('warn', `ORPHAN-WATCHDOG: parent PID ${_parentPid} is dead — idle=${Math.round(idleMs / 1000)}s, projects=${hasProjects}`);
239
+ }
240
+
241
+ const orphanDuration = Date.now() - _orphanDetectedAt;
242
+
243
+ // Case 1: Parent dead + idle for ORPHAN_IDLE_TIMEOUT → definitely a zombie
244
+ if (idleMs >= ORPHAN_IDLE_TIMEOUT) {
245
+ gracefulShutdown(
246
+ `parent dead + no requests for ${Math.round(idleMs / 1000)}s — self-terminating`
247
+ );
248
+ return;
185
249
  }
250
+
251
+ // Case 2: Parent dead + no live projects + grace period elapsed → nothing to serve
252
+ if (!hasProjects && orphanDuration >= ORPHAN_GRACE_PERIOD) {
253
+ gracefulShutdown(
254
+ `parent dead + no live projects for ${Math.round(orphanDuration / 1000)}s — self-terminating`
255
+ );
256
+ return;
257
+ }
258
+
259
+ // Case 3: Parent dead but still getting requests or has live projects — keep serving
260
+ // (Another MCP server may have connected, or requests still flowing)
186
261
  }, KYS_INTERVAL);
187
262
  kysTimer.unref();
188
263
  }
@@ -0,0 +1,439 @@
1
+ /**
2
+ * Context Vault — stash house for thicc tool outputs
3
+ * ===================================================
4
+ *
5
+ * when tool results are bussin a little TOO hard (>5KB),
6
+ * we chunk em, index with tsvector, and hand back a receipt.
7
+ * claude can dig for specifics later without burning tokens.
8
+ *
9
+ * 315KB tool result → ~800 byte receipt = 99.7% token reduction. it slaps.
10
+ *
11
+ * inspired by claude-context-mode, rebuilt from scratch on postgres.
12
+ * no sqlite, no cap.
13
+ *
14
+ * @author hardwicksoftwareservices
15
+ * @website https://justcalljon.pro
16
+ */
17
+
18
+ import { createHash } from 'crypto';
19
+ import { logger } from '../utils/logger.js';
20
+
21
+ // --- Config ---
22
+ const VAULT_THRESHOLD = 5000; // chars — auto-stash anything bigger
23
+ const CHUNK_TARGET_SIZE = 1500; // chars per chunk (target, not hard limit)
24
+ const CHUNK_MAX_SIZE = 4000; // hard cap — code blocks can be big
25
+ const CHUNK_MIN_SIZE = 100; // merge anything smaller
26
+ const VAULT_TTL_HOURS = 24; // auto-expire after 24h
27
+ const MAX_VOCAB_TERMS = 30; // vocabulary terms in the receipt
28
+ const PREVIEW_CHARS = 300; // preview length in receipt
29
+
30
+ // tools that should NEVER get auto-vaulted (prevent recursion)
31
+ const VAULT_SKIP_TOOLS = new Set([
32
+ 'stash_the_goods',
33
+ 'dig_in_the_vault',
34
+ ]);
35
+
36
+ // english stopwords — filtered from vocabulary extraction
37
+ const STOPWORDS = new Set([
38
+ 'the','be','to','of','and','a','in','that','have','i','it','for','not','on',
39
+ 'with','he','as','you','do','at','this','but','his','by','from','they','we',
40
+ 'say','her','she','or','an','will','my','one','all','would','there','their',
41
+ 'what','so','up','out','if','about','who','get','which','go','me','when',
42
+ 'make','can','like','time','no','just','him','know','take','people','into',
43
+ 'year','your','good','some','could','them','see','other','than','then','now',
44
+ 'look','only','come','its','over','think','also','back','after','use','two',
45
+ 'how','our','work','first','well','way','even','new','want','because','any',
46
+ 'these','give','day','most','us','is','are','was','were','been','has','had',
47
+ 'did','does','done','may','might','shall','should','must','need','could',
48
+ 'would','will','true','false','null','undefined','function','return','const',
49
+ 'let','var','import','export','from','class','async','await','try','catch',
50
+ 'throw','typeof','instanceof','void','this','super','extends',
51
+ 'type','string','number','boolean','object','array',
52
+ ]);
53
+
54
+ let _pool = null;
55
+ let _tableReady = false;
56
+ let _projectPath = '/';
57
+
58
+ // --- Init ---
59
+ export function initContextVault(pool, projectPath) {
60
+ _pool = pool;
61
+ _projectPath = projectPath || '/';
62
+ }
63
+
64
+ export function setVaultProjectPath(p) { _projectPath = p || '/'; }
65
+
66
+ export { VAULT_THRESHOLD, VAULT_SKIP_TOOLS };
67
+
68
+ // --- Table creation (fallback if migration 38 hasn't run yet) ---
69
+ async function ensureTable() {
70
+ if (_tableReady || !_pool) return;
71
+ try {
72
+ await _pool.query(`
73
+ CREATE TABLE IF NOT EXISTS context_vault (
74
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
75
+ vault_id VARCHAR(16) NOT NULL,
76
+ chunk_idx INTEGER NOT NULL DEFAULT 0,
77
+ content TEXT NOT NULL,
78
+ content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', content)) STORED,
79
+ source_tool VARCHAR(128),
80
+ source_size INTEGER DEFAULT 0,
81
+ metadata JSONB DEFAULT '{}',
82
+ project_path VARCHAR(500) DEFAULT '/',
83
+ stashed_at TIMESTAMPTZ DEFAULT NOW(),
84
+ expires_at TIMESTAMPTZ DEFAULT (NOW() + INTERVAL '24 hours'),
85
+ UNIQUE(vault_id, chunk_idx)
86
+ );
87
+ CREATE INDEX IF NOT EXISTS idx_ctx_vault_tsv ON context_vault USING GIN(content_tsv);
88
+ CREATE INDEX IF NOT EXISTS idx_ctx_vault_id ON context_vault(vault_id);
89
+ CREATE INDEX IF NOT EXISTS idx_ctx_vault_expires ON context_vault(expires_at);
90
+ CREATE INDEX IF NOT EXISTS idx_ctx_vault_project ON context_vault(project_path);
91
+ `);
92
+ _tableReady = true;
93
+ } catch (err) {
94
+ // table might already exist from migration 38 — that's fine
95
+ if (err.message?.includes('already exists') || err.code === '42P07') {
96
+ _tableReady = true;
97
+ } else {
98
+ logger.warn({ error: err.message }, 'context_vault ensureTable failed');
99
+ }
100
+ }
101
+ }
102
+
103
+ // =============================================================================
104
+ // Chunking — split content by natural boundaries, keep code blocks intact
105
+ // =============================================================================
106
+
107
+ function chunkContent(content) {
108
+ if (content.length <= CHUNK_TARGET_SIZE) return [content];
109
+
110
+ // Phase 1: protect code blocks from splitting
111
+ const codeBlocks = [];
112
+ const withPlaceholders = content.replace(/```[\s\S]*?```/g, (match) => {
113
+ const idx = codeBlocks.length;
114
+ codeBlocks.push(match);
115
+ return `\n__CB${idx}__\n`;
116
+ });
117
+
118
+ // Phase 2: split by markdown headings first, then paragraphs
119
+ const rawParts = withPlaceholders.split(/(?=^#{1,4}\s)/m);
120
+
121
+ const expandedParts = [];
122
+ for (const part of rawParts) {
123
+ if (part.length <= CHUNK_TARGET_SIZE) {
124
+ expandedParts.push(part);
125
+ } else {
126
+ // long section — split by paragraphs
127
+ const paragraphs = part.split(/\n{2,}/);
128
+ expandedParts.push(...paragraphs);
129
+ }
130
+ }
131
+
132
+ // Phase 3: merge small parts, split oversized ones
133
+ const chunks = [];
134
+ let buffer = '';
135
+
136
+ for (let raw of expandedParts) {
137
+ // restore code blocks in this part
138
+ raw = raw.replace(/__CB(\d+)__/g, (_, idx) => codeBlocks[parseInt(idx)] || '');
139
+
140
+ if (!raw.trim()) continue;
141
+
142
+ if (buffer.length + raw.length + 2 <= CHUNK_TARGET_SIZE) {
143
+ buffer += (buffer ? '\n\n' : '') + raw;
144
+ } else {
145
+ if (buffer.trim()) chunks.push(buffer.trim());
146
+
147
+ if (raw.length > CHUNK_MAX_SIZE) {
148
+ // split oversized by sentences
149
+ const sentences = raw.match(/[^.!?\n]+[.!?\n]+/g) || [raw];
150
+ let sentBuf = '';
151
+ for (const sent of sentences) {
152
+ if (sentBuf.length + sent.length > CHUNK_TARGET_SIZE && sentBuf) {
153
+ chunks.push(sentBuf.trim());
154
+ sentBuf = '';
155
+ }
156
+ sentBuf += sent;
157
+ }
158
+ buffer = sentBuf;
159
+ } else {
160
+ buffer = raw;
161
+ }
162
+ }
163
+ }
164
+ if (buffer.trim()) chunks.push(buffer.trim());
165
+
166
+ // Phase 4: merge tiny chunks into neighbors
167
+ const merged = [];
168
+ let mergeBuffer = '';
169
+ for (const chunk of chunks) {
170
+ if (chunk.length < CHUNK_MIN_SIZE && mergeBuffer.length + chunk.length < CHUNK_TARGET_SIZE) {
171
+ mergeBuffer += (mergeBuffer ? '\n\n' : '') + chunk;
172
+ } else {
173
+ if (mergeBuffer) {
174
+ if (mergeBuffer.length < CHUNK_MIN_SIZE && merged.length > 0) {
175
+ merged[merged.length - 1] += '\n\n' + mergeBuffer;
176
+ } else {
177
+ merged.push(mergeBuffer);
178
+ }
179
+ mergeBuffer = '';
180
+ }
181
+ merged.push(chunk);
182
+ }
183
+ }
184
+ if (mergeBuffer) {
185
+ if (merged.length > 0) {
186
+ merged[merged.length - 1] += '\n\n' + mergeBuffer;
187
+ } else {
188
+ merged.push(mergeBuffer);
189
+ }
190
+ }
191
+
192
+ return merged.filter(c => c.length > 0);
193
+ }
194
+
195
+ // =============================================================================
196
+ // Vocabulary extraction — top N terms by frequency, excluding stopwords
197
+ // =============================================================================
198
+
199
+ function extractVocabulary(content) {
200
+ const words = content
201
+ .toLowerCase()
202
+ .replace(/[^a-z0-9_\s-]/g, ' ')
203
+ .split(/\s+/)
204
+ .filter(w => w.length > 2 && w.length < 30 && !STOPWORDS.has(w));
205
+
206
+ const freq = {};
207
+ for (const w of words) {
208
+ freq[w] = (freq[w] || 0) + 1;
209
+ }
210
+
211
+ return Object.entries(freq)
212
+ .sort((a, b) => b[1] - a[1])
213
+ .slice(0, MAX_VOCAB_TERMS)
214
+ .map(([word]) => word);
215
+ }
216
+
217
+ // =============================================================================
218
+ // Stash — chunk + index + store
219
+ // =============================================================================
220
+
221
+ export async function stashTheGoods(content, metadata = {}) {
222
+ if (!_pool) throw new Error('context vault not initialized — call initContextVault first');
223
+ await ensureTable();
224
+
225
+ // Reap expired stashes (piggyback on stash calls — no separate timer needed)
226
+ reapExpired().catch(() => {});
227
+
228
+ // vault_id from content hash — idempotent, same content = same ID = no dupes
229
+ const vaultId = createHash('sha256').update(content).digest('hex').slice(0, 12);
230
+
231
+ // Check if already stashed (idempotent dedup)
232
+ try {
233
+ const existing = await _pool.query(
234
+ 'SELECT COUNT(*)::int as cnt FROM context_vault WHERE vault_id = $1',
235
+ [vaultId]
236
+ );
237
+ if (existing.rows[0]?.cnt > 0) {
238
+ const vocab = extractVocabulary(content);
239
+ return {
240
+ vaultId,
241
+ chunks: existing.rows[0].cnt,
242
+ totalChars: content.length,
243
+ preview: content.slice(0, PREVIEW_CHARS),
244
+ vocabulary: vocab,
245
+ deduplicated: true,
246
+ };
247
+ }
248
+ } catch (checkErr) {
249
+ // table might not exist yet — ensureTable should have handled it
250
+ logger.warn({ error: checkErr.message }, 'vault dedup check failed');
251
+ }
252
+
253
+ // Chunk the content
254
+ const chunks = chunkContent(content);
255
+ const vocab = extractVocabulary(content);
256
+
257
+ // Batch insert chunks
258
+ for (let i = 0; i < chunks.length; i++) {
259
+ try {
260
+ await _pool.query(
261
+ `INSERT INTO context_vault (vault_id, chunk_idx, content, source_tool, source_size, metadata, project_path, expires_at)
262
+ VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, NOW() + make_interval(hours => $8))
263
+ ON CONFLICT (vault_id, chunk_idx) DO NOTHING`,
264
+ [
265
+ vaultId,
266
+ i,
267
+ chunks[i],
268
+ metadata.tool || 'unknown',
269
+ content.length,
270
+ JSON.stringify(metadata),
271
+ _projectPath,
272
+ VAULT_TTL_HOURS,
273
+ ]
274
+ );
275
+ } catch (insertErr) {
276
+ logger.warn({ error: insertErr.message, chunk: i }, 'vault chunk insert failed');
277
+ }
278
+ }
279
+
280
+ logger.info({
281
+ vaultId,
282
+ chunks: chunks.length,
283
+ totalChars: content.length,
284
+ tool: metadata.tool,
285
+ }, `vault: stashed ${chunks.length} chunks (${content.length} chars)`);
286
+
287
+ return {
288
+ vaultId,
289
+ chunks: chunks.length,
290
+ totalChars: content.length,
291
+ preview: content.slice(0, PREVIEW_CHARS),
292
+ vocabulary: vocab,
293
+ deduplicated: false,
294
+ };
295
+ }
296
+
297
+ // =============================================================================
298
+ // Search — BM25-ranked retrieval via tsvector
299
+ // =============================================================================
300
+
301
+ export async function digInTheVault(query, vaultId = null, limit = 10) {
302
+ if (!_pool) throw new Error('context vault not initialized');
303
+ await ensureTable();
304
+
305
+ let sql, params;
306
+
307
+ if (vaultId) {
308
+ // Search within a specific stash
309
+ sql = `
310
+ SELECT vault_id, chunk_idx, content,
311
+ ts_rank(content_tsv, plainto_tsquery('english', $1)) as rank
312
+ FROM context_vault
313
+ WHERE vault_id = $2
314
+ AND content_tsv @@ plainto_tsquery('english', $1)
315
+ AND expires_at > NOW()
316
+ ORDER BY rank DESC
317
+ LIMIT $3
318
+ `;
319
+ params = [query, vaultId, limit];
320
+ } else {
321
+ // Search across all stashes for this project
322
+ sql = `
323
+ SELECT vault_id, chunk_idx, content,
324
+ ts_rank(content_tsv, plainto_tsquery('english', $1)) as rank
325
+ FROM context_vault
326
+ WHERE content_tsv @@ plainto_tsquery('english', $1)
327
+ AND project_path = $2
328
+ AND expires_at > NOW()
329
+ ORDER BY rank DESC
330
+ LIMIT $3
331
+ `;
332
+ params = [query, _projectPath, limit];
333
+ }
334
+
335
+ const { rows } = await _pool.query(sql, params);
336
+
337
+ // If tsvector found nothing and we have a vault_id, fall back to ILIKE
338
+ if (rows.length === 0 && vaultId) {
339
+ const fallback = await _pool.query(
340
+ `SELECT vault_id, chunk_idx, content, 0.0::float as rank
341
+ FROM context_vault
342
+ WHERE vault_id = $1 AND content ILIKE '%' || $2 || '%' AND expires_at > NOW()
343
+ ORDER BY chunk_idx ASC LIMIT $3`,
344
+ [vaultId, query, limit]
345
+ );
346
+ return fallback.rows;
347
+ }
348
+
349
+ return rows;
350
+ }
351
+
352
+ // =============================================================================
353
+ // Get full stash — all chunks ordered for complete retrieval
354
+ // =============================================================================
355
+
356
+ export async function getFullStash(vaultId) {
357
+ if (!_pool) throw new Error('context vault not initialized');
358
+ await ensureTable();
359
+
360
+ const { rows } = await _pool.query(
361
+ `SELECT chunk_idx, content FROM context_vault
362
+ WHERE vault_id = $1 AND expires_at > NOW()
363
+ ORDER BY chunk_idx ASC`,
364
+ [vaultId]
365
+ );
366
+
367
+ if (rows.length === 0) return null;
368
+ return rows.map(r => r.content).join('\n\n');
369
+ }
370
+
371
+ // =============================================================================
372
+ // Stats — what's in the vault right now
373
+ // =============================================================================
374
+
375
+ export async function getVaultStats() {
376
+ if (!_pool) return { totalStashes: 0, totalChunks: 0, totalChars: 0 };
377
+ await ensureTable();
378
+
379
+ try {
380
+ const { rows } = await _pool.query(`
381
+ SELECT
382
+ COUNT(DISTINCT vault_id)::int as stash_count,
383
+ COUNT(*)::int as chunk_count,
384
+ COALESCE(SUM(LENGTH(content)), 0)::int as total_chars,
385
+ MIN(stashed_at) as oldest
386
+ FROM context_vault
387
+ WHERE project_path = $1 AND expires_at > NOW()
388
+ `, [_projectPath]);
389
+
390
+ return {
391
+ totalStashes: rows[0]?.stash_count || 0,
392
+ totalChunks: rows[0]?.chunk_count || 0,
393
+ totalChars: rows[0]?.total_chars || 0,
394
+ oldestStash: rows[0]?.oldest || null,
395
+ };
396
+ } catch (err) {
397
+ return { totalStashes: 0, totalChunks: 0, totalChars: 0, error: err.message };
398
+ }
399
+ }
400
+
401
+ // =============================================================================
402
+ // Reaper — clean expired stashes (runs piggyback on stash operations)
403
+ // =============================================================================
404
+
405
+ async function reapExpired() {
406
+ if (!_pool) return;
407
+ try {
408
+ const result = await _pool.query('DELETE FROM context_vault WHERE expires_at < NOW()');
409
+ if (result.rowCount > 0) {
410
+ logger.info({ reaped: result.rowCount }, 'vault: reaped expired stashes');
411
+ }
412
+ } catch {
413
+ // non-fatal, swallow
414
+ }
415
+ }
416
+
417
+ // =============================================================================
418
+ // Receipt formatter — compact reference that replaces the full output
419
+ // =============================================================================
420
+
421
+ export function formatVaultReceipt(stashResult) {
422
+ const { vaultId, chunks, totalChars, preview, vocabulary, deduplicated } = stashResult;
423
+ const receiptSize = PREVIEW_CHARS + 200 + (vocabulary.length * 8);
424
+ const savings = ((1 - (receiptSize / totalChars)) * 100).toFixed(1);
425
+
426
+ return [
427
+ `<vault_receipt id="${vaultId}" chunks="${chunks}" total_chars="${totalChars}" savings="${savings}%"${deduplicated ? ' dedup="true"' : ''}>`,
428
+ `Content stashed (${totalChars.toLocaleString()} chars chunked into ${chunks} searchable blocks)`,
429
+ ``,
430
+ `Preview:`,
431
+ preview + (totalChars > PREVIEW_CHARS ? '...' : ''),
432
+ ``,
433
+ `Key terms: ${vocabulary.join(', ')}`,
434
+ ``,
435
+ `Retrieve: dig_in_the_vault(query: "search terms", vault_id: "${vaultId}")`,
436
+ `Full dump: dig_in_the_vault(vault_id: "${vaultId}", get_all: true)`,
437
+ `</vault_receipt>`,
438
+ ].join('\n');
439
+ }