specmem-hardwicksoftware 3.7.35 → 3.7.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +11 -15
- package/bin/specmem-console.cjs +839 -51
- package/claude-hooks/agent-chooser-hook.js +6 -6
- package/claude-hooks/agent-loading-hook.cjs +16 -16
- package/claude-hooks/agent-loading-hook.js +18 -18
- package/claude-hooks/agent-type-matcher.js +1 -1
- package/claude-hooks/background-completion-silencer.js +1 -1
- package/claude-hooks/file-claim-enforcer.cjs +37 -36
- package/claude-hooks/output-cleaner.cjs +1 -1
- package/claude-hooks/settings.json +27 -3
- package/claude-hooks/specmem-search-enforcer.cjs +2 -11
- package/claude-hooks/specmem-team-member-inject.js +1 -1
- package/claude-hooks/specmem-unified-hook.py +1 -1
- package/claude-hooks/subagent-loading-hook.cjs +1 -1
- package/claude-hooks/task-progress-hook.cjs +7 -7
- package/claude-hooks/task-progress-hook.js +3 -3
- package/claude-hooks/team-comms-enforcer.cjs +49 -47
- package/dist/claude-sessions/sessionParser.js +5 -0
- package/dist/codebase/codebaseIndexer.js +48 -17
- package/dist/codebase/exclusions.js +3 -4
- package/dist/codebase/index.js +4 -0
- package/dist/codebase/pdfExtractor.js +298 -0
- package/dist/dashboard/api/taskTeamMembers.js +2 -2
- package/dist/db/bigBrainMigrations.js +29 -0
- package/dist/hooks/hookManager.js +4 -4
- package/dist/hooks/teamFramingCli.js +1 -1
- package/dist/hooks/teamMemberPrepromptHook.js +5 -5
- package/dist/init/claudeConfigInjector.js +2 -2
- package/dist/mcp/compactionProxy.js +834 -186
- package/dist/mcp/compactionProxyDaemon.js +112 -37
- package/dist/mcp/contextVault.js +439 -0
- package/dist/mcp/embeddingServerManager.js +61 -1
- package/dist/mcp/mcpProtocolHandler.js +6 -1
- package/dist/mcp/miniCOTServerManager.js +82 -8
- package/dist/mcp/specMemServer.js +45 -10
- package/dist/mcp/toolRegistry.js +6 -0
- package/dist/startup/startupIndexing.js +14 -0
- package/dist/team-members/taskOrchestrator.js +3 -3
- package/dist/team-members/taskTeamMemberLogger.js +2 -2
- package/dist/tools/goofy/deployTeamMember.js +3 -3
- package/dist/tools/goofy/digInTheVault.js +81 -0
- package/dist/tools/goofy/stashTheGoods.js +56 -0
- package/dist/tools/teamMemberDeployer.js +2 -2
- package/dist/watcher/changeHandler.js +65 -8
- package/dist/watcher/changeQueue.js +20 -1
- package/embedding-sandbox/mini-cot-service.py +11 -13
- package/embedding-sandbox/pdf-text-extract.py +208 -0
- package/package.json +1 -1
- package/scripts/deploy-hooks.cjs +2 -2
- package/scripts/global-postinstall.cjs +2 -2
- package/scripts/specmem-init.cjs +130 -36
- package/specmem/model-config.json +6 -6
- package/specmem/supervisord.conf +1 -1
- package/svg-sections/readme-token-compaction.svg +246 -0
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Compaction Proxy Daemon — Standalone Persistent Process
|
|
4
4
|
* ========================================================
|
|
5
5
|
*
|
|
6
|
-
* Survives MCP server restarts. Self-terminates when
|
|
6
|
+
* Survives MCP server restarts. Self-terminates when parent is gone.
|
|
7
7
|
*
|
|
8
8
|
* Spawned by startCompactionProxy() in compactionProxy.js.
|
|
9
9
|
* Imports the core handleRequest and all compression logic — zero duplication.
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
* 1. MCP starts → startCompactionProxy() spawns this daemon (detached)
|
|
13
13
|
* 2. MCP dies/restarts → daemon stays alive, port file persists
|
|
14
14
|
* 3. Next MCP start → detects running daemon via PID file, skips spawn
|
|
15
|
-
* 4.
|
|
15
|
+
* 4. Parent exits → orphan watchdog detects dead parent PID, starts idle countdown
|
|
16
|
+
* 5. No requests for ORPHAN_IDLE_TIMEOUT → graceful self-termination
|
|
16
17
|
*
|
|
17
18
|
* @author hardwicksoftwareservices
|
|
18
19
|
* @website https://justcalljon.pro
|
|
@@ -20,7 +21,6 @@
|
|
|
20
21
|
|
|
21
22
|
import { createServer } from 'http';
|
|
22
23
|
import { writeFileSync, unlinkSync, existsSync, mkdirSync } from 'fs';
|
|
23
|
-
import { execSync } from 'child_process';
|
|
24
24
|
import { createConnection } from 'net';
|
|
25
25
|
import {
|
|
26
26
|
handleRequest,
|
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
PID_FILE,
|
|
35
35
|
DISABLED_FILE,
|
|
36
36
|
CLAUDE_DIR,
|
|
37
|
+
getLastRequestTime,
|
|
37
38
|
} from './compactionProxy.js';
|
|
38
39
|
|
|
39
40
|
// ============================================================================
|
|
@@ -68,28 +69,66 @@ function cleanup() {
|
|
|
68
69
|
}
|
|
69
70
|
|
|
70
71
|
// ============================================================================
|
|
71
|
-
//
|
|
72
|
+
// Orphan Detection & KYS Watchdog
|
|
72
73
|
// ============================================================================
|
|
74
|
+
//
|
|
75
|
+
// Three-layer orphan detection:
|
|
76
|
+
// 1. Parent PID monitoring — process.kill(ppid, 0) every 30s
|
|
77
|
+
// 2. Socket activity timeout — no requests for N min after parent death
|
|
78
|
+
// 3. Live project registry — hasLiveProjects() as secondary signal
|
|
79
|
+
//
|
|
80
|
+
// Parent alive + idle = fine (Claude may just be thinking).
|
|
81
|
+
// Parent dead + active requests = keep serving (another MCP may have connected).
|
|
82
|
+
// Parent dead + idle + no projects = self-terminate after grace period.
|
|
73
83
|
|
|
74
|
-
const KYS_INTERVAL = 30000;
|
|
75
|
-
const
|
|
76
|
-
const
|
|
84
|
+
const KYS_INTERVAL = 30000; // Check every 30s
|
|
85
|
+
const ORPHAN_IDLE_TIMEOUT = 10 * 60 * 1000; // 10 min idle after parent death → exit
|
|
86
|
+
const ORPHAN_GRACE_PERIOD = 60000; // 60s grace — no kill within first 60s of orphan
|
|
87
|
+
const STARTUP_GRACE_PERIOD = 60000; // 60s grace after daemon startup
|
|
77
88
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
89
|
+
// Capture parent PID at module load — this is the spawner (MCP server / bootstrap)
|
|
90
|
+
const _parentPid = process.ppid;
|
|
91
|
+
|
|
92
|
+
let _orphanDetectedAt = 0; // 0 = not orphaned
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Check if parent process (MCP server / bootstrap) is still alive.
|
|
96
|
+
* Uses process.kill(pid, 0) — throws ESRCH if process doesn't exist.
|
|
97
|
+
* Much more reliable than pgrep, and doesn't match unrelated "claude" processes.
|
|
98
|
+
*/
|
|
99
|
+
function isParentAlive() {
|
|
100
|
+
if (!_parentPid || _parentPid <= 0) return false;
|
|
101
|
+
// If we've been reparented to init (PID 1), parent is definitely dead
|
|
102
|
+
if (process.ppid === 1 && _parentPid !== 1) return false;
|
|
82
103
|
try {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
104
|
+
process.kill(_parentPid, 0);
|
|
105
|
+
return true;
|
|
106
|
+
} catch (e) {
|
|
107
|
+
// ESRCH = no such process → dead
|
|
108
|
+
// EPERM = process exists but no permission to signal → still alive
|
|
109
|
+
return e.code === 'EPERM';
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Touch activity timestamp. Called on every incoming request.
|
|
115
|
+
*/
|
|
116
|
+
function touchActivity() {
|
|
117
|
+
// If we got a request while orphaned, someone is still using us — reset countdown
|
|
118
|
+
if (_orphanDetectedAt > 0) {
|
|
119
|
+
log('info', `ORPHAN-WATCHDOG: request received while orphaned — resetting idle countdown`);
|
|
120
|
+
_orphanDetectedAt = 0;
|
|
90
121
|
}
|
|
91
122
|
}
|
|
92
123
|
|
|
124
|
+
/**
|
|
125
|
+
* Wrapped handleRequest that tracks activity for orphan detection.
|
|
126
|
+
*/
|
|
127
|
+
function wrappedHandleRequest(req, res) {
|
|
128
|
+
touchActivity();
|
|
129
|
+
return handleRequest(req, res);
|
|
130
|
+
}
|
|
131
|
+
|
|
93
132
|
// ============================================================================
|
|
94
133
|
// Main
|
|
95
134
|
// ============================================================================
|
|
@@ -107,12 +146,13 @@ async function main() {
|
|
|
107
146
|
mkdirSync(CLAUDE_DIR, { recursive: true });
|
|
108
147
|
}
|
|
109
148
|
|
|
110
|
-
// Write PID file: pid:timestamp
|
|
111
|
-
const pidData = `${process.pid}:${Date.now()}`;
|
|
149
|
+
// Write PID file: pid:timestamp:parentPid
|
|
150
|
+
const pidData = `${process.pid}:${Date.now()}:${_parentPid}`;
|
|
112
151
|
writeFileSync(PID_FILE, pidData, { mode: 0o644 });
|
|
152
|
+
log('info', `Daemon: wrote PID file (pid=${process.pid}, parent=${_parentPid})`);
|
|
113
153
|
|
|
114
|
-
// Create HTTP server using
|
|
115
|
-
const server = createServer(
|
|
154
|
+
// Create HTTP server using wrapped handleRequest for activity tracking
|
|
155
|
+
const server = createServer(wrappedHandleRequest);
|
|
116
156
|
server.keepAliveTimeout = 300000;
|
|
117
157
|
server.headersTimeout = 310000;
|
|
118
158
|
|
|
@@ -131,9 +171,9 @@ async function main() {
|
|
|
131
171
|
// If disabled flag existed at startup, start in passthrough mode
|
|
132
172
|
if (startPaused) {
|
|
133
173
|
setPaused(true);
|
|
134
|
-
log('info', `Daemon started in PASSTHROUGH mode (disabled flag): PID=${process.pid} port=${PROXY_PORT}`);
|
|
174
|
+
log('info', `Daemon started in PASSTHROUGH mode (disabled flag): PID=${process.pid} port=${PROXY_PORT} parent=${_parentPid}`);
|
|
135
175
|
} else {
|
|
136
|
-
log('info', `Daemon started: PID=${process.pid} port=${PROXY_PORT}`);
|
|
176
|
+
log('info', `Daemon started: PID=${process.pid} port=${PROXY_PORT} parent=${_parentPid}`);
|
|
137
177
|
}
|
|
138
178
|
// Start the stale project reaper
|
|
139
179
|
startReaper();
|
|
@@ -162,27 +202,62 @@ async function main() {
|
|
|
162
202
|
log('info', `Daemon: SIGUSR1 → proxy ${paused ? 'PAUSED' : 'RESUMED'}`);
|
|
163
203
|
});
|
|
164
204
|
|
|
165
|
-
// --- KYS Watchdog ---
|
|
205
|
+
// --- Orphan Detection & KYS Watchdog ---
|
|
166
206
|
const startedAt = Date.now();
|
|
167
|
-
|
|
207
|
+
|
|
208
|
+
function gracefulShutdown(reason) {
|
|
209
|
+
log('warn', `ORPHAN-WATCHDOG: ${reason}`);
|
|
210
|
+
cleanup();
|
|
211
|
+
server.close(() => process.exit(0));
|
|
212
|
+
// Force exit after 5s if server.close() hangs
|
|
213
|
+
setTimeout(() => process.exit(0), 5000).unref();
|
|
214
|
+
}
|
|
168
215
|
|
|
169
216
|
const kysTimer = setInterval(() => {
|
|
170
|
-
//
|
|
171
|
-
if (Date.now() - startedAt <
|
|
217
|
+
// Startup grace period — don't kill right after spawn
|
|
218
|
+
if (Date.now() - startedAt < STARTUP_GRACE_PERIOD) return;
|
|
219
|
+
|
|
220
|
+
const parentAlive = isParentAlive();
|
|
221
|
+
const idleMs = Date.now() - getLastRequestTime();
|
|
222
|
+
const hasProjects = hasLiveProjects();
|
|
172
223
|
|
|
173
|
-
if (
|
|
174
|
-
|
|
224
|
+
if (parentAlive) {
|
|
225
|
+
// Parent alive — everything is fine, reset orphan state if needed
|
|
226
|
+
if (_orphanDetectedAt > 0) {
|
|
227
|
+
log('info', 'ORPHAN-WATCHDOG: parent alive again (re-adopted?) — clearing orphan state');
|
|
228
|
+
_orphanDetectedAt = 0;
|
|
229
|
+
}
|
|
175
230
|
return;
|
|
176
231
|
}
|
|
177
232
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
233
|
+
// --- Parent is dead ---
|
|
234
|
+
|
|
235
|
+
if (_orphanDetectedAt === 0) {
|
|
236
|
+
// First detection of orphan state
|
|
237
|
+
_orphanDetectedAt = Date.now();
|
|
238
|
+
log('warn', `ORPHAN-WATCHDOG: parent PID ${_parentPid} is dead — idle=${Math.round(idleMs / 1000)}s, projects=${hasProjects}`);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
const orphanDuration = Date.now() - _orphanDetectedAt;
|
|
242
|
+
|
|
243
|
+
// Case 1: Parent dead + idle for ORPHAN_IDLE_TIMEOUT → definitely a zombie
|
|
244
|
+
if (idleMs >= ORPHAN_IDLE_TIMEOUT) {
|
|
245
|
+
gracefulShutdown(
|
|
246
|
+
`parent dead + no requests for ${Math.round(idleMs / 1000)}s — self-terminating`
|
|
247
|
+
);
|
|
248
|
+
return;
|
|
185
249
|
}
|
|
250
|
+
|
|
251
|
+
// Case 2: Parent dead + no live projects + grace period elapsed → nothing to serve
|
|
252
|
+
if (!hasProjects && orphanDuration >= ORPHAN_GRACE_PERIOD) {
|
|
253
|
+
gracefulShutdown(
|
|
254
|
+
`parent dead + no live projects for ${Math.round(orphanDuration / 1000)}s — self-terminating`
|
|
255
|
+
);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// Case 3: Parent dead but still getting requests or has live projects — keep serving
|
|
260
|
+
// (Another MCP server may have connected, or requests still flowing)
|
|
186
261
|
}, KYS_INTERVAL);
|
|
187
262
|
kysTimer.unref();
|
|
188
263
|
}
|
|
@@ -0,0 +1,439 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context Vault — stash house for thicc tool outputs
|
|
3
|
+
* ===================================================
|
|
4
|
+
*
|
|
5
|
+
* when tool results are bussin a little TOO hard (>5KB),
|
|
6
|
+
* we chunk em, index with tsvector, and hand back a receipt.
|
|
7
|
+
* claude can dig for specifics later without burning tokens.
|
|
8
|
+
*
|
|
9
|
+
* 315KB tool result → ~800 byte receipt = 99.7% token reduction. it slaps.
|
|
10
|
+
*
|
|
11
|
+
* inspired by claude-context-mode, rebuilt from scratch on postgres.
|
|
12
|
+
* no sqlite, no cap.
|
|
13
|
+
*
|
|
14
|
+
* @author hardwicksoftwareservices
|
|
15
|
+
* @website https://justcalljon.pro
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { createHash } from 'crypto';
|
|
19
|
+
import { logger } from '../utils/logger.js';
|
|
20
|
+
|
|
21
|
+
// --- Config ---
|
|
22
|
+
const VAULT_THRESHOLD = 5000; // chars — auto-stash anything bigger
|
|
23
|
+
const CHUNK_TARGET_SIZE = 1500; // chars per chunk (target, not hard limit)
|
|
24
|
+
const CHUNK_MAX_SIZE = 4000; // hard cap — code blocks can be big
|
|
25
|
+
const CHUNK_MIN_SIZE = 100; // merge anything smaller
|
|
26
|
+
const VAULT_TTL_HOURS = 24; // auto-expire after 24h
|
|
27
|
+
const MAX_VOCAB_TERMS = 30; // vocabulary terms in the receipt
|
|
28
|
+
const PREVIEW_CHARS = 300; // preview length in receipt
|
|
29
|
+
|
|
30
|
+
// tools that should NEVER get auto-vaulted (prevent recursion)
|
|
31
|
+
const VAULT_SKIP_TOOLS = new Set([
|
|
32
|
+
'stash_the_goods',
|
|
33
|
+
'dig_in_the_vault',
|
|
34
|
+
]);
|
|
35
|
+
|
|
36
|
+
// english stopwords — filtered from vocabulary extraction
|
|
37
|
+
const STOPWORDS = new Set([
|
|
38
|
+
'the','be','to','of','and','a','in','that','have','i','it','for','not','on',
|
|
39
|
+
'with','he','as','you','do','at','this','but','his','by','from','they','we',
|
|
40
|
+
'say','her','she','or','an','will','my','one','all','would','there','their',
|
|
41
|
+
'what','so','up','out','if','about','who','get','which','go','me','when',
|
|
42
|
+
'make','can','like','time','no','just','him','know','take','people','into',
|
|
43
|
+
'year','your','good','some','could','them','see','other','than','then','now',
|
|
44
|
+
'look','only','come','its','over','think','also','back','after','use','two',
|
|
45
|
+
'how','our','work','first','well','way','even','new','want','because','any',
|
|
46
|
+
'these','give','day','most','us','is','are','was','were','been','has','had',
|
|
47
|
+
'did','does','done','may','might','shall','should','must','need','could',
|
|
48
|
+
'would','will','true','false','null','undefined','function','return','const',
|
|
49
|
+
'let','var','import','export','from','class','async','await','try','catch',
|
|
50
|
+
'throw','typeof','instanceof','void','this','super','extends',
|
|
51
|
+
'type','string','number','boolean','object','array',
|
|
52
|
+
]);
|
|
53
|
+
|
|
54
|
+
let _pool = null;
|
|
55
|
+
let _tableReady = false;
|
|
56
|
+
let _projectPath = '/';
|
|
57
|
+
|
|
58
|
+
// --- Init ---
|
|
59
|
+
export function initContextVault(pool, projectPath) {
|
|
60
|
+
_pool = pool;
|
|
61
|
+
_projectPath = projectPath || '/';
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function setVaultProjectPath(p) { _projectPath = p || '/'; }
|
|
65
|
+
|
|
66
|
+
export { VAULT_THRESHOLD, VAULT_SKIP_TOOLS };
|
|
67
|
+
|
|
68
|
+
// --- Table creation (fallback if migration 38 hasn't run yet) ---
|
|
69
|
+
async function ensureTable() {
|
|
70
|
+
if (_tableReady || !_pool) return;
|
|
71
|
+
try {
|
|
72
|
+
await _pool.query(`
|
|
73
|
+
CREATE TABLE IF NOT EXISTS context_vault (
|
|
74
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
75
|
+
vault_id VARCHAR(16) NOT NULL,
|
|
76
|
+
chunk_idx INTEGER NOT NULL DEFAULT 0,
|
|
77
|
+
content TEXT NOT NULL,
|
|
78
|
+
content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', content)) STORED,
|
|
79
|
+
source_tool VARCHAR(128),
|
|
80
|
+
source_size INTEGER DEFAULT 0,
|
|
81
|
+
metadata JSONB DEFAULT '{}',
|
|
82
|
+
project_path VARCHAR(500) DEFAULT '/',
|
|
83
|
+
stashed_at TIMESTAMPTZ DEFAULT NOW(),
|
|
84
|
+
expires_at TIMESTAMPTZ DEFAULT (NOW() + INTERVAL '24 hours'),
|
|
85
|
+
UNIQUE(vault_id, chunk_idx)
|
|
86
|
+
);
|
|
87
|
+
CREATE INDEX IF NOT EXISTS idx_ctx_vault_tsv ON context_vault USING GIN(content_tsv);
|
|
88
|
+
CREATE INDEX IF NOT EXISTS idx_ctx_vault_id ON context_vault(vault_id);
|
|
89
|
+
CREATE INDEX IF NOT EXISTS idx_ctx_vault_expires ON context_vault(expires_at);
|
|
90
|
+
CREATE INDEX IF NOT EXISTS idx_ctx_vault_project ON context_vault(project_path);
|
|
91
|
+
`);
|
|
92
|
+
_tableReady = true;
|
|
93
|
+
} catch (err) {
|
|
94
|
+
// table might already exist from migration 38 — that's fine
|
|
95
|
+
if (err.message?.includes('already exists') || err.code === '42P07') {
|
|
96
|
+
_tableReady = true;
|
|
97
|
+
} else {
|
|
98
|
+
logger.warn({ error: err.message }, 'context_vault ensureTable failed');
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// =============================================================================
|
|
104
|
+
// Chunking — split content by natural boundaries, keep code blocks intact
|
|
105
|
+
// =============================================================================
|
|
106
|
+
|
|
107
|
+
function chunkContent(content) {
|
|
108
|
+
if (content.length <= CHUNK_TARGET_SIZE) return [content];
|
|
109
|
+
|
|
110
|
+
// Phase 1: protect code blocks from splitting
|
|
111
|
+
const codeBlocks = [];
|
|
112
|
+
const withPlaceholders = content.replace(/```[\s\S]*?```/g, (match) => {
|
|
113
|
+
const idx = codeBlocks.length;
|
|
114
|
+
codeBlocks.push(match);
|
|
115
|
+
return `\n__CB${idx}__\n`;
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
// Phase 2: split by markdown headings first, then paragraphs
|
|
119
|
+
const rawParts = withPlaceholders.split(/(?=^#{1,4}\s)/m);
|
|
120
|
+
|
|
121
|
+
const expandedParts = [];
|
|
122
|
+
for (const part of rawParts) {
|
|
123
|
+
if (part.length <= CHUNK_TARGET_SIZE) {
|
|
124
|
+
expandedParts.push(part);
|
|
125
|
+
} else {
|
|
126
|
+
// long section — split by paragraphs
|
|
127
|
+
const paragraphs = part.split(/\n{2,}/);
|
|
128
|
+
expandedParts.push(...paragraphs);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Phase 3: merge small parts, split oversized ones
|
|
133
|
+
const chunks = [];
|
|
134
|
+
let buffer = '';
|
|
135
|
+
|
|
136
|
+
for (let raw of expandedParts) {
|
|
137
|
+
// restore code blocks in this part
|
|
138
|
+
raw = raw.replace(/__CB(\d+)__/g, (_, idx) => codeBlocks[parseInt(idx)] || '');
|
|
139
|
+
|
|
140
|
+
if (!raw.trim()) continue;
|
|
141
|
+
|
|
142
|
+
if (buffer.length + raw.length + 2 <= CHUNK_TARGET_SIZE) {
|
|
143
|
+
buffer += (buffer ? '\n\n' : '') + raw;
|
|
144
|
+
} else {
|
|
145
|
+
if (buffer.trim()) chunks.push(buffer.trim());
|
|
146
|
+
|
|
147
|
+
if (raw.length > CHUNK_MAX_SIZE) {
|
|
148
|
+
// split oversized by sentences
|
|
149
|
+
const sentences = raw.match(/[^.!?\n]+[.!?\n]+/g) || [raw];
|
|
150
|
+
let sentBuf = '';
|
|
151
|
+
for (const sent of sentences) {
|
|
152
|
+
if (sentBuf.length + sent.length > CHUNK_TARGET_SIZE && sentBuf) {
|
|
153
|
+
chunks.push(sentBuf.trim());
|
|
154
|
+
sentBuf = '';
|
|
155
|
+
}
|
|
156
|
+
sentBuf += sent;
|
|
157
|
+
}
|
|
158
|
+
buffer = sentBuf;
|
|
159
|
+
} else {
|
|
160
|
+
buffer = raw;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (buffer.trim()) chunks.push(buffer.trim());
|
|
165
|
+
|
|
166
|
+
// Phase 4: merge tiny chunks into neighbors
|
|
167
|
+
const merged = [];
|
|
168
|
+
let mergeBuffer = '';
|
|
169
|
+
for (const chunk of chunks) {
|
|
170
|
+
if (chunk.length < CHUNK_MIN_SIZE && mergeBuffer.length + chunk.length < CHUNK_TARGET_SIZE) {
|
|
171
|
+
mergeBuffer += (mergeBuffer ? '\n\n' : '') + chunk;
|
|
172
|
+
} else {
|
|
173
|
+
if (mergeBuffer) {
|
|
174
|
+
if (mergeBuffer.length < CHUNK_MIN_SIZE && merged.length > 0) {
|
|
175
|
+
merged[merged.length - 1] += '\n\n' + mergeBuffer;
|
|
176
|
+
} else {
|
|
177
|
+
merged.push(mergeBuffer);
|
|
178
|
+
}
|
|
179
|
+
mergeBuffer = '';
|
|
180
|
+
}
|
|
181
|
+
merged.push(chunk);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
if (mergeBuffer) {
|
|
185
|
+
if (merged.length > 0) {
|
|
186
|
+
merged[merged.length - 1] += '\n\n' + mergeBuffer;
|
|
187
|
+
} else {
|
|
188
|
+
merged.push(mergeBuffer);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
return merged.filter(c => c.length > 0);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// =============================================================================
|
|
196
|
+
// Vocabulary extraction — top N terms by frequency, excluding stopwords
|
|
197
|
+
// =============================================================================
|
|
198
|
+
|
|
199
|
+
function extractVocabulary(content) {
|
|
200
|
+
const words = content
|
|
201
|
+
.toLowerCase()
|
|
202
|
+
.replace(/[^a-z0-9_\s-]/g, ' ')
|
|
203
|
+
.split(/\s+/)
|
|
204
|
+
.filter(w => w.length > 2 && w.length < 30 && !STOPWORDS.has(w));
|
|
205
|
+
|
|
206
|
+
const freq = {};
|
|
207
|
+
for (const w of words) {
|
|
208
|
+
freq[w] = (freq[w] || 0) + 1;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
return Object.entries(freq)
|
|
212
|
+
.sort((a, b) => b[1] - a[1])
|
|
213
|
+
.slice(0, MAX_VOCAB_TERMS)
|
|
214
|
+
.map(([word]) => word);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// =============================================================================
|
|
218
|
+
// Stash — chunk + index + store
|
|
219
|
+
// =============================================================================
|
|
220
|
+
|
|
221
|
+
export async function stashTheGoods(content, metadata = {}) {
|
|
222
|
+
if (!_pool) throw new Error('context vault not initialized — call initContextVault first');
|
|
223
|
+
await ensureTable();
|
|
224
|
+
|
|
225
|
+
// Reap expired stashes (piggyback on stash calls — no separate timer needed)
|
|
226
|
+
reapExpired().catch(() => {});
|
|
227
|
+
|
|
228
|
+
// vault_id from content hash — idempotent, same content = same ID = no dupes
|
|
229
|
+
const vaultId = createHash('sha256').update(content).digest('hex').slice(0, 12);
|
|
230
|
+
|
|
231
|
+
// Check if already stashed (idempotent dedup)
|
|
232
|
+
try {
|
|
233
|
+
const existing = await _pool.query(
|
|
234
|
+
'SELECT COUNT(*)::int as cnt FROM context_vault WHERE vault_id = $1',
|
|
235
|
+
[vaultId]
|
|
236
|
+
);
|
|
237
|
+
if (existing.rows[0]?.cnt > 0) {
|
|
238
|
+
const vocab = extractVocabulary(content);
|
|
239
|
+
return {
|
|
240
|
+
vaultId,
|
|
241
|
+
chunks: existing.rows[0].cnt,
|
|
242
|
+
totalChars: content.length,
|
|
243
|
+
preview: content.slice(0, PREVIEW_CHARS),
|
|
244
|
+
vocabulary: vocab,
|
|
245
|
+
deduplicated: true,
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
} catch (checkErr) {
|
|
249
|
+
// table might not exist yet — ensureTable should have handled it
|
|
250
|
+
logger.warn({ error: checkErr.message }, 'vault dedup check failed');
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// Chunk the content
|
|
254
|
+
const chunks = chunkContent(content);
|
|
255
|
+
const vocab = extractVocabulary(content);
|
|
256
|
+
|
|
257
|
+
// Batch insert chunks
|
|
258
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
259
|
+
try {
|
|
260
|
+
await _pool.query(
|
|
261
|
+
`INSERT INTO context_vault (vault_id, chunk_idx, content, source_tool, source_size, metadata, project_path, expires_at)
|
|
262
|
+
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, NOW() + make_interval(hours => $8))
|
|
263
|
+
ON CONFLICT (vault_id, chunk_idx) DO NOTHING`,
|
|
264
|
+
[
|
|
265
|
+
vaultId,
|
|
266
|
+
i,
|
|
267
|
+
chunks[i],
|
|
268
|
+
metadata.tool || 'unknown',
|
|
269
|
+
content.length,
|
|
270
|
+
JSON.stringify(metadata),
|
|
271
|
+
_projectPath,
|
|
272
|
+
VAULT_TTL_HOURS,
|
|
273
|
+
]
|
|
274
|
+
);
|
|
275
|
+
} catch (insertErr) {
|
|
276
|
+
logger.warn({ error: insertErr.message, chunk: i }, 'vault chunk insert failed');
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
logger.info({
|
|
281
|
+
vaultId,
|
|
282
|
+
chunks: chunks.length,
|
|
283
|
+
totalChars: content.length,
|
|
284
|
+
tool: metadata.tool,
|
|
285
|
+
}, `vault: stashed ${chunks.length} chunks (${content.length} chars)`);
|
|
286
|
+
|
|
287
|
+
return {
|
|
288
|
+
vaultId,
|
|
289
|
+
chunks: chunks.length,
|
|
290
|
+
totalChars: content.length,
|
|
291
|
+
preview: content.slice(0, PREVIEW_CHARS),
|
|
292
|
+
vocabulary: vocab,
|
|
293
|
+
deduplicated: false,
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// =============================================================================
|
|
298
|
+
// Search — BM25-ranked retrieval via tsvector
|
|
299
|
+
// =============================================================================
|
|
300
|
+
|
|
301
|
+
export async function digInTheVault(query, vaultId = null, limit = 10) {
|
|
302
|
+
if (!_pool) throw new Error('context vault not initialized');
|
|
303
|
+
await ensureTable();
|
|
304
|
+
|
|
305
|
+
let sql, params;
|
|
306
|
+
|
|
307
|
+
if (vaultId) {
|
|
308
|
+
// Search within a specific stash
|
|
309
|
+
sql = `
|
|
310
|
+
SELECT vault_id, chunk_idx, content,
|
|
311
|
+
ts_rank(content_tsv, plainto_tsquery('english', $1)) as rank
|
|
312
|
+
FROM context_vault
|
|
313
|
+
WHERE vault_id = $2
|
|
314
|
+
AND content_tsv @@ plainto_tsquery('english', $1)
|
|
315
|
+
AND expires_at > NOW()
|
|
316
|
+
ORDER BY rank DESC
|
|
317
|
+
LIMIT $3
|
|
318
|
+
`;
|
|
319
|
+
params = [query, vaultId, limit];
|
|
320
|
+
} else {
|
|
321
|
+
// Search across all stashes for this project
|
|
322
|
+
sql = `
|
|
323
|
+
SELECT vault_id, chunk_idx, content,
|
|
324
|
+
ts_rank(content_tsv, plainto_tsquery('english', $1)) as rank
|
|
325
|
+
FROM context_vault
|
|
326
|
+
WHERE content_tsv @@ plainto_tsquery('english', $1)
|
|
327
|
+
AND project_path = $2
|
|
328
|
+
AND expires_at > NOW()
|
|
329
|
+
ORDER BY rank DESC
|
|
330
|
+
LIMIT $3
|
|
331
|
+
`;
|
|
332
|
+
params = [query, _projectPath, limit];
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
const { rows } = await _pool.query(sql, params);
|
|
336
|
+
|
|
337
|
+
// If tsvector found nothing and we have a vault_id, fall back to ILIKE
|
|
338
|
+
if (rows.length === 0 && vaultId) {
|
|
339
|
+
const fallback = await _pool.query(
|
|
340
|
+
`SELECT vault_id, chunk_idx, content, 0.0::float as rank
|
|
341
|
+
FROM context_vault
|
|
342
|
+
WHERE vault_id = $1 AND content ILIKE '%' || $2 || '%' AND expires_at > NOW()
|
|
343
|
+
ORDER BY chunk_idx ASC LIMIT $3`,
|
|
344
|
+
[vaultId, query, limit]
|
|
345
|
+
);
|
|
346
|
+
return fallback.rows;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
return rows;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
// =============================================================================
|
|
353
|
+
// Get full stash — all chunks ordered for complete retrieval
|
|
354
|
+
// =============================================================================
|
|
355
|
+
|
|
356
|
+
export async function getFullStash(vaultId) {
|
|
357
|
+
if (!_pool) throw new Error('context vault not initialized');
|
|
358
|
+
await ensureTable();
|
|
359
|
+
|
|
360
|
+
const { rows } = await _pool.query(
|
|
361
|
+
`SELECT chunk_idx, content FROM context_vault
|
|
362
|
+
WHERE vault_id = $1 AND expires_at > NOW()
|
|
363
|
+
ORDER BY chunk_idx ASC`,
|
|
364
|
+
[vaultId]
|
|
365
|
+
);
|
|
366
|
+
|
|
367
|
+
if (rows.length === 0) return null;
|
|
368
|
+
return rows.map(r => r.content).join('\n\n');
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
// =============================================================================
|
|
372
|
+
// Stats — what's in the vault right now
|
|
373
|
+
// =============================================================================
|
|
374
|
+
|
|
375
|
+
export async function getVaultStats() {
|
|
376
|
+
if (!_pool) return { totalStashes: 0, totalChunks: 0, totalChars: 0 };
|
|
377
|
+
await ensureTable();
|
|
378
|
+
|
|
379
|
+
try {
|
|
380
|
+
const { rows } = await _pool.query(`
|
|
381
|
+
SELECT
|
|
382
|
+
COUNT(DISTINCT vault_id)::int as stash_count,
|
|
383
|
+
COUNT(*)::int as chunk_count,
|
|
384
|
+
COALESCE(SUM(LENGTH(content)), 0)::int as total_chars,
|
|
385
|
+
MIN(stashed_at) as oldest
|
|
386
|
+
FROM context_vault
|
|
387
|
+
WHERE project_path = $1 AND expires_at > NOW()
|
|
388
|
+
`, [_projectPath]);
|
|
389
|
+
|
|
390
|
+
return {
|
|
391
|
+
totalStashes: rows[0]?.stash_count || 0,
|
|
392
|
+
totalChunks: rows[0]?.chunk_count || 0,
|
|
393
|
+
totalChars: rows[0]?.total_chars || 0,
|
|
394
|
+
oldestStash: rows[0]?.oldest || null,
|
|
395
|
+
};
|
|
396
|
+
} catch (err) {
|
|
397
|
+
return { totalStashes: 0, totalChunks: 0, totalChars: 0, error: err.message };
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
// =============================================================================
|
|
402
|
+
// Reaper — clean expired stashes (runs piggyback on stash operations)
|
|
403
|
+
// =============================================================================
|
|
404
|
+
|
|
405
|
+
async function reapExpired() {
|
|
406
|
+
if (!_pool) return;
|
|
407
|
+
try {
|
|
408
|
+
const result = await _pool.query('DELETE FROM context_vault WHERE expires_at < NOW()');
|
|
409
|
+
if (result.rowCount > 0) {
|
|
410
|
+
logger.info({ reaped: result.rowCount }, 'vault: reaped expired stashes');
|
|
411
|
+
}
|
|
412
|
+
} catch {
|
|
413
|
+
// non-fatal, swallow
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
// =============================================================================
|
|
418
|
+
// Receipt formatter — compact reference that replaces the full output
|
|
419
|
+
// =============================================================================
|
|
420
|
+
|
|
421
|
+
export function formatVaultReceipt(stashResult) {
|
|
422
|
+
const { vaultId, chunks, totalChars, preview, vocabulary, deduplicated } = stashResult;
|
|
423
|
+
const receiptSize = PREVIEW_CHARS + 200 + (vocabulary.length * 8);
|
|
424
|
+
const savings = ((1 - (receiptSize / totalChars)) * 100).toFixed(1);
|
|
425
|
+
|
|
426
|
+
return [
|
|
427
|
+
`<vault_receipt id="${vaultId}" chunks="${chunks}" total_chars="${totalChars}" savings="${savings}%"${deduplicated ? ' dedup="true"' : ''}>`,
|
|
428
|
+
`Content stashed (${totalChars.toLocaleString()} chars chunked into ${chunks} searchable blocks)`,
|
|
429
|
+
``,
|
|
430
|
+
`Preview:`,
|
|
431
|
+
preview + (totalChars > PREVIEW_CHARS ? '...' : ''),
|
|
432
|
+
``,
|
|
433
|
+
`Key terms: ${vocabulary.join(', ')}`,
|
|
434
|
+
``,
|
|
435
|
+
`Retrieve: dig_in_the_vault(query: "search terms", vault_id: "${vaultId}")`,
|
|
436
|
+
`Full dump: dig_in_the_vault(vault_id: "${vaultId}", get_all: true)`,
|
|
437
|
+
`</vault_receipt>`,
|
|
438
|
+
].join('\n');
|
|
439
|
+
}
|