specmem-hardwicksoftware 3.5.99 → 3.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/specmem-statusbar.cjs +154 -298
- package/claude-hooks/agent-loading-hook.js +8 -4
- package/claude-hooks/team-comms-enforcer.cjs +109 -92
- package/dist/config/embeddingTimeouts.js +4 -4
- package/dist/database.js +52 -6
- package/dist/db/bigBrainMigrations.js +7 -6
- package/dist/db/memoryDrilldown.sql +1 -1
- package/dist/db/projectSchemaInit.sql +21 -0
- package/dist/index.js +238 -13
- package/dist/installer/firstRun.js +2 -2
- package/dist/mcp/embeddingServerManager.js +225 -7
- package/dist/mcp/healthMonitor.js +165 -32
- package/dist/mcp/tools/embeddingControl.js +31 -0
- package/dist/mcp/tools/teamComms.js +16 -0
- package/dist/mcp/watcherIntegration.js +50 -7
- package/dist/services/CameraZoomSearch.js +62 -5
- package/dist/services/DimensionService.js +73 -6
- package/dist/services/EmbeddingQueue.js +64 -0
- package/dist/services/MemoryDrilldown.js +19 -12
- package/dist/tools/goofy/findCodePointers.js +11 -7
- package/dist/tools/goofy/findWhatISaid.js +145 -53
- package/dist/utils/qoms.js +187 -4
- package/dist/watcher/changeHandler.js +54 -4
- package/dist/watcher/fileWatcher.js +121 -1
- package/dist/watcher/index.js +75 -31
- package/dist/watcher/syncChecker.js +248 -63
- package/embedding-sandbox/__pycache__/frankenstein-embeddings.cpython-313.pyc +0 -0
- package/embedding-sandbox/frankenstein-embeddings.py +175 -64
- package/package.json +1 -1
|
@@ -22,6 +22,12 @@ import { getProjectContext } from '../services/ProjectContext.js';
|
|
|
22
22
|
import * as path from 'path';
|
|
23
23
|
/** Map of projectPath -> WatcherState - each project isolated */
|
|
24
24
|
const watcherStateByProject = new Map();
|
|
25
|
+
/**
|
|
26
|
+
* Initialization lock per project - prevents double watcher initialization (Issue #12).
|
|
27
|
+
* Maps projectPath -> Promise that resolves when initialization completes.
|
|
28
|
+
* If init is already in progress, subsequent calls await the existing promise.
|
|
29
|
+
*/
|
|
30
|
+
const initializationLockByProject = new Map();
|
|
25
31
|
/**
|
|
26
32
|
* getWatcherState - get or create watcher state for a project
|
|
27
33
|
* This is the key to per-project isolation - no more cross-pollution!
|
|
@@ -94,19 +100,48 @@ async function getWatchedPathsFromDb() {
|
|
|
94
100
|
* Each SpecMem instance only watches its own project.
|
|
95
101
|
*/
|
|
96
102
|
export async function initializeWatcher(embeddingProvider) {
|
|
97
|
-
// check if watcher is
|
|
98
|
-
//
|
|
99
|
-
|
|
100
|
-
|
|
103
|
+
// check if watcher is EXPLICITLY disabled in config
|
|
104
|
+
// Default is ENABLED (true) - only skip if user explicitly set SPECMEM_WATCHER_ENABLED=false
|
|
105
|
+
const watcherEnabled = config.watcher?.enabled ?? true;
|
|
106
|
+
const envOverride = process.env['SPECMEM_WATCHER_ENABLED'];
|
|
107
|
+
// Only disable if explicitly set to 'false' - empty string, undefined, 'true' all mean enabled
|
|
108
|
+
if (envOverride === 'false' || (envOverride === undefined && watcherEnabled === false)) {
|
|
109
|
+
logger.info('file watcher EXPLICITLY disabled via SPECMEM_WATCHER_ENABLED=false - skipping initialization');
|
|
101
110
|
return null;
|
|
102
111
|
}
|
|
103
112
|
const projectPath = getProjectPath();
|
|
104
113
|
const state = getWatcherState(projectPath);
|
|
105
|
-
//
|
|
114
|
+
// Issue #12 Fix: Check if watcher is already fully initialized for this project
|
|
106
115
|
if (state.manager) {
|
|
107
|
-
logger.info({ projectPath }, 'watcher already initialized for this project');
|
|
116
|
+
logger.info({ projectPath }, 'watcher already initialized for this project - returning existing manager');
|
|
108
117
|
return state.manager;
|
|
109
118
|
}
|
|
119
|
+
// Issue #12 Fix: Promise-based initialization lock prevents double init race condition.
|
|
120
|
+
// If init is already in progress from another code path (MCP tool call, startup, reconnection),
|
|
121
|
+
// subsequent calls await the existing init promise instead of creating a second watcher.
|
|
122
|
+
if (initializationLockByProject.has(projectPath)) {
|
|
123
|
+
logger.warn({ projectPath }, 'watcher initialization already in progress - awaiting existing init (preventing double init race condition)');
|
|
124
|
+
return await initializationLockByProject.get(projectPath);
|
|
125
|
+
}
|
|
126
|
+
// Create the initialization promise and store it as the lock
|
|
127
|
+
const initPromise = _doInitializeWatcher(embeddingProvider, projectPath, state);
|
|
128
|
+
initializationLockByProject.set(projectPath, initPromise);
|
|
129
|
+
try {
|
|
130
|
+
const result = await initPromise;
|
|
131
|
+
return result;
|
|
132
|
+
}
|
|
133
|
+
finally {
|
|
134
|
+
// Always release the lock when init completes (success or failure)
|
|
135
|
+
initializationLockByProject.delete(projectPath);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* _doInitializeWatcher - internal init logic, called once per project under lock
|
|
140
|
+
*
|
|
141
|
+
* Separated from initializeWatcher to keep the lock logic clean.
|
|
142
|
+
* This function does the actual work of creating and starting the watcher.
|
|
143
|
+
*/
|
|
144
|
+
async function _doInitializeWatcher(embeddingProvider, projectPath, state) {
|
|
110
145
|
logger.info({ projectPath }, 'initializing PROJECT-SCOPED file watcher...');
|
|
111
146
|
try {
|
|
112
147
|
// get database context - check if it's initialized first
|
|
@@ -136,13 +171,17 @@ export async function initializeWatcher(embeddingProvider) {
|
|
|
136
171
|
paths: watchPaths,
|
|
137
172
|
source: dbPaths.length > 0 ? 'database (project-scoped)' : 'project path only'
|
|
138
173
|
}, 'resolved PROJECT-SCOPED watch paths');
|
|
174
|
+
// Issue #12 Fix: Watcher event debounce is configurable via SPECMEM_WATCHER_DEBOUNCE_MS
|
|
175
|
+
// Default is 1000ms. This prevents excessive processing of rapid file change events.
|
|
176
|
+
const debounceMs = parseInt(process.env['SPECMEM_WATCHER_DEBOUNCE_MS'] || String(config.watcher.debounceMs || 1000), 10);
|
|
177
|
+
logger.info({ debounceMs }, 'using configurable watcher debounce delay');
|
|
139
178
|
// build watcher config
|
|
140
179
|
const watcherConfig = {
|
|
141
180
|
watcher: {
|
|
142
181
|
rootPath: primaryPath,
|
|
143
182
|
additionalPaths: additionalPaths,
|
|
144
183
|
ignorePath: config.watcher.ignorePath,
|
|
145
|
-
debounceMs:
|
|
184
|
+
debounceMs: debounceMs,
|
|
146
185
|
autoRestart: config.watcher.autoRestart,
|
|
147
186
|
maxRestarts: config.watcher.maxRestarts,
|
|
148
187
|
verbose: config.logging.level === 'debug' || config.logging.level === 'trace'
|
|
@@ -226,12 +265,16 @@ export async function initializeWatcher(embeddingProvider) {
|
|
|
226
265
|
projectPath,
|
|
227
266
|
paths: watchPaths,
|
|
228
267
|
syncCheckIntervalMinutes: config.watcher.syncCheckIntervalMinutes,
|
|
268
|
+
debounceMs: debounceMs,
|
|
229
269
|
filesWatched: status.watcher.filesWatched
|
|
230
270
|
}, 'PROJECT-SCOPED file watcher initialized and VERIFIED running');
|
|
231
271
|
return state.manager;
|
|
232
272
|
}
|
|
233
273
|
catch (error) {
|
|
234
274
|
logger.error({ error, projectPath }, 'failed to initialize project-scoped file watcher');
|
|
275
|
+
// Clean up state on failure so a retry can succeed
|
|
276
|
+
state.manager = null;
|
|
277
|
+
state.watchedPaths.clear();
|
|
235
278
|
return null;
|
|
236
279
|
}
|
|
237
280
|
}
|
|
@@ -40,14 +40,64 @@ class DrilldownRegistry {
|
|
|
40
40
|
registry = new Map();
|
|
41
41
|
reverseRegistry = new Map();
|
|
42
42
|
nextID = 1;
|
|
43
|
-
maxEntries = 10000;
|
|
44
|
-
|
|
43
|
+
maxEntries = parseInt(process.env['SPECMEM_DRILLDOWN_MAX_SIZE'] || '10000');
|
|
44
|
+
ttlMs = parseInt(process.env['SPECMEM_DRILLDOWN_TTL_MS'] || '1800000'); // 30 min default
|
|
45
|
+
cleanupIntervalMs = parseInt(process.env['SPECMEM_DRILLDOWN_CLEANUP_INTERVAL_MS'] || '300000'); // 5 min default
|
|
46
|
+
_cleanupTimer = null;
|
|
47
|
+
constructor() {
|
|
48
|
+
// Start periodic TTL cleanup
|
|
49
|
+
this._startPeriodicCleanup();
|
|
50
|
+
}
|
|
45
51
|
static getInstance() {
|
|
46
52
|
if (!DrilldownRegistry.instance) {
|
|
47
53
|
DrilldownRegistry.instance = new DrilldownRegistry();
|
|
48
54
|
}
|
|
49
55
|
return DrilldownRegistry.instance;
|
|
50
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* Start periodic cleanup timer for TTL-expired entries
|
|
59
|
+
*/
|
|
60
|
+
_startPeriodicCleanup() {
|
|
61
|
+
if (this._cleanupTimer) {
|
|
62
|
+
clearInterval(this._cleanupTimer);
|
|
63
|
+
}
|
|
64
|
+
this._cleanupTimer = setInterval(() => {
|
|
65
|
+
this._expireTTLEntries();
|
|
66
|
+
}, this.cleanupIntervalMs);
|
|
67
|
+
// Allow the process to exit even if the timer is running
|
|
68
|
+
if (this._cleanupTimer && typeof this._cleanupTimer.unref === 'function') {
|
|
69
|
+
this._cleanupTimer.unref();
|
|
70
|
+
}
|
|
71
|
+
logger.debug({ cleanupIntervalMs: this.cleanupIntervalMs, ttlMs: this.ttlMs }, 'DrilldownRegistry periodic TTL cleanup started');
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Remove entries that have exceeded their TTL
|
|
75
|
+
*/
|
|
76
|
+
_expireTTLEntries() {
|
|
77
|
+
const now = Date.now();
|
|
78
|
+
let expiredCount = 0;
|
|
79
|
+
for (const [id, entry] of this.registry.entries()) {
|
|
80
|
+
const lastTouched = entry.lastAccessed?.getTime() || entry.createdAt.getTime();
|
|
81
|
+
if ((now - lastTouched) > this.ttlMs) {
|
|
82
|
+
this.registry.delete(id);
|
|
83
|
+
this.reverseRegistry.delete(entry.memoryID);
|
|
84
|
+
expiredCount++;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (expiredCount > 0) {
|
|
88
|
+
logger.info({ expiredCount, remaining: this.registry.size, ttlMs: this.ttlMs }, 'DrilldownRegistry TTL expiration: removed stale entries');
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Stop the periodic cleanup timer (for shutdown)
|
|
93
|
+
*/
|
|
94
|
+
shutdown() {
|
|
95
|
+
if (this._cleanupTimer) {
|
|
96
|
+
clearInterval(this._cleanupTimer);
|
|
97
|
+
this._cleanupTimer = null;
|
|
98
|
+
logger.debug('DrilldownRegistry cleanup timer cleared on shutdown');
|
|
99
|
+
}
|
|
100
|
+
}
|
|
51
101
|
/**
|
|
52
102
|
* Register a memory and get its drilldown ID
|
|
53
103
|
* Returns existing ID if already registered
|
|
@@ -58,6 +108,7 @@ class DrilldownRegistry {
|
|
|
58
108
|
if (existing !== undefined) {
|
|
59
109
|
const entry = this.registry.get(existing);
|
|
60
110
|
if (entry) {
|
|
111
|
+
// Refresh TTL on access
|
|
61
112
|
entry.lastAccessed = new Date();
|
|
62
113
|
entry.accessCount++;
|
|
63
114
|
}
|
|
@@ -70,12 +121,13 @@ class DrilldownRegistry {
|
|
|
70
121
|
memoryID,
|
|
71
122
|
type,
|
|
72
123
|
createdAt: new Date(),
|
|
124
|
+
lastAccessed: new Date(),
|
|
73
125
|
accessCount: 1,
|
|
74
126
|
...context
|
|
75
127
|
};
|
|
76
128
|
this.registry.set(drilldownID, entry);
|
|
77
129
|
this.reverseRegistry.set(memoryID, drilldownID);
|
|
78
|
-
// Cleanup old entries if needed
|
|
130
|
+
// Cleanup old entries if needed (LRU eviction when over max size)
|
|
79
131
|
this.cleanup();
|
|
80
132
|
logger.debug({ drilldownID, memoryID, type }, 'Registered drilldown ID');
|
|
81
133
|
return drilldownID;
|
|
@@ -88,6 +140,7 @@ class DrilldownRegistry {
|
|
|
88
140
|
if (typeof drilldownID === 'number') {
|
|
89
141
|
const entry = this.registry.get(drilldownID);
|
|
90
142
|
if (entry) {
|
|
143
|
+
// Refresh TTL on access (touch)
|
|
91
144
|
entry.lastAccessed = new Date();
|
|
92
145
|
entry.accessCount++;
|
|
93
146
|
return entry;
|
|
@@ -106,6 +159,7 @@ class DrilldownRegistry {
|
|
|
106
159
|
for (const [id, entry] of this.registry.entries()) {
|
|
107
160
|
const memID = entry.memoryID.toLowerCase().replace(/-/g, '');
|
|
108
161
|
if (memID.startsWith(shortCode) || memID.includes(shortCode)) {
|
|
162
|
+
// Refresh TTL on access (touch)
|
|
109
163
|
entry.lastAccessed = new Date();
|
|
110
164
|
entry.accessCount++;
|
|
111
165
|
return entry;
|
|
@@ -132,7 +186,7 @@ class DrilldownRegistry {
|
|
|
132
186
|
return result;
|
|
133
187
|
}
|
|
134
188
|
/**
|
|
135
|
-
* Cleanup old entries when registry is full
|
|
189
|
+
* Cleanup old entries when registry is full (LRU eviction)
|
|
136
190
|
*/
|
|
137
191
|
cleanup() {
|
|
138
192
|
if (this.registry.size <= this.maxEntries)
|
|
@@ -151,7 +205,7 @@ class DrilldownRegistry {
|
|
|
151
205
|
this.registry.delete(id);
|
|
152
206
|
this.reverseRegistry.delete(entry.memoryID);
|
|
153
207
|
}
|
|
154
|
-
logger.info({ removed: toRemove, remaining: this.registry.size }, 'Cleaned up drilldown registry');
|
|
208
|
+
logger.info({ removed: toRemove, remaining: this.registry.size }, 'Cleaned up drilldown registry (LRU eviction)');
|
|
155
209
|
}
|
|
156
210
|
/**
|
|
157
211
|
* Get registry stats
|
|
@@ -161,6 +215,9 @@ class DrilldownRegistry {
|
|
|
161
215
|
const dates = entries.map(e => e.createdAt.getTime());
|
|
162
216
|
return {
|
|
163
217
|
totalEntries: this.registry.size,
|
|
218
|
+
maxEntries: this.maxEntries,
|
|
219
|
+
ttlMs: this.ttlMs,
|
|
220
|
+
cleanupIntervalMs: this.cleanupIntervalMs,
|
|
164
221
|
oldestEntry: dates.length > 0 ? new Date(Math.min(...dates)) : undefined,
|
|
165
222
|
newestEntry: dates.length > 0 ? new Date(Math.max(...dates)) : undefined
|
|
166
223
|
};
|
|
@@ -41,15 +41,27 @@ const EMBEDDING_TABLES = [
|
|
|
41
41
|
/**
|
|
42
42
|
* Service for dynamic embedding dimension management.
|
|
43
43
|
* All dimension lookups go through the database - no hardcoded values.
|
|
44
|
+
*
|
|
45
|
+
* Issue #15 FIX: Cache TTL is now configurable via SPECMEM_DIMENSION_CACHE_TTL_MS (default 300000 = 5min).
|
|
46
|
+
* Stale cache entries are kept as fallback on fetch failure but logged as warnings.
|
|
47
|
+
* Dimension override via SPECMEM_EMBEDDING_DIMENSIONS env var.
|
|
48
|
+
* invalidateCache() method for embedding service restart coordination.
|
|
44
49
|
*/
|
|
45
50
|
export class DimensionService {
|
|
46
51
|
db;
|
|
47
52
|
embeddingProvider = null;
|
|
48
53
|
dimensionCache = new Map();
|
|
49
|
-
|
|
54
|
+
// Issue #15 FIX: Cache TTL is env-var configurable (default 5 minutes)
|
|
55
|
+
CACHE_TTL_MS = parseInt(process.env['SPECMEM_DIMENSION_CACHE_TTL_MS'] || '300000', 10);
|
|
56
|
+
// Issue #15 FIX: Track last successful verification timestamp
|
|
57
|
+
lastVerified = null;
|
|
50
58
|
constructor(db, embeddingProvider) {
|
|
51
59
|
this.db = db;
|
|
52
60
|
this.embeddingProvider = embeddingProvider || null;
|
|
61
|
+
logger.info({
|
|
62
|
+
cacheTtlMs: this.CACHE_TTL_MS,
|
|
63
|
+
dimensionOverride: process.env['SPECMEM_EMBEDDING_DIMENSIONS'] || 'auto-detect'
|
|
64
|
+
}, 'DimensionService initialized with configurable cache TTL (Issue #15 fix)');
|
|
53
65
|
}
|
|
54
66
|
/**
|
|
55
67
|
* Set the embedding provider (for lazy initialization)
|
|
@@ -66,12 +78,23 @@ export class DimensionService {
|
|
|
66
78
|
* @returns The dimension, or null if table/column doesn't exist
|
|
67
79
|
*/
|
|
68
80
|
async getTableDimension(tableName, columnName = 'embedding') {
|
|
69
|
-
// Check
|
|
81
|
+
// Issue #15 FIX: Check for dimension override via env var (skip DB entirely)
|
|
82
|
+
const dimensionOverride = parseInt(process.env['SPECMEM_EMBEDDING_DIMENSIONS'] || '0', 10);
|
|
83
|
+
if (dimensionOverride > 0) {
|
|
84
|
+
logger.debug({ dimensionOverride, tableName }, 'Using SPECMEM_EMBEDDING_DIMENSIONS override');
|
|
85
|
+
return dimensionOverride;
|
|
86
|
+
}
|
|
87
|
+
// Check cache first - Issue #15 FIX: Use configurable TTL
|
|
70
88
|
const cacheKey = `${tableName}.${columnName}`;
|
|
71
89
|
const cached = this.dimensionCache.get(cacheKey);
|
|
72
90
|
if (cached && Date.now() - cached.timestamp < this.CACHE_TTL_MS) {
|
|
73
91
|
return cached.dimension;
|
|
74
92
|
}
|
|
93
|
+
// Issue #15 FIX: If cache expired, log it for debugging stale cache issues
|
|
94
|
+
if (cached) {
|
|
95
|
+
const ageMs = Date.now() - cached.timestamp;
|
|
96
|
+
logger.debug({ cacheKey, ageMs, ttl: this.CACHE_TTL_MS }, 'Dimension cache expired, re-fetching from database');
|
|
97
|
+
}
|
|
75
98
|
try {
|
|
76
99
|
const result = await this.db.query(`SELECT atttypmod FROM pg_attribute
|
|
77
100
|
WHERE attrelid = $1::regclass AND attname = $2`, [tableName, columnName]);
|
|
@@ -80,14 +103,28 @@ export class DimensionService {
|
|
|
80
103
|
return null;
|
|
81
104
|
}
|
|
82
105
|
const dimension = result.rows[0].atttypmod;
|
|
83
|
-
// Cache the result
|
|
106
|
+
// Cache the result with timestamp for TTL and lastVerified tracking
|
|
84
107
|
this.dimensionCache.set(cacheKey, { dimension, timestamp: Date.now() });
|
|
108
|
+
this.lastVerified = Date.now();
|
|
85
109
|
logger.debug({ tableName, columnName, dimension }, 'Retrieved vector dimension from database');
|
|
86
110
|
return dimension;
|
|
87
111
|
}
|
|
88
112
|
catch (error) {
|
|
89
|
-
//
|
|
90
|
-
|
|
113
|
+
// Issue #15 FIX: If fetch fails but we have a stale cached value, use it
|
|
114
|
+
// and log a warning. This prevents dimension mismatch errors during transient DB issues.
|
|
115
|
+
if (cached) {
|
|
116
|
+
logger.warn({
|
|
117
|
+
error: error instanceof Error ? error.message : String(error),
|
|
118
|
+
tableName,
|
|
119
|
+
columnName,
|
|
120
|
+
staleDimension: cached.dimension,
|
|
121
|
+
staleAgeMs: Date.now() - cached.timestamp
|
|
122
|
+
}, 'Failed to refresh dimension from DB - using stale cached value. Will retry on next access.');
|
|
123
|
+
// Do NOT update the timestamp - keep it stale so next access retries
|
|
124
|
+
return cached.dimension;
|
|
125
|
+
}
|
|
126
|
+
// Table might not exist yet and no cache to fall back on
|
|
127
|
+
logger.debug({ error, tableName, columnName }, 'Failed to get dimension (table may not exist, no cache fallback)');
|
|
91
128
|
return null;
|
|
92
129
|
}
|
|
93
130
|
}
|
|
@@ -238,18 +275,30 @@ export class DimensionService {
|
|
|
238
275
|
}
|
|
239
276
|
/**
|
|
240
277
|
* Get cache statistics for debugging.
|
|
278
|
+
* Issue #15 FIX: Now includes lastVerified, TTL, and stale entry info.
|
|
241
279
|
*/
|
|
242
280
|
getCacheStats() {
|
|
243
281
|
const entries = [];
|
|
244
282
|
const now = Date.now();
|
|
283
|
+
let staleCount = 0;
|
|
245
284
|
this.dimensionCache.forEach((entry, key) => {
|
|
285
|
+
const age = now - entry.timestamp;
|
|
286
|
+
const isStale = age >= this.CACHE_TTL_MS;
|
|
287
|
+
if (isStale) staleCount++;
|
|
246
288
|
entries.push({
|
|
247
289
|
key,
|
|
248
|
-
age
|
|
290
|
+
age,
|
|
291
|
+
isStale,
|
|
292
|
+
dimension: entry.dimension
|
|
249
293
|
});
|
|
250
294
|
});
|
|
251
295
|
return {
|
|
252
296
|
size: this.dimensionCache.size,
|
|
297
|
+
staleCount,
|
|
298
|
+
cacheTtlMs: this.CACHE_TTL_MS,
|
|
299
|
+
lastVerified: this.lastVerified,
|
|
300
|
+
lastVerifiedAge: this.lastVerified ? now - this.lastVerified : null,
|
|
301
|
+
dimensionOverride: parseInt(process.env['SPECMEM_EMBEDDING_DIMENSIONS'] || '0', 10) || null,
|
|
253
302
|
entries
|
|
254
303
|
};
|
|
255
304
|
}
|
|
@@ -313,6 +362,24 @@ export class DimensionService {
|
|
|
313
362
|
});
|
|
314
363
|
keysToDelete.forEach(key => this.dimensionCache.delete(key));
|
|
315
364
|
}
|
|
365
|
+
/**
|
|
366
|
+
* Issue #15 FIX: Invalidate ALL cached dimensions.
|
|
367
|
+
* Call this when the embedding service restarts or the model changes.
|
|
368
|
+
* The next dimension request will re-fetch from the database.
|
|
369
|
+
*/
|
|
370
|
+
invalidateCache() {
|
|
371
|
+
const cacheSize = this.dimensionCache.size;
|
|
372
|
+
this.dimensionCache.clear();
|
|
373
|
+
this.lastVerified = null;
|
|
374
|
+
logger.info({ clearedEntries: cacheSize }, 'DimensionService cache invalidated (embedding service restart or model change)');
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Issue #15 FIX: Get the timestamp of the last successful dimension verification.
|
|
378
|
+
* Returns null if no verification has occurred since startup/last invalidation.
|
|
379
|
+
*/
|
|
380
|
+
getLastVerified() {
|
|
381
|
+
return this.lastVerified;
|
|
382
|
+
}
|
|
316
383
|
/**
|
|
317
384
|
* Validate a query embedding against a table's expected dimension.
|
|
318
385
|
*
|
|
@@ -24,10 +24,21 @@ export class EmbeddingQueue {
|
|
|
24
24
|
projectId;
|
|
25
25
|
initialized = false;
|
|
26
26
|
pendingCallbacks = new Map();
|
|
27
|
+
// Track when each callback was queued for expiry cleanup
|
|
28
|
+
callbackTimestamps = new Map();
|
|
27
29
|
isDraining = false;
|
|
30
|
+
// Configurable limits via environment variables
|
|
31
|
+
maxQueueAge = parseInt(process.env['SPECMEM_EMBED_QUEUE_MAX_AGE_MS'] || '300000'); // 5 min default
|
|
32
|
+
cleanupIntervalMs = parseInt(process.env['SPECMEM_EMBED_QUEUE_CLEANUP_INTERVAL_MS'] || '60000'); // 1 min default
|
|
33
|
+
maxQueueSize = parseInt(process.env['SPECMEM_EMBED_QUEUE_MAX_SIZE'] || '500');
|
|
34
|
+
cleanupTimer = null;
|
|
28
35
|
constructor(pool) {
|
|
29
36
|
this.pool = pool;
|
|
30
37
|
this.projectId = getProjectDirName();
|
|
38
|
+
// Start periodic cleanup of expired callbacks
|
|
39
|
+
this.cleanupTimer = setInterval(() => {
|
|
40
|
+
this.expireStaleCallbacks();
|
|
41
|
+
}, this.cleanupIntervalMs);
|
|
31
42
|
// CRITICAL: Set search_path on every new connection to ensure queries
|
|
32
43
|
// hit the correct project schema, not public
|
|
33
44
|
this.pool.on('connect', async (client) => {
|
|
@@ -82,6 +93,12 @@ export class EmbeddingQueue {
|
|
|
82
93
|
*/
|
|
83
94
|
async queueForEmbedding(text, priority = 5) {
|
|
84
95
|
await this.initialize();
|
|
96
|
+
// Reject new items when queue is full
|
|
97
|
+
if (this.pendingCallbacks.size >= this.maxQueueSize) {
|
|
98
|
+
const err = new Error(`EmbeddingQueue full (${this.pendingCallbacks.size}/${this.maxQueueSize}) - rejecting new request`);
|
|
99
|
+
logger.warn({ pendingCount: this.pendingCallbacks.size, maxQueueSize: this.maxQueueSize }, err.message);
|
|
100
|
+
throw err;
|
|
101
|
+
}
|
|
85
102
|
return new Promise(async (resolve, reject) => {
|
|
86
103
|
try {
|
|
87
104
|
// Insert into queue
|
|
@@ -107,6 +124,8 @@ export class EmbeddingQueue {
|
|
|
107
124
|
reject(new Error('No embedding returned'));
|
|
108
125
|
}
|
|
109
126
|
});
|
|
127
|
+
// Track when this callback was added for expiry
|
|
128
|
+
this.callbackTimestamps.set(queueId, Date.now());
|
|
110
129
|
}
|
|
111
130
|
catch (err) {
|
|
112
131
|
logger.error({ err, text: text.substring(0, 50) }, 'Failed to queue embedding');
|
|
@@ -191,6 +210,7 @@ export class EmbeddingQueue {
|
|
|
191
210
|
if (callback) {
|
|
192
211
|
callback(embedding);
|
|
193
212
|
this.pendingCallbacks.delete(item.id);
|
|
213
|
+
this.callbackTimestamps.delete(item.id);
|
|
194
214
|
}
|
|
195
215
|
processed++;
|
|
196
216
|
logger.debug({ queueId: item.id, processed }, 'Processed queued embedding');
|
|
@@ -206,6 +226,7 @@ export class EmbeddingQueue {
|
|
|
206
226
|
if (callback) {
|
|
207
227
|
callback(null, err instanceof Error ? err : new Error(errorMsg));
|
|
208
228
|
this.pendingCallbacks.delete(item.id);
|
|
229
|
+
this.callbackTimestamps.delete(item.id);
|
|
209
230
|
}
|
|
210
231
|
logger.warn({ queueId: item.id, error: errorMsg }, 'Failed to process queued embedding');
|
|
211
232
|
}
|
|
@@ -218,6 +239,49 @@ export class EmbeddingQueue {
|
|
|
218
239
|
}
|
|
219
240
|
return processed;
|
|
220
241
|
}
|
|
242
|
+
/**
|
|
243
|
+
* Expire stale callbacks that have been waiting longer than maxQueueAge.
|
|
244
|
+
* Called periodically by the cleanup timer to prevent unbounded callback growth.
|
|
245
|
+
*/
|
|
246
|
+
expireStaleCallbacks() {
|
|
247
|
+
const now = Date.now();
|
|
248
|
+
let expiredCount = 0;
|
|
249
|
+
for (const [queueId, timestamp] of this.callbackTimestamps.entries()) {
|
|
250
|
+
if (now - timestamp > this.maxQueueAge) {
|
|
251
|
+
const callback = this.pendingCallbacks.get(queueId);
|
|
252
|
+
if (callback) {
|
|
253
|
+
callback(null, new Error(`EmbeddingQueue callback expired after ${this.maxQueueAge}ms (queueId: ${queueId})`));
|
|
254
|
+
this.pendingCallbacks.delete(queueId);
|
|
255
|
+
}
|
|
256
|
+
this.callbackTimestamps.delete(queueId);
|
|
257
|
+
expiredCount++;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
if (expiredCount > 0) {
|
|
261
|
+
logger.warn({ expiredCount, remainingCallbacks: this.pendingCallbacks.size }, 'Expired stale EmbeddingQueue callbacks');
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Shutdown the queue - clears cleanup timer and rejects all pending callbacks.
|
|
266
|
+
* Must be called when the EmbeddingQueue is no longer needed.
|
|
267
|
+
*/
|
|
268
|
+
shutdown() {
|
|
269
|
+
if (this.cleanupTimer) {
|
|
270
|
+
clearInterval(this.cleanupTimer);
|
|
271
|
+
this.cleanupTimer = null;
|
|
272
|
+
}
|
|
273
|
+
// Reject all remaining pending callbacks
|
|
274
|
+
let rejectedCount = 0;
|
|
275
|
+
for (const [queueId, callback] of this.pendingCallbacks.entries()) {
|
|
276
|
+
callback(null, new Error('EmbeddingQueue shutting down'));
|
|
277
|
+
rejectedCount++;
|
|
278
|
+
}
|
|
279
|
+
this.pendingCallbacks.clear();
|
|
280
|
+
this.callbackTimestamps.clear();
|
|
281
|
+
if (rejectedCount > 0) {
|
|
282
|
+
logger.info({ rejectedCount }, 'EmbeddingQueue shutdown - rejected pending callbacks');
|
|
283
|
+
}
|
|
284
|
+
}
|
|
221
285
|
/**
|
|
222
286
|
* Clean up old completed/failed entries
|
|
223
287
|
* Keep last 7 days by default
|
|
@@ -93,7 +93,7 @@ export class MemoryDrilldown {
|
|
|
93
93
|
m.metadata,
|
|
94
94
|
m.embedding,
|
|
95
95
|
EXISTS(SELECT 1 FROM codebase_pointers WHERE memory_id = m.id) as has_code,
|
|
96
|
-
EXISTS(SELECT 1 FROM team_member_conversations WHERE memory_id = m.id) as has_conversation,
|
|
96
|
+
EXISTS(SELECT 1 FROM team_member_conversations tmc WHERE tmc.memory_id = m.id) as has_conversation,
|
|
97
97
|
1 - (m.embedding <=> $1::vector) as relevance
|
|
98
98
|
FROM memories m
|
|
99
99
|
WHERE m.content ILIKE $2
|
|
@@ -218,7 +218,8 @@ export class MemoryDrilldown {
|
|
|
218
218
|
* Fetch the conversation that spawned this memory
|
|
219
219
|
*/
|
|
220
220
|
async getConversation(memoryId) {
|
|
221
|
-
|
|
221
|
+
try {
|
|
222
|
+
const result = await this.db.query(`
|
|
222
223
|
SELECT
|
|
223
224
|
team_member_id,
|
|
224
225
|
team_member_name,
|
|
@@ -230,17 +231,23 @@ export class MemoryDrilldown {
|
|
|
230
231
|
ORDER BY timestamp DESC
|
|
231
232
|
LIMIT 1
|
|
232
233
|
`, [memoryId]);
|
|
233
|
-
|
|
234
|
+
if (result.rows.length === 0) {
|
|
235
|
+
return null;
|
|
236
|
+
}
|
|
237
|
+
const row = result.rows[0];
|
|
238
|
+
return {
|
|
239
|
+
team_member_id: row.team_member_id,
|
|
240
|
+
team_member_name: row.team_member_name,
|
|
241
|
+
timestamp: row.timestamp,
|
|
242
|
+
summary: row.summary,
|
|
243
|
+
full_transcript: row.full_transcript
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
catch (err) {
|
|
247
|
+
// Table may not exist yet - gracefully return null
|
|
248
|
+
logger.warn({ err: err?.message, memoryId }, 'getConversation failed (table may not exist)');
|
|
234
249
|
return null;
|
|
235
250
|
}
|
|
236
|
-
const row = result.rows[0];
|
|
237
|
-
return {
|
|
238
|
-
team_member_id: row.team_member_id,
|
|
239
|
-
team_member_name: row.team_member_name,
|
|
240
|
-
timestamp: row.timestamp,
|
|
241
|
-
summary: row.summary,
|
|
242
|
-
full_transcript: row.full_transcript
|
|
243
|
-
};
|
|
244
251
|
}
|
|
245
252
|
/**
|
|
246
253
|
* GET RELATED MEMORIES
|
|
@@ -256,7 +263,7 @@ export class MemoryDrilldown {
|
|
|
256
263
|
m.tags,
|
|
257
264
|
m.metadata,
|
|
258
265
|
EXISTS(SELECT 1 FROM codebase_pointers WHERE memory_id = m.id) as has_code,
|
|
259
|
-
EXISTS(SELECT 1 FROM team_member_conversations WHERE memory_id = m.id) as has_conversation,
|
|
266
|
+
EXISTS(SELECT 1 FROM team_member_conversations tmc WHERE tmc.memory_id = m.id) as has_conversation,
|
|
260
267
|
1 - (m.embedding <=> $1::vector) as relevance
|
|
261
268
|
FROM memories m
|
|
262
269
|
WHERE m.id != $2
|
|
@@ -419,13 +419,17 @@ export class FindCodePointers {
|
|
|
419
419
|
const includeTracebacks = params.includeTracebacks !== false;
|
|
420
420
|
// Broadcast COT start to dashboard
|
|
421
421
|
cotStart('find_code', params.query);
|
|
422
|
-
//
|
|
422
|
+
// Get configurable timeout (default 60s, env: SPECMEM_CODE_SEARCH_TIMEOUT)
|
|
423
423
|
const timeoutMs = getCodeSearchTimeout();
|
|
424
|
-
// FIX:
|
|
425
|
-
//
|
|
426
|
-
|
|
427
|
-
//
|
|
428
|
-
|
|
424
|
+
// FIX: Per-operation timeout budget instead of shared deadline
|
|
425
|
+
// The old shared deadline caused a cascade: by retry #2, only 10s remained (the Math.max floor).
|
|
426
|
+
// Now each operation phase gets its own timeout allocation:
|
|
427
|
+
// - Embedding generation (with retries): 50% of total budget per attempt
|
|
428
|
+
// - DB search + post-processing: remaining budget
|
|
429
|
+
const embeddingTimeoutPerAttempt = Math.max(15000, Math.floor(timeoutMs * 0.5));
|
|
430
|
+
const postEmbeddingDeadline = Date.now() + timeoutMs;
|
|
431
|
+
// For post-embedding operations, use remaining time from overall deadline (min 10s)
|
|
432
|
+
const remainingTime = () => Math.max(10000, postEmbeddingDeadline - Date.now());
|
|
429
433
|
// FIXER AGENT 7: Log socket path for debugging (same detection as find_memory)
|
|
430
434
|
const socketPath = getEmbeddingSocketPath();
|
|
431
435
|
if (process.env['SPECMEM_DEBUG']) {
|
|
@@ -446,7 +450,7 @@ export class FindCodePointers {
|
|
|
446
450
|
maxRetries: getMaxRetries(),
|
|
447
451
|
socketPath // FIXER AGENT 7: Include socket path in logs
|
|
448
452
|
}, '[CodePointers] Generating embedding for query');
|
|
449
|
-
const rawEmbedding = await withRetry(() => withTimeout(() => this.embeddingProvider.generateEmbedding(params.query),
|
|
453
|
+
const rawEmbedding = await withRetry(() => withTimeout(() => this.embeddingProvider.generateEmbedding(params.query), embeddingTimeoutPerAttempt, 'Embedding generation'), 'Embedding generation');
|
|
450
454
|
// Validate and prepare embeddings for each table (may have different dimensions)
|
|
451
455
|
// FIXER AGENT 8: Added timeout wrapper for dimension preparation
|
|
452
456
|
const [queryEmbeddingFiles, queryEmbeddingDefs] = await withTimeout(() => Promise.all([
|