specmem-hardwicksoftware 3.5.99 → 3.6.1

package/dist/tools/goofy/findWhatISaid.js

@@ -25,6 +25,38 @@ import { cotStart, cotResult, cotError } from '../../utils/cotBroadcast.js';
 const __debugLog = process.env['SPECMEM_DEBUG'] === '1'
     ? (...args) => console.error('[DEBUG]', ...args) // stderr, not stdout!
     : () => { };
+// ============================================================================
+// RETRY HELPER for find_memory embedding generation
+// ============================================================================
+const FIND_MEMORY_MAX_RETRIES = parseInt(process.env['SPECMEM_FIND_MEMORY_RETRIES'] || '2');
+function isTransientEmbeddingError(error) {
+    if (!(error instanceof Error)) return false;
+    const msg = error.message.toLowerCase();
+    return (msg.includes('timeout') || msg.includes('econnreset') ||
+        msg.includes('econnrefused') || msg.includes('socket hang up') ||
+        msg.includes('aborted') || msg.includes('etimedout') ||
+        msg.includes('qoms') || msg.includes('resource') || msg.includes('busy'));
+}
+async function withEmbeddingRetry(operation, operationName, maxRetries = FIND_MEMORY_MAX_RETRIES) {
+    let lastError = null;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            return await operation();
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : new Error(String(error));
+            if (attempt < maxRetries && isTransientEmbeddingError(error)) {
+                const delay = Math.min(1000 * Math.pow(2, attempt), 8000);
+                logger.warn({ operationName, attempt: attempt + 1, maxRetries: maxRetries + 1, error: lastError.message, retryInMs: delay }, `[find_memory] ${operationName} failed, retrying in ${delay}ms`);
+                await new Promise(resolve => setTimeout(resolve, delay));
+            }
+            else {
+                break;
+            }
+        }
+    }
+    throw lastError;
+}
 /**
  * Extract discoverable paths from memory content
  * This is the KEY to getting lots of info from few memories
@@ -744,23 +776,35 @@ export class FindWhatISaid {
                 socketPath,
                 query: safeParams.query?.slice(0, 50)
             });
-            const embeddingPromise = this.embeddingProvider.generateEmbedding(safeParams.query);
-            const timeoutPromise = new Promise((_, reject) => {
-                setTimeout(() => {
-                    const timeoutError = new Error(`Embedding generation timeout after ${formatTimeout(EMBEDDING_TIMEOUT_MS)}. ` +
-                        `Socket: ${socketPath}. ` +
-                        `Set SPECMEM_EMBEDDING_TIMEOUT env var to increase timeout.`);
-                    timeoutError.socketPath = socketPath;
-                    timeoutError.code = 'EMBEDDING_TIMEOUT';
-                    reject(timeoutError);
-                }, EMBEDDING_TIMEOUT_MS);
-            });
             let rawEmbedding;
             try {
                 __debugLog('[FIND_MEMORY DEBUG]', Date.now(), 'AWAITING_EMBEDDING_PROMISE', {
                     elapsedMs: Date.now() - startTime
                 });
-                rawEmbedding = await Promise.race([embeddingPromise, timeoutPromise]);
+                // Retry wrapper: retries transient failures (timeouts, socket errors) with exponential backoff
+                rawEmbedding = await withEmbeddingRetry(async () => {
+                    const embeddingPromise = this.embeddingProvider.generateEmbedding(safeParams.query);
+                    let embeddingTimeoutId;
+                    const timeoutPromise = new Promise((_, reject) => {
+                        embeddingTimeoutId = setTimeout(() => {
+                            const timeoutError = new Error(`Embedding generation timeout after ${formatTimeout(EMBEDDING_TIMEOUT_MS)}. ` +
+                                `Socket: ${socketPath}. ` +
+                                `Set SPECMEM_EMBEDDING_TIMEOUT env var to increase timeout.`);
+                            timeoutError.socketPath = socketPath;
+                            timeoutError.code = 'EMBEDDING_TIMEOUT';
+                            reject(timeoutError);
+                        }, EMBEDDING_TIMEOUT_MS);
+                    });
+                    try {
+                        const result = await Promise.race([embeddingPromise, timeoutPromise]);
+                        clearTimeout(embeddingTimeoutId);
+                        return result;
+                    }
+                    catch (err) {
+                        clearTimeout(embeddingTimeoutId);
+                        throw err;
+                    }
+                }, 'Embedding generation');
                 const embeddingDuration = Date.now() - embeddingStartTime;
                 // ============================================================================
                 // DEEP DEBUG: After Embedding Generation (Success)
@@ -777,6 +821,7 @@ export class FindWhatISaid {
                 });
             }
             catch (embeddingError) {
+                clearTimeout(embeddingTimeoutId); // Prevent dangling timer on error path
                 const embeddingDuration = Date.now() - embeddingStartTime;
                 const err = embeddingError;
                 // ============================================================================
@@ -849,8 +894,9 @@ export class FindWhatISaid {
                 ...safeParams,
                 dateRange
             }, queryEmbedding);
+            let searchTimeoutId;
             const searchTimeoutPromise = new Promise((_, reject) => {
-                setTimeout(() => {
+                searchTimeoutId = setTimeout(() => {
                     const timeoutError = new Error(`Search timeout after ${formatTimeout(SEARCH_TIMEOUT_MS)}. ` +
                         `Query: "${safeParams.query.slice(0, 50)}...". ` +
                         `Set SPECMEM_EMBEDDING_TIMEOUT env var to increase timeout.`);
@@ -864,6 +910,7 @@ export class FindWhatISaid {
                     elapsedMs: Date.now() - startTime
                 });
                 results = await Promise.race([searchPromise, searchTimeoutPromise]);
+                clearTimeout(searchTimeoutId);
                 const searchDuration = Date.now() - searchStartTime;
                 // ============================================================================
                 // DEEP DEBUG: After Database Query (Success)
@@ -881,6 +928,7 @@ export class FindWhatISaid {
                 });
             }
             catch (searchError) {
+                clearTimeout(searchTimeoutId); // Prevent dangling timer on error path
                 const searchDuration = Date.now() - searchStartTime;
                 const err = searchError;
                 // ============================================================================
@@ -946,12 +994,42 @@ export class FindWhatISaid {
                     semanticResults: results.length,
                     topSimilarity: results[0]?.similarity
                 }, '[I5 FIX] Low/no semantic results, triggering keyword fallback');
-                keywordResults = await this.keywordSearch(safeParams.query, safeParams);
+                const KEYWORD_FALLBACK_TIMEOUT = parseInt(process.env['SPECMEM_KEYWORD_FALLBACK_TIMEOUT_MS'] || '30000');
+                let keywordTimeoutId;
+                try {
+                    keywordResults = await Promise.race([
+                        this.keywordSearch(safeParams.query, safeParams),
+                        new Promise((_, reject) => {
+                            keywordTimeoutId = setTimeout(() => reject(new Error(`Keyword fallback timed out after ${KEYWORD_FALLBACK_TIMEOUT}ms`)), KEYWORD_FALLBACK_TIMEOUT);
+                        })
+                    ]);
+                    clearTimeout(keywordTimeoutId);
+                }
+                catch (err) {
+                    clearTimeout(keywordTimeoutId);
+                    logger.warn({ error: err?.message, timeoutMs: KEYWORD_FALLBACK_TIMEOUT, query: safeParams.query }, '[I5 FIX] Keyword fallback timed out or failed - continuing with semantic results only');
+                    keywordResults = [];
+                }
             }
             // I5 FIX: Get recent memories if requested
             let recentResults = [];
             if (includeRecentCount > 0) {
-                recentResults = await this.getRecentMemories(includeRecentCount, safeParams);
+                const RECENT_LOOKUP_TIMEOUT = parseInt(process.env['SPECMEM_RECENT_LOOKUP_TIMEOUT_MS'] || '15000');
+                let recentTimeoutId;
+                try {
+                    recentResults = await Promise.race([
+                        this.getRecentMemories(includeRecentCount, safeParams),
+                        new Promise((_, reject) => {
+                            recentTimeoutId = setTimeout(() => reject(new Error(`Recent memories lookup timed out after ${RECENT_LOOKUP_TIMEOUT}ms`)), RECENT_LOOKUP_TIMEOUT);
+                        })
+                    ]);
+                    clearTimeout(recentTimeoutId);
+                }
+                catch (err) {
+                    clearTimeout(recentTimeoutId);
+                    logger.warn({ error: err?.message, timeoutMs: RECENT_LOOKUP_TIMEOUT, includeRecentCount }, '[I5 FIX] Recent memories lookup timed out or failed - continuing without recent results');
+                    recentResults = [];
+                }
                 logger.info({
                     recentRequested: includeRecentCount,
                     recentFound: recentResults.length
@@ -1021,47 +1099,61 @@ export class FindWhatISaid {
             // ============================================================================
             if (safeParams.galleryMode === true) {
                 logger.info({ query: safeParams.query, resultCount: results.length }, 'Gallery mode enabled - sending to Mini COT');
+                const GALLERY_TIMEOUT = parseInt(process.env['SPECMEM_GALLERY_TIMEOUT_MS'] || '60000');
+                let galleryTimeoutId;
                 try {
-                    const miniCOT = new MiniCOTProvider();
-                    // Prepare memories for gallery creation (send ENGLISH to CoT!)
-                    const memoriesForGallery = results.map(result => ({
-                        id: result.memory.id,
-                        keywords: result.memory.metadata?._semanticHints || result.memory.tags.join(', '),
-                        snippet: result.memory.content.slice(0, 300), // First 300 chars
-                        timestamp: result.memory.metadata?.timestamp, // When it was said
-                        role: result.memory.metadata?.role // Who said it (user/assistant)
-                    }));
-                    // Call Mini COT to create gallery (CoT analyzes in ENGLISH)
-                    const gallery = await miniCOT.createGallery(safeParams.query, memoriesForGallery);
-                    // ROUND-TRIP VERIFIED compression - compress CoT OUTPUT for token efficiency
-                    // Uses smartCompress: EN→Chinese→EN comparison, keeps English where context lost
-                    // MED-40 FIX: Add null check before compression to avoid undefined errors
-                    gallery.gallery = gallery.gallery.map(item => ({
-                        ...item,
-                        thumbnail: item.thumbnail ? smartCompress(item.thumbnail, { threshold: 0.75 }).result : '',
-                        cot: item.cot ? smartCompress(item.cot, { threshold: 0.75 }).result : ''
-                    }));
-                    logger.info({
-                        query: safeParams.query,
-                        galleryItems: gallery.gallery.length,
-                        researchedTerms: gallery.total_researched_terms
-                    }, 'Gallery created by Mini COT and compressed');
-                    // Always use humanReadable format
-                    const humanReadableData = gallery.gallery.map((item, idx) => ({
-                        id: item.id || `gallery-${idx}`,
-                        similarity: item.relevance ? item.relevance / 100 : 0.5,
-                        content: `[GALLERY] ${item.thumbnail || item.cot || 'No preview'}`,
-                    }));
-                    return formatHumanReadable('find_memory', humanReadableData, {
-                        grey: true,
-                        showSimilarity: true,
-                        query: safeParams.query,
-                        mode: 'gallery'
-                    });
+                    const galleryOperation = async () => {
+                        const miniCOT = new MiniCOTProvider();
+                        // Prepare memories for gallery creation (send ENGLISH to CoT!)
+                        const memoriesForGallery = results.map(result => ({
+                            id: result.memory.id,
+                            keywords: result.memory.metadata?._semanticHints || result.memory.tags.join(', '),
+                            snippet: result.memory.content.slice(0, 300), // First 300 chars
+                            timestamp: result.memory.metadata?.timestamp, // When it was said
+                            role: result.memory.metadata?.role // Who said it (user/assistant)
+                        }));
+                        // Call Mini COT to create gallery (CoT analyzes in ENGLISH)
+                        const gallery = await miniCOT.createGallery(safeParams.query, memoriesForGallery);
+                        // ROUND-TRIP VERIFIED compression - compress CoT OUTPUT for token efficiency
+                        // Uses smartCompress: EN→Chinese→EN comparison, keeps English where context lost
+                        // MED-40 FIX: Add null check before compression to avoid undefined errors
+                        gallery.gallery = gallery.gallery.map(item => ({
+                            ...item,
+                            thumbnail: item.thumbnail ? smartCompress(item.thumbnail, { threshold: 0.75 }).result : '',
+                            cot: item.cot ? smartCompress(item.cot, { threshold: 0.75 }).result : ''
+                        }));
+                        logger.info({
+                            query: safeParams.query,
+                            galleryItems: gallery.gallery.length,
+                            researchedTerms: gallery.total_researched_terms
+                        }, 'Gallery created by Mini COT and compressed');
+                        // Always use humanReadable format
+                        const humanReadableData = gallery.gallery.map((item, idx) => ({
+                            id: item.id || `gallery-${idx}`,
+                            similarity: item.relevance ? item.relevance / 100 : 0.5,
+                            content: `[GALLERY] ${item.thumbnail || item.cot || 'No preview'}`,
+                        }));
+                        return formatHumanReadable('find_memory', humanReadableData, {
+                            grey: true,
+                            showSimilarity: true,
+                            query: safeParams.query,
+                            mode: 'gallery'
+                        });
+                    };
+                    const galleryResult = await Promise.race([
+                        galleryOperation(),
+                        new Promise((_, reject) => {
+                            galleryTimeoutId = setTimeout(() => reject(new Error(`Gallery mode timed out after ${GALLERY_TIMEOUT}ms`)), GALLERY_TIMEOUT);
+                        })
+                    ]);
+                    clearTimeout(galleryTimeoutId);
+                    return galleryResult;
                 }
                 catch (error) {
-                    logger.error({ error, query: safeParams.query }, 'Mini COT gallery creation failed - falling back to normal results');
-                    // Fall through to normal results on error
+                    clearTimeout(galleryTimeoutId);
+                    const isTimeout = error?.message?.includes('timed out');
+                    logger.error({ error: error?.message, query: safeParams.query, isTimeout, timeoutMs: GALLERY_TIMEOUT }, isTimeout ? 'Gallery mode timed out - falling back to normal results' : 'Mini COT gallery creation failed - falling back to normal results');
+                    // Fall through to normal results on error or timeout
                 }
             }
             // ============================================================================

package/dist/utils/qoms.js

@@ -43,13 +43,22 @@ const CONFIG = {
     maxRetries: 3, // Max retry attempts before DLQ
     baseRetryDelayMs: 1000, // Base delay for exponential backoff (1s, 2s, 4s)
     maxRetryDelayMs: 30000, // Cap retry delay at 30s
-    leaseTimeoutMs: 60000, // 60s lease - requeue if not completed
+    leaseTimeoutMs: parseInt(process.env['SPECMEM_QOMS_LEASE_TIMEOUT'] || '120000'), // 120s lease (was 60s) - configurable via env
     agePromotionMs: 30000, // Promote priority after 30s waiting
     // DLQ settings
-    dlqMaxSize: 1000, // Max DLQ size (oldest evicted)
+    dlqMaxSize: parseInt(process.env['SPECMEM_QOMS_MAX_DLQ_SIZE'] || '500'), // Max DLQ size (oldest evicted) - Issue #8
     dlqRetentionMs: 3600000, // Keep DLQ items for 1 hour
     // Metrics cache
     metricsCacheMs: 500, // Cache metrics for 500ms
+    // Issue #5: Periodic lease expiry check interval (default 10s)
+    leaseCheckIntervalMs: parseInt(process.env['SPECMEM_QOMS_LEASE_CHECK_INTERVAL_MS'] || '10000'),
+    // Issue #8: Queue size limits (backpressure)
+    maxQueueSize: parseInt(process.env['SPECMEM_QOMS_MAX_QUEUE_SIZE'] || '1000'), // Total max across all priorities
+    maxHighQueue: parseInt(process.env['SPECMEM_QOMS_MAX_HIGH_QUEUE'] || '500'),
+    maxMediumQueue: parseInt(process.env['SPECMEM_QOMS_MAX_MEDIUM_QUEUE'] || '300'),
+    maxLowQueue: parseInt(process.env['SPECMEM_QOMS_MAX_LOW_QUEUE'] || '200'),
+    // Issue #8: Queue depth metrics logging interval (default 1min)
+    metricsIntervalMs: parseInt(process.env['SPECMEM_QOMS_METRICS_INTERVAL_MS'] || '60000'),
 };
 // ============================================================================
 // Types
@@ -91,6 +100,18 @@ let lastCpuInfo = null;
 let lastCpuTime = 0;
 // Operation ID counter
 let operationIdCounter = 0;
+// Issue #5: Periodic lease check interval handle
+let leaseCheckInterval = null;
+// Issue #8: Periodic metrics logging interval handle
+let metricsInterval = null;
+// Issue #8: Per-priority max queue size map
+const perPriorityMaxSize = new Map([
+    [Priority.CRITICAL, Infinity], // Critical operations are never rejected
+    [Priority.HIGH, CONFIG.maxHighQueue],
+    [Priority.MEDIUM, CONFIG.maxMediumQueue],
+    [Priority.LOW, CONFIG.maxLowQueue],
+    [Priority.IDLE, CONFIG.maxLowQueue], // IDLE shares LOW limit
+]);
 // ============================================================================
 // Utility Functions
 // ============================================================================
@@ -355,16 +376,37 @@ function nack(opId, error) {
 }
 /**
  * Check for lease timeouts and requeue expired items
+ * @param {boolean} periodic - Whether this was triggered by the periodic check (Issue #5)
  */
-function checkLeaseTimeouts() {
+function checkLeaseTimeouts(periodic = false) {
     const now = Date.now();
+    let expiredCount = 0;
     for (const [opId, item] of processingItems.entries()) {
         if (item.leaseExpiresAt && now > item.leaseExpiresAt) {
-            __debugLog('[QOMS DEBUG]', Date.now(), 'LEASE_TIMEOUT', { opId, expiredAgo: now - item.leaseExpiresAt });
+            const expiredAgoMs = now - item.leaseExpiresAt;
+            if (periodic) {
+                // Issue #5: Log with more detail when periodic check catches expired leases
+                logger.warn({
+                    opId,
+                    priority: Priority[item.priority],
+                    expiredAgoMs,
+                    enqueuedAt: item.enqueuedAt,
+                    startedAt: item.startedAt,
+                    retryCount: item.retryCount,
+                }, 'QOMS: Periodic lease check expired stale operation');
+            }
+            __debugLog('[QOMS DEBUG]', Date.now(), 'LEASE_TIMEOUT', {
+                opId,
+                expiredAgo: expiredAgoMs,
+                periodic,
+                priority: Priority[item.priority],
+            });
             // Treat as failure, trigger retry
             nack(opId, new Error('Lease timeout - operation took too long'));
+            expiredCount++;
         }
     }
+    return expiredCount;
 }
 // ============================================================================
 // Queue Processor
@@ -475,6 +517,116 @@ async function processQueue() {
     }
 }
 // ============================================================================
+// Issue #5: Periodic Lease Expiry Check
+// ============================================================================
+/**
+ * Start periodic lease expiry check.
+ * Runs every SPECMEM_QOMS_LEASE_CHECK_INTERVAL_MS (default 10s).
+ * If expired leases are found and released, triggers queue processing
+ * so waiting items can take the freed slots.
+ */
+function startPeriodicLeaseCheck() {
+    if (leaseCheckInterval) {
+        return; // Already running
+    }
+    const intervalMs = CONFIG.leaseCheckIntervalMs;
+    __debugLog('[QOMS DEBUG]', Date.now(), 'PERIODIC_LEASE_CHECK_START', { intervalMs });
+    leaseCheckInterval = setInterval(() => {
+        try {
+            const expiredCount = checkLeaseTimeouts(true);
+            if (expiredCount > 0) {
+                __debugLog('[QOMS DEBUG]', Date.now(), 'PERIODIC_LEASE_CHECK_EXPIRED', { expiredCount });
+                // Trigger queue processing to fill freed slots
+                processQueue().catch(err => {
+                    logger.error({ error: err }, 'QOMS: queue processing error after periodic lease check');
+                });
+            }
+        }
+        catch (err) {
+            logger.error({ error: err }, 'QOMS: periodic lease check error');
+        }
+    }, intervalMs);
+    // Prevent the interval from keeping the process alive
+    if (leaseCheckInterval && typeof leaseCheckInterval.unref === 'function') {
+        leaseCheckInterval.unref();
+    }
+}
+// ============================================================================
+// Issue #8: Periodic Queue Depth Metrics Logging
+// ============================================================================
+/**
+ * Start periodic queue depth metrics logging.
+ * Runs every SPECMEM_QOMS_METRICS_INTERVAL_MS (default 60s).
+ * Logs queue depths, processing count, DLQ size for monitoring.
+ */
+function startMetricsLogging() {
+    if (metricsInterval) {
+        return; // Already running
+    }
+    const intervalMs = CONFIG.metricsIntervalMs;
+    __debugLog('[QOMS DEBUG]', Date.now(), 'METRICS_LOGGING_START', { intervalMs });
+    metricsInterval = setInterval(() => {
+        try {
+            const queueDepths = {};
+            let totalQueued = 0;
+            for (const [priority, queue] of priorityQueues.entries()) {
+                const name = Priority[priority];
+                queueDepths[name] = queue.length;
+                totalQueued += queue.length;
+            }
+            logger.info({
+                queueDepths,
+                totalQueued,
+                processing: processingItems.size,
+                dlqSize: dlq.length,
+                totalProcessed,
+                totalRetries,
+                maxQueueSize: CONFIG.maxQueueSize,
+            }, 'QOMS: queue depth metrics');
+            __debugLog('[QOMS DEBUG]', Date.now(), 'METRICS_LOG', {
+                queueDepths,
+                totalQueued,
+                processing: processingItems.size,
+                dlqSize: dlq.length,
+            });
+        }
+        catch (err) {
+            logger.error({ error: err }, 'QOMS: metrics logging error');
+        }
+    }, intervalMs);
+    // Prevent the interval from keeping the process alive
+    if (metricsInterval && typeof metricsInterval.unref === 'function') {
+        metricsInterval.unref();
+    }
+}
+// ============================================================================
+// Issue #5 + #8: Cleanup / Destroy
+// ============================================================================
+/**
+ * Cleanup QOMS - clears all intervals and timers.
+ * Call this on shutdown to prevent resource leaks.
+ */
+function cleanup() {
+    if (leaseCheckInterval) {
+        clearInterval(leaseCheckInterval);
+        leaseCheckInterval = null;
+        __debugLog('[QOMS DEBUG]', Date.now(), 'PERIODIC_LEASE_CHECK_STOPPED');
+    }
+    if (metricsInterval) {
+        clearInterval(metricsInterval);
+        metricsInterval = null;
+        __debugLog('[QOMS DEBUG]', Date.now(), 'METRICS_LOGGING_STOPPED');
+    }
+    logger.info('QOMS: cleanup complete - all intervals cleared');
+}
+// Alias for cleanup
+const destroy = cleanup;
+// ============================================================================
+// Auto-start periodic checks
+// ============================================================================
+startPeriodicLeaseCheck();
+startMetricsLogging();
+// ============================================================================
 // Public API
 // ============================================================================
 /**
@@ -491,6 +643,26 @@ export async function enqueue(operation, priority = Priority.MEDIUM) {
         priority: Priority[priority],
         totalQueued: getTotalQueueLength()
     });
+    // Issue #8: Check queue size limits (backpressure) - skip for CRITICAL priority
+    if (priority !== Priority.CRITICAL) {
+        const totalQueued = getTotalQueueLength();
+        // Check total queue size limit
+        if (totalQueued >= CONFIG.maxQueueSize) {
+            const errMsg = `QOMS: Queue full (${totalQueued}/${CONFIG.maxQueueSize}). Rejecting operation ${opId} with priority ${Priority[priority]}. Configure SPECMEM_QOMS_MAX_QUEUE_SIZE to increase limit.`;
+            logger.warn({ opId, priority: Priority[priority], totalQueued, maxQueueSize: CONFIG.maxQueueSize }, errMsg);
+            __debugLog('[QOMS DEBUG]', Date.now(), 'QUEUE_FULL_REJECTED', { opId, totalQueued, maxQueueSize: CONFIG.maxQueueSize });
+            throw new Error(errMsg);
+        }
+        // Check per-priority queue size limit
+        const priorityQueue = priorityQueues.get(priority);
+        const maxForPriority = perPriorityMaxSize.get(priority) ?? CONFIG.maxQueueSize;
+        if (priorityQueue.length >= maxForPriority) {
+            const errMsg = `QOMS: ${Priority[priority]} queue full (${priorityQueue.length}/${maxForPriority}). Rejecting operation ${opId}. Configure SPECMEM_QOMS_MAX_${Priority[priority]}_QUEUE to increase limit.`;
+            logger.warn({ opId, priority: Priority[priority], queueLength: priorityQueue.length, maxForPriority }, errMsg);
+            __debugLog('[QOMS DEBUG]', Date.now(), 'PRIORITY_QUEUE_FULL_REJECTED', { opId, priority: Priority[priority], queueLength: priorityQueue.length, maxForPriority });
+            throw new Error(errMsg);
+        }
+    }
     // Check if we can execute immediately (empty queue, resources available)
     const queue = priorityQueues.get(priority);
     if (getTotalQueueLength() === 0 && processingItems.size === 0 && canExecute(priority, opId)) {
@@ -568,10 +740,19 @@ export function getQueueStats() {
         pendingRetries,
         totalRetries,
         dlqSize: dlq.length,
+        dlqMaxSize: CONFIG.dlqMaxSize,
         isProcessing,
         avgWaitTimeMs: totalProcessed > 0 ? totalWaitTimeMs / totalProcessed : 0,
         metrics: getSystemMetrics(),
         limits: CONFIG,
+        // Issue #8: Queue capacity info
+        queueCapacity: {
+            maxTotal: CONFIG.maxQueueSize,
+            maxHigh: CONFIG.maxHighQueue,
+            maxMedium: CONFIG.maxMediumQueue,
+            maxLow: CONFIG.maxLowQueue,
+            remainingTotal: CONFIG.maxQueueSize - getTotalQueueLength(),
+        },
     };
 }
 /**
@@ -644,6 +825,8 @@ export const qoms = {
     getDLQ,
     clearDLQ,
     retryDLQItem,
+    cleanup,
+    destroy,
     Priority,
 };
 export default qoms;

package/dist/watcher/changeHandler.js

@@ -20,6 +20,32 @@ import { logger } from '../utils/logger.js';
 import { getCoordinator } from '../coordination/integration.js';
 import { isMinifiedOrBundled, isBinaryFile, EXCLUSION_CONFIG } from '../codebase/exclusions.js';
 import { getProjectPathForInsert } from '../services/ProjectContext.js';
+import { getEmbeddingTimeout } from '../config/embeddingTimeouts.js';
+// Retry helper for transient embedding failures (timeout, socket reset, etc.)
+const WATCHER_MAX_RETRIES = parseInt(process.env['SPECMEM_WATCHER_RETRIES'] || '2');
+async function withWatcherRetry(operation, filePath) {
+    let lastError = null;
+    for (let attempt = 0; attempt <= WATCHER_MAX_RETRIES; attempt++) {
+        try {
+            return await operation();
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : new Error(String(error));
+            const msg = lastError.message.toLowerCase();
+            const isTransient = msg.includes('timeout') || msg.includes('econnreset') ||
+                msg.includes('econnrefused') || msg.includes('socket') || msg.includes('qoms');
+            if (attempt < WATCHER_MAX_RETRIES && isTransient) {
+                const delay = Math.min(1000 * Math.pow(2, attempt), 8000);
+                logger.warn({ filePath, attempt: attempt + 1, retryInMs: delay, error: lastError.message }, `[Watcher] Embedding retry ${attempt + 1}/${WATCHER_MAX_RETRIES}`);
+                await new Promise(resolve => setTimeout(resolve, delay));
+            }
+            else {
+                break;
+            }
+        }
+    }
+    throw lastError;
+}
 /**
  * autoUpdateTheMemories - main change handler class
  *
@@ -135,10 +161,22 @@ export class AutoUpdateTheMemories {
                 this.stats.filesSkipped++;
                 return;
             }
-            // generate embedding with retry and queue fallback
+            // generate embedding with retry + timeout protection
             let embedding;
+            const WATCHER_EMBEDDING_TIMEOUT = getEmbeddingTimeout('fileWatcher');
             try {
-                embedding = await this.config.embeddingProvider.generateEmbedding(content);
+                embedding = await withWatcherRetry(async () => {
+                    return new Promise((resolve, reject) => {
+                        const timeoutId = setTimeout(() => {
+                            const err = new Error(`[Watcher] Embedding generation timed out after ${Math.round(WATCHER_EMBEDDING_TIMEOUT / 1000)}s for ${metadata.relativePath}`);
+                            err.code = 'WATCHER_EMBEDDING_TIMEOUT';
+                            reject(err);
+                        }, WATCHER_EMBEDDING_TIMEOUT);
+                        this.config.embeddingProvider.generateEmbedding(content)
+                            .then(result => { clearTimeout(timeoutId); resolve(result); })
+                            .catch(error => { clearTimeout(timeoutId); reject(error); });
+                    });
+                }, metadata.relativePath);
             }
             catch (embeddingError) {
                 logger.error({
@@ -236,10 +274,22 @@ export class AutoUpdateTheMemories {
                 this.stats.filesSkipped++;
                 return;
             }
-            // generate new embedding with retry and queue fallback
+            // generate new embedding with retry and queue fallback + timeout protection
             let embedding;
+            const WATCHER_EMBEDDING_TIMEOUT_MOD = getEmbeddingTimeout('fileWatcher');
             try {
-                embedding = await this.config.embeddingProvider.generateEmbedding(content);
+                embedding = await withWatcherRetry(async () => {
+                    return new Promise((resolve, reject) => {
+                        const timeoutId = setTimeout(() => {
+                            const err = new Error(`[Watcher] Embedding generation timed out after ${Math.round(WATCHER_EMBEDDING_TIMEOUT_MOD / 1000)}s for ${metadata.relativePath}`);
+                            err.code = 'WATCHER_EMBEDDING_TIMEOUT';
+                            reject(err);
+                        }, WATCHER_EMBEDDING_TIMEOUT_MOD);
+                        this.config.embeddingProvider.generateEmbedding(content)
+                            .then(result => { clearTimeout(timeoutId); resolve(result); })
+                            .catch(error => { clearTimeout(timeoutId); reject(error); });
+                    });
+                }, metadata.relativePath);
             }
             catch (embeddingError) {
                 logger.error({