specmem-hardwicksoftware 3.5.99 → 3.6.1

package/dist/watcher/fileWatcher.js

@@ -44,6 +44,13 @@ export class WatchForChangesNoCap {
     debouncedHandlers = new Map();
     // FIX MED-14 & LOW-15: Track latest event data per key to avoid stale closures
     pendingEventData = new Map();
+    // FIX Issue #13: Track when each debounce entry was created for age-based cleanup
+    debounceEntryTimestamps = new Map();
+    // FIX Issue #13: Configurable limits for debounce map growth
+    debounceMapMaxSize = parseInt(process.env['SPECMEM_DEBOUNCE_MAP_MAX_SIZE'] || '5000');
+    debounceCleanupIntervalMs = parseInt(process.env['SPECMEM_DEBOUNCE_CLEANUP_INTERVAL_MS'] || '300000'); // 5 min
+    debounceMaxAgeMs = parseInt(process.env['SPECMEM_DEBOUNCE_MAX_AGE_MS'] || '60000'); // 60s
+    debounceCleanupTimer = null;
     // stats tracking
     stats = {
         filesWatched: 0,
@@ -130,6 +137,11 @@ export class WatchForChangesNoCap {
                     '**/memory-dumps/**',
                     '**/tmp/**',
                     '**/temp/**',
+                    // SpecMem runtime directories - NEVER watch these
+                    // statusbar-state.json changes every few seconds causing infinite feedback loops
+                    '**/specmem/sockets/**',
+                    '**/specmem/run/**',
+                    '**/.specmem/**',
                     // Test artifacts
                     '**/coverage/**',
                     '**/__pycache__/**',
@@ -185,6 +197,10 @@ export class WatchForChangesNoCap {
             this.setupEventHandlers();
             this.isWatching = true;
             this.restartCount = 0;
+            // FIX Issue #13: Start periodic cleanup of stale debounce entries
+            this.debounceCleanupTimer = setInterval(() => {
+                this.cleanupStaleDebounceEntries();
+            }, this.debounceCleanupIntervalMs);
             logger.info('file watcher started successfully - were LIVE');
         }
         catch (error) {
@@ -205,6 +221,11 @@ export class WatchForChangesNoCap {
             await this.watcher.close();
             this.watcher = null;
         }
+        // FIX Issue #13: Clear debounce cleanup timer
+        if (this.debounceCleanupTimer) {
+            clearInterval(this.debounceCleanupTimer);
+            this.debounceCleanupTimer = null;
+        }
         // FIX MED-13: Cancel all debounced handlers before clearing to prevent memory leaks
         // The debounce library's clear() method cancels pending timer execution
         for (const handler of this.debouncedHandlers.values()) {
@@ -212,6 +233,7 @@ export class WatchForChangesNoCap {
         }
         this.debouncedHandlers.clear();
         this.pendingEventData.clear();
+        this.debounceEntryTimestamps.clear();
         this.isWatching = false;
         logger.info({ stats: this.stats }, 'file watcher stopped - peace out');
     }
@@ -319,8 +341,14 @@ export class WatchForChangesNoCap {
             // already debouncing this file - the handler will use updated pendingEventData
             // FIX MED-14: Don't return early - we've already updated the event data above
             // The debounced function will pick up the latest event when it fires
+            // Update the timestamp for the existing entry
+            this.debounceEntryTimestamps.set(key, Date.now());
             return;
         }
+        // FIX Issue #13: If debounce map exceeds max size, flush oldest entries immediately
+        if (this.debouncedHandlers.size >= this.debounceMapMaxSize) {
+            this.flushOldestDebounceEntries();
+        }
         // create debounced handler that reads latest event data when executing
         const debouncedHandler = debounce(async () => {
             try {
@@ -342,17 +370,109 @@ export class WatchForChangesNoCap {
                 logger.error({ error, event: latestEvent }, 'error processing file change');
             }
             finally {
-                // remove from debounce map and pending data
+                // remove from debounce map, pending data, and timestamps
                 this.debouncedHandlers.delete(key);
                 this.pendingEventData.delete(key);
+                this.debounceEntryTimestamps.delete(key);
             }
         }, this.config.debounceMs);
         this.debouncedHandlers.set(key, debouncedHandler);
+        // FIX Issue #13: Track when this debounce entry was created
+        this.debounceEntryTimestamps.set(key, Date.now());
         debouncedHandler();
         if (this.config.verbose) {
             logger.debug({ event }, 'file change detected');
         }
     }
+    /**
+     * FIX Issue #13: Flush the oldest debounce entries immediately when map exceeds max size.
+     * Fires the debounced handlers immediately (processes them without waiting for debounce timer).
+     */
+    flushOldestDebounceEntries() {
+        // Sort entries by timestamp (oldest first) and flush 10% of the map
+        const entries = [...this.debounceEntryTimestamps.entries()]
+            .sort((a, b) => a[1] - b[1]);
+        const flushCount = Math.max(1, Math.ceil(entries.length * 0.1));
+        let flushedCount = 0;
+        for (let i = 0; i < flushCount && i < entries.length; i++) {
+            const [key] = entries[i];
+            const handler = this.debouncedHandlers.get(key);
+            if (handler) {
+                // Cancel the debounce timer and fire immediately
+                handler.clear();
+                this.debouncedHandlers.delete(key);
+                this.debounceEntryTimestamps.delete(key);
+                // Fire the handler immediately with the pending event data
+                const latestEvent = this.pendingEventData.get(key);
+                if (latestEvent && this.changeHandler) {
+                    this.pendingEventData.delete(key);
+                    // Fire and forget - don't await, just dispatch
+                    Promise.resolve().then(async () => {
+                        try {
+                            latestEvent.timestamp = new Date();
+                            await this.changeHandler(latestEvent);
+                            this.stats.eventsProcessed++;
+                            this.stats.lastEventTime = new Date();
+                        }
+                        catch (error) {
+                            this.stats.errors++;
+                            logger.error({ error, event: latestEvent }, 'error processing flushed debounce entry');
+                        }
+                    });
+                }
+                else {
+                    this.pendingEventData.delete(key);
+                }
+                flushedCount++;
+            }
+        }
+        if (flushedCount > 0) {
+            logger.warn({ flushedCount, mapSize: this.debouncedHandlers.size, maxSize: this.debounceMapMaxSize }, 'Flushed oldest debounce entries due to map size limit');
+        }
+    }
+    /**
+     * FIX Issue #13: Periodic cleanup of stale debounce entries older than debounceMaxAgeMs.
+     * Prevents unbounded growth when debounce timers keep getting reset.
+     */
+    cleanupStaleDebounceEntries() {
+        const now = Date.now();
+        let cleanedCount = 0;
+        for (const [key, timestamp] of this.debounceEntryTimestamps.entries()) {
+            if (now - timestamp > this.debounceMaxAgeMs) {
+                const handler = this.debouncedHandlers.get(key);
+                if (handler) {
+                    // Cancel the debounce timer and fire immediately
+                    handler.clear();
+                    this.debouncedHandlers.delete(key);
+                    // Fire the handler immediately with the pending event data
+                    const latestEvent = this.pendingEventData.get(key);
+                    if (latestEvent && this.changeHandler) {
+                        this.pendingEventData.delete(key);
+                        Promise.resolve().then(async () => {
+                            try {
+                                latestEvent.timestamp = new Date();
+                                await this.changeHandler(latestEvent);
+                                this.stats.eventsProcessed++;
+                                this.stats.lastEventTime = new Date();
+                            }
+                            catch (error) {
+                                this.stats.errors++;
+                                logger.error({ error, event: latestEvent }, 'error processing stale debounce entry');
+                            }
+                        });
+                    }
+                    else {
+                        this.pendingEventData.delete(key);
+                    }
+                }
+                this.debounceEntryTimestamps.delete(key);
+                cleanedCount++;
+            }
+        }
+        if (cleanedCount > 0) {
+            logger.warn({ cleanedCount, remainingEntries: this.debouncedHandlers.size }, 'Cleaned up stale debounce entries');
+        }
+    }
     /**
      * skipTheBoringShit - checks if file should be ignored
      *

package/dist/watcher/index.js

@@ -27,6 +27,8 @@ export class WatcherManager {
     syncChecker;
     isRunning = false;
     syncInterval = null;
+    syncInProgress = false;
+    syncTimeout = null;
     constructor(config) {
         // Create handler first - it's the core component
         this.handler = new AutoUpdateTheMemories(config.handler);
@@ -59,36 +61,13 @@ export class WatcherManager {
             });
             // Mark as running IMMEDIATELY - don't wait for sync
             this.isRunning = true;
-            // 3. start periodic sync checking (this is cheap, just sets up interval)
-            this.syncInterval = setInterval(async () => {
-                try {
-                    const report = await this.syncChecker.checkSync();
-                    await this.writeSyncScore(report.syncScore);
-                    if (!report.inSync) {
-                        logger.warn({
-                            driftPercentage: report.driftPercentage,
-                            missingFromMcp: report.missingFromMcp.length,
-                            contentMismatch: report.contentMismatch.length
-                        }, 'drift detected during periodic check');
-                        // Auto-resync when drift is detected
-                        if (report.missingFromMcp.length > 0 || report.contentMismatch.length > 0) {
-                            logger.info('periodic check triggering auto-resync...');
-                            const resyncResult = await this.syncChecker.resyncEverythingFrFr();
-                            logger.info({
-                                filesAdded: resyncResult.filesAdded,
-                                filesUpdated: resyncResult.filesUpdated,
-                                errors: resyncResult.errors.length
-                            }, 'periodic auto-resync complete');
-                            // Update score after resync
-                            const postReport = await this.syncChecker.checkSync();
-                            await this.writeSyncScore(postReport.syncScore);
-                        }
-                    }
-                }
-                catch (error) {
-                    logger.error({ error }, 'periodic sync check failed');
-                }
-            }, syncCheckIntervalMinutes * 60 * 1000);
+            // 3. start periodic sync checking using setTimeout + recursive scheduling
+            // This prevents sync interval stacking (Issue #2) - if a sync takes longer
+            // than the interval, the next one won't start until the previous completes.
+            // Interval is configurable via SPECMEM_SYNC_CHECK_INTERVAL_MS env var.
+            const syncIntervalMs = parseInt(process.env['SPECMEM_SYNC_CHECK_INTERVAL_MS'] || String(syncCheckIntervalMinutes * 60 * 1000), 10);
+            logger.info({ syncIntervalMs }, 'scheduling periodic sync checks with recursive setTimeout');
+            this.scheduleSyncCheck(syncIntervalMs);
             logger.info('watcher manager started - ready for changes');
             // 4. BACKGROUND sync check - does NOT block startup
             // Runs the full filesystem scan in background so  becomes responsive immediately
@@ -104,6 +83,67 @@ export class WatcherManager {
             throw error;
         }
     }
+    /**
+     * scheduleSyncCheck - schedules the next sync check using setTimeout
+     * Uses recursive scheduling instead of setInterval to prevent stacking (Issue #2).
+     * The next check is only scheduled AFTER the current one completes.
+     */
+    scheduleSyncCheck(intervalMs) {
+        if (!this.isRunning) {
+            return;
+        }
+        this.syncTimeout = setTimeout(async () => {
+            await this.runPeriodicSync();
+            // Schedule next check only after current one completes - prevents stacking
+            this.scheduleSyncCheck(intervalMs);
+        }, intervalMs);
+        // Allow process to exit even if timeout is pending
+        if (this.syncTimeout && typeof this.syncTimeout.unref === 'function') {
+            this.syncTimeout.unref();
+        }
+    }
+    /**
+     * runPeriodicSync - executes a single periodic sync check with guard
+     * Uses syncInProgress flag to prevent concurrent sync operations (Issue #2).
+     * If a sync is already running, logs a skip and returns immediately.
+     */
+    async runPeriodicSync() {
+        if (this.syncInProgress) {
+            logger.warn('periodic sync check skipped - another sync is already in progress (preventing stacking)');
+            return;
+        }
+        this.syncInProgress = true;
+        try {
+            const report = await this.syncChecker.checkSync();
+            await this.writeSyncScore(report.syncScore);
+            if (!report.inSync) {
+                logger.warn({
+                    driftPercentage: report.driftPercentage,
+                    missingFromMcp: report.missingFromMcp.length,
+                    contentMismatch: report.contentMismatch.length
+                }, 'drift detected during periodic check');
+                // Auto-resync when drift is detected
+                if (report.missingFromMcp.length > 0 || report.contentMismatch.length > 0) {
+                    logger.info('periodic check triggering auto-resync...');
+                    const resyncResult = await this.syncChecker.resyncEverythingFrFr();
+                    logger.info({
+                        filesAdded: resyncResult.filesAdded,
+                        filesUpdated: resyncResult.filesUpdated,
+                        errors: resyncResult.errors.length
+                    }, 'periodic auto-resync complete');
+                    // Update score after resync
+                    const postReport = await this.syncChecker.checkSync();
+                    await this.writeSyncScore(postReport.syncScore);
+                }
+            }
+        }
+        catch (error) {
+            logger.error({ error }, 'periodic sync check failed');
+        }
+        finally {
+            this.syncInProgress = false;
+        }
+    }
     /**
      * runBackgroundSyncCheck - runs sync check in background without blocking
      *
@@ -169,11 +209,15 @@ export class WatcherManager {
             return;
         }
         logger.info('stopping watcher manager...');
-        // stop sync checking
+        // stop sync checking (clear both legacy interval and new timeout)
         if (this.syncInterval) {
             clearInterval(this.syncInterval);
             this.syncInterval = null;
         }
+        if (this.syncTimeout) {
+            clearTimeout(this.syncTimeout);
+            this.syncTimeout = null;
+        }
         // stop file watcher
         await this.watcher.stopWatching();
         // stop queue processing