specmem-hardwicksoftware 3.7.34 → 3.7.36

package/dist/watcher/changeHandler.js

@@ -21,6 +21,7 @@ import { getCoordinator } from '../coordination/integration.js';
 import { isMinifiedOrBundled, isBinaryFile, EXCLUSION_CONFIG } from '../codebase/exclusions.js';
 import { getProjectPathForInsert } from '../services/ProjectContext.js';
 import { getEmbeddingTimeout } from '../config/embeddingTimeouts.js';
+import { extractPdfText, isPdfFile } from '../codebase/pdfExtractor.js';
 // Retry helper for transient embedding failures (timeout, socket reset, etc.)
 const WATCHER_MAX_RETRIES = parseInt(process.env['SPECMEM_WATCHER_RETRIES'] || '2');
 async function withWatcherRetry(operation, filePath) {
@@ -132,6 +133,51 @@ export class AutoUpdateTheMemories {
                 this.stats.filesSkipped++;
                 return;
             }
+            // PDF files: extract text via PyMuPDF instead of reading as UTF-8
+            if (isPdfFile(event.path)) {
+                const pdfResult = await extractPdfText(event.path);
+                if (!pdfResult || !pdfResult.text) {
+                    logger.debug({ path: event.relativePath }, 'PDF extraction failed or empty — skipping');
+                    this.stats.filesSkipped++;
+                    return;
+                }
+                // Use extracted PDF text as content for the standard metadata flow
+                const metadata = await this.extractFileMetadata(event.path, event.relativePath, pdfResult.text);
+                if (metadata.size > this.config.maxFileSizeBytes) {
+                    this.stats.filesSkipped++;
+                    return;
+                }
+                const content = pdfResult.text;
+                const existingMemory = await this.findMemoryByContentHash(metadata.contentHash);
+                if (existingMemory) {
+                    this.stats.filesSkipped++;
+                    return;
+                }
+                let embedding;
+                const WATCHER_EMBEDDING_TIMEOUT = getEmbeddingTimeout('fileWatcher');
+                try {
+                    embedding = await withWatcherRetry(async () => {
+                        return new Promise((resolve, reject) => {
+                            const timeoutId = setTimeout(() => reject(new Error('Embedding timeout for PDF')), WATCHER_EMBEDDING_TIMEOUT);
+                            this.config.embeddingProvider.generateEmbedding(content)
+                                .then(result => { clearTimeout(timeoutId); resolve(result); })
+                                .catch(error => { clearTimeout(timeoutId); reject(error); });
+                        });
+                    }, event.path);
+                } catch (embErr) {
+                    logger.warn({ path: event.relativePath, error: embErr.message }, 'PDF embedding failed — storing without embedding');
+                }
+                await this.storeMemory({
+                    content,
+                    metadata,
+                    embedding,
+                    tags: ['codebase', 'auto-ingested', 'pdf'],
+                });
+                this.stats.filesIngested++;
+                logger.info({ path: event.relativePath, pages: pdfResult.pages, chars: pdfResult.chars }, 'PDF file indexed');
+                this.coordinator.emitFileAdded(event.path, event.relativePath, metadata.size);
+                return;
+            }
             // check if binary
             if (await isBinaryFile(event.path)) {
                 logger.debug({ path: event.relativePath }, 'skipping binary file');
@@ -250,14 +296,25 @@ export class AutoUpdateTheMemories {
                 this.stats.filesSkipped++;
                 return;
             }
-            // FIX MED-13: Check binary before extractFileMetadata (same as handleFileAdded)
-            if (await isBinaryFile(event.path)) {
-                logger.debug({ path: event.relativePath }, 'skipping binary file update');
-                this.stats.filesSkipped++;
-                return;
+            // PDF files: extract text via PyMuPDF
+            let pdfContent = null;
+            if (isPdfFile(event.path)) {
+                const pdfResult = await extractPdfText(event.path);
+                if (!pdfResult || !pdfResult.text) {
+                    this.stats.filesSkipped++;
+                    return;
+                }
+                pdfContent = pdfResult.text;
+            } else {
+                // FIX MED-13: Check binary before extractFileMetadata (same as handleFileAdded)
+                if (await isBinaryFile(event.path)) {
+                    logger.debug({ path: event.relativePath }, 'skipping binary file update');
+                    this.stats.filesSkipped++;
+                    return;
+                }
             }
             // extract new metadata (FIX 7.04: content included to avoid double read)
-            const metadata = await this.extractFileMetadata(event.path, event.relativePath);
+            const metadata = await this.extractFileMetadata(event.path, event.relativePath, pdfContent);
             // check file size
             if (metadata.size > this.config.maxFileSizeBytes) {
                 logger.warn({
@@ -419,9 +476,9 @@ export class AutoUpdateTheMemories {
     /**
      * extractFileMetadata - reads file and generates metadata
      */
-    async extractFileMetadata(path, relativePath) {
+    async extractFileMetadata(path, relativePath, preExtractedContent) {
         const stats = await fs.stat(path);
-        const content = await fs.readFile(path, 'utf-8');
+        const content = preExtractedContent || await fs.readFile(path, 'utf-8');
         const contentHash = this.hashContent(content);
         return {
             path,

package/dist/watcher/changeQueue.js

@@ -22,6 +22,7 @@ export class QueueTheChangesUp {
     config;
     queue = [];
     processing = false;
+    paused = false; // pause queue processing without stopping (e.g. during background indexing)
     processingInterval = null;
     changeHandler;
     // deduplication map: path -> latest queued change
@@ -162,13 +163,31 @@ export class QueueTheChangesUp {
             logger.debug({ cancelledTimeouts: cancelledCount }, 'cancelled pending retry timeouts');
         }
     }
+    /**
+     * pause - temporarily halt batch processing without stopping the queue.
+     * Changes still enqueue but won't be processed until resume().
+     * Used during background indexing to avoid resource contention.
+     */
+    pause(reason = '') {
+        if (this.paused) return;
+        this.paused = true;
+        logger.info({ reason, pendingCount: this.queue.length }, 'queue PAUSED');
+    }
+    /**
+     * resume - resume batch processing after pause
+     */
+    resume() {
+        if (!this.paused) return;
+        this.paused = false;
+        logger.info({ pendingCount: this.queue.length }, 'queue RESUMED');
+    }
     /**
      * processBatch - processes a batch of changes
      *
      * nah bruh processing this whole batch at once
      */
     async processBatch() {
-        if (this.queue.length === 0) {
+        if (this.paused || this.queue.length === 0) {
             return;
         }
         logger.debug({

package/dist/watcher/index.js

@@ -31,6 +31,12 @@ export class WatcherManager {
     syncTimeout = null;
     lastLowScoreResyncAt = 0;
     lastLowScoreResyncScore = null; // track score at last resync to detect drops
+    // Drift-resync plateau detection: stop resyncing if score isn't improving
+    lastDriftResyncAt = 0;
+    lastDriftResyncScore = null;
+    driftResyncNoImprovementCount = 0; // consecutive resyncs that didn't improve score
+    static DRIFT_RESYNC_MAX_NO_IMPROVEMENT = 2; // after 2 consecutive no-improvement resyncs, accept plateau
+    static DRIFT_RESYNC_COOLDOWN_MS = 15 * 60 * 1000; // 15 min cooldown between drift resyncs
     constructor(config) {
         // Create handler first - it's the core component
         this.handler = new AutoUpdateTheMemories(config.handler);
@@ -126,7 +132,7 @@ export class WatcherManager {
             const LOW_SCORE_THRESHOLD = parseFloat(process.env['SPECMEM_LOW_SCORE_THRESHOLD'] || '0.85');
             const LOW_SCORE_DROP_THRESHOLD = parseFloat(process.env['SPECMEM_LOW_SCORE_DROP_THRESHOLD'] || '0.10');
             const LOW_SCORE_DEBOUNCE_MS = parseInt(process.env['SPECMEM_LOW_SCORE_DEBOUNCE_MS'] || String(15 * 60 * 1000), 10);
-            if (report.syncScore < LOW_SCORE_THRESHOLD) {
+            if (report.syncScore <= LOW_SCORE_THRESHOLD) {
                 // First time seeing low score — always resync
                 // After that, only resync if score dropped by >=10% from the post-resync score
                 const scoreDrop = this.lastLowScoreResyncScore !== null
@@ -163,8 +169,21 @@ export class WatcherManager {
                     missingFromMcp: report.missingFromMcp.length,
                     contentMismatch: report.contentMismatch.length
                 }, 'drift detected during periodic check');
-                // Auto-resync when drift is detected
+                // Auto-resync when drift is detected — with plateau detection + cooldown
                 if (report.missingFromMcp.length > 0 || report.contentMismatch.length > 0) {
+                    // Plateau guard: if we've resynced N times without improvement, accept the score
+                    if (this.driftResyncNoImprovementCount >= WatcherManager.DRIFT_RESYNC_MAX_NO_IMPROVEMENT) {
+                        logger.info({ syncScore: report.syncScore, noImprovementCount: this.driftResyncNoImprovementCount }, 'drift-resync plateau reached — score is stable, accepting current sync level');
+                        return;
+                    }
+                    // Cooldown guard: don't resync more than once per 15 min via drift path
+                    const now = Date.now();
+                    const driftCooldownRemaining = WatcherManager.DRIFT_RESYNC_COOLDOWN_MS - (now - this.lastDriftResyncAt);
+                    if (driftCooldownRemaining > 0) {
+                        logger.debug({ syncScore: report.syncScore, cooldownRemainingSec: Math.round(driftCooldownRemaining / 1000) }, 'drift-resync on cooldown — skipping');
+                        return;
+                    }
+                    this.lastDriftResyncAt = now;
                     logger.info('periodic check triggering auto-resync...');
                     const resyncResult = await this.syncChecker.resyncEverythingFrFr();
                     logger.info({
@@ -175,6 +194,16 @@ export class WatcherManager {
                     // Update score after resync
                     const postReport = await this.syncChecker.checkSync();
                     await this.writeSyncScore(postReport.syncScore);
+                    // Plateau detection: did this resync actually improve the score?
+                    const improvement = postReport.syncScore - (this.lastDriftResyncScore ?? 0);
+                    if (improvement < 0.01) { // less than 1% improvement = no meaningful change
+                        this.driftResyncNoImprovementCount++;
+                        logger.warn({ syncScore: postReport.syncScore, previousScore: this.lastDriftResyncScore, noImprovementCount: this.driftResyncNoImprovementCount, maxAllowed: WatcherManager.DRIFT_RESYNC_MAX_NO_IMPROVEMENT }, 'drift-resync did not improve score — tracking plateau');
+                    } else {
+                        // Score improved — reset plateau counter
+                        this.driftResyncNoImprovementCount = 0;
+                    }
+                    this.lastDriftResyncScore = postReport.syncScore;
                 }
             }
         }
@@ -307,6 +336,12 @@ export class WatcherManager {
      * resync - manually trigger full resync
      */
     async resync() {
+        // Manual resync resets all plateau/cooldown state so it always runs fresh
+        this.driftResyncNoImprovementCount = 0;
+        this.lastDriftResyncAt = 0;
+        this.lastDriftResyncScore = null;
+        this.lastLowScoreResyncScore = null;
+        this.lastLowScoreResyncAt = 0;
         return await this.syncChecker.resyncEverythingFrFr();
     }
     /**

package/embedding-sandbox/mini-cot-service.py

@@ -391,20 +391,18 @@ class ModelManager:
                     file_name="model_quantized.onnx"
                 )
             else:
-                # Fallback: download PyTorch model from HuggingFace
-                print(f"🧠 No local model, downloading {self.model_name}...", file=sys.stderr)
-                from transformers import AutoModelForCausalLM
-                cache_dir = '/tmp/mini-cot-models'
-                os.makedirs(cache_dir, exist_ok=True)
-                self.torch_model = AutoModelForCausalLM.from_pretrained(
-                    self.model_name,
-                    cache_dir=cache_dir,
-                    torch_dtype=torch.float32,
-                    low_cpu_mem_usage=True,
-                    trust_remote_code=True
+                # No local model found — refuse to download from the internet
+                search_paths = [
+                    os.environ.get('SPECMEM_MODEL_CACHE', '(not set)'),
+                    os.path.join(os.path.dirname(__file__), '..', 'models', 'pythia-410m-onnx-quant'),
+                    '/app/models/pythia-onnx-quant',
+                ]
+                raise RuntimeError(
+                    f"Local ONNX model not found. Searched:\n"
+                    + "\n".join(f"  - {p}" for p in search_paths)
+                    + "\n\nRun `specmem init` to download models via Git LFS release tarball."
+                    + "\nSpecMem will NOT download models from the internet at runtime."
                 )
-                self.torch_model.eval()
-                torch.set_grad_enabled(False)
 
             self.torch_loaded = True
             print(f"🧠 Generation model loaded for crawl analysis", file=sys.stderr)

package/embedding-sandbox/pdf-text-extract.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""
+pdf-text-extract.py — PDF text extraction for SpecMem codebase indexing
+
+Uses PyMuPDF (fitz) for instant digital PDF text extraction (0.003s/page).
+Falls back to Tesseract OCR via PyMuPDF's built-in integration for scanned pages.
+
+Usage:
+    # Single file mode
+    python3 pdf-text-extract.py <pdf_path> [--max-pages N] [--language LANG]
+
+    # Batch mode (JSONL — one result per line, one Python startup for N PDFs)
+    python3 pdf-text-extract.py --batch file1.pdf file2.pdf ... [--max-pages N]
+
+Output (JSON/JSONL to stdout):
+    {"path": "/abs/path.pdf", "text": "...", "pages": 5, "scanned_pages": [3], "chars": 12345}
+    {"path": "/abs/path2.pdf", "error": "..."}
+"""
+
+import sys
+import json
+import os
+import argparse
+
+# ---------------------------------------------------------------------------
+# Auto-install pymupdf if missing (matches frankenstein-embeddings pattern)
+# ---------------------------------------------------------------------------
+def _ensure_pymupdf():
+    try:
+        import pymupdf
+        return pymupdf
+    except ImportError:
+        pass
+    # Try legacy import name
+    try:
+        import fitz
+        return fitz
+    except ImportError:
+        pass
+    # Auto-install
+    try:
+        import subprocess
+        sys.stderr.write('[pdf-text-extract] pymupdf not found, installing...\n')
+        subprocess.check_call(
+            [sys.executable, '-m', 'pip', 'install', '--quiet', 'pymupdf'],
+            stdout=subprocess.DEVNULL
+        )
+        try:
+            import pymupdf
+            return pymupdf
+        except ImportError:
+            import fitz
+            return fitz
+    except Exception as e:
+        _error_exit(f'Failed to install pymupdf: {e}')
+
+
+def _error_exit(msg):
+    """Print error JSON and exit."""
+    print(json.dumps({'error': str(msg)}, ensure_ascii=False))
+    sys.exit(1)
+
+
+def _is_scanned_page(page, text):
+    """
+    Heuristic: page is likely scanned if:
+    1. Extracted text is very short (< 50 chars after stripping)
+    2. Page has images covering >60% of page area
+    """
+    stripped = text.strip()
+    if len(stripped) > 50:
+        return False
+
+    try:
+        images = page.get_image_info()
+        if not images:
+            return False
+        page_area = abs(page.rect)
+        if page_area == 0:
+            return False
+        image_area = 0
+        for img in images:
+            if 'bbox' in img:
+                try:
+                    import pymupdf
+                    r = pymupdf.Rect(img['bbox'])
+                except (ImportError, Exception):
+                    import fitz
+                    r = fitz.Rect(img['bbox'])
+                image_area += abs(r)
+        return (image_area / page_area) >= 0.6
+    except Exception:
+        return False
+
+
+def _ocr_page(page, language='eng'):
+    """
+    Attempt Tesseract OCR on a scanned page via PyMuPDF's built-in integration.
+    Returns extracted text or empty string if tesseract unavailable.
+    """
+    try:
+        tp = page.get_textpage_ocr(language=language, dpi=300)
+        return page.get_text(textpage=tp).strip()
+    except Exception as e:
+        msg = str(e).lower()
+        if 'tesseract' in msg or 'not installed' in msg or 'not found' in msg:
+            # Tesseract not installed — skip OCR, return what we have
+            sys.stderr.write(f'[pdf-text-extract] Tesseract not available, skipping OCR for scanned page\n')
+            return ''
+        # Other error — still don't crash
+        sys.stderr.write(f'[pdf-text-extract] OCR failed: {e}\n')
+        return ''
+
+
+def extract_pdf(pdf_path, max_pages=100, language='eng'):
+    """
+    Extract text from PDF using PyMuPDF.
+    Digital pages: instant text extraction.
+    Scanned pages: Tesseract OCR fallback.
+    """
+    pymupdf = _ensure_pymupdf()
+
+    if not os.path.isfile(pdf_path):
+        return {'error': f'File not found: {pdf_path}'}
+
+    try:
+        doc = pymupdf.open(pdf_path)
+    except Exception as e:
+        msg = str(e).lower()
+        if 'password' in msg or 'encrypt' in msg:
+            return {'error': f'PDF is password-protected: {pdf_path}'}
+        return {'error': f'Failed to open PDF: {e}'}
+
+    total_pages = len(doc)
+    process_count = min(total_pages, max_pages)
+    truncated = total_pages > max_pages
+
+    texts = []
+    scanned_pages = []
+
+    for i in range(process_count):
+        page = doc[i]
+        text = page.get_text().strip()
+
+        if _is_scanned_page(page, text):
+            # Try OCR
+            ocr_text = _ocr_page(page, language)
+            if ocr_text:
+                text = ocr_text
+                scanned_pages.append(i + 1)  # 1-indexed
+            # If OCR also empty, keep whatever minimal text we got
+
+        if text:
+            if process_count > 1:
+                texts.append(f'--- Page {i + 1} ---\n{text}')
+            else:
+                texts.append(text)
+
+    doc.close()
+
+    full_text = '\n\n'.join(texts)
+
+    result = {
+        'text': full_text,
+        'pages': process_count,
+        'chars': len(full_text),
+    }
+
+    if scanned_pages:
+        result['scanned_pages'] = scanned_pages
+    if truncated:
+        result['truncated'] = True
+        result['total_pages'] = total_pages
+
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Extract text from PDF files')
+    parser.add_argument('pdf_path', nargs='?', help='Path to the PDF file (single mode)')
+    parser.add_argument('--batch', nargs='+', metavar='PDF',
+                        help='Batch mode: extract multiple PDFs (JSONL output, one line per PDF)')
+    parser.add_argument('--max-pages', type=int, default=100,
+                        help='Maximum pages to process per PDF (default: 100)')
+    parser.add_argument('--language', default='eng',
+                        help='Tesseract language for OCR fallback (default: eng)')
+
+    args = parser.parse_args()
+
+    if args.batch:
+        # Batch mode — JSONL output, one result per line
+        # Single Python startup for N PDFs (avoids repeated interpreter overhead)
+        for pdf_path in args.batch:
+            result = extract_pdf(pdf_path, args.max_pages, args.language)
+            result['path'] = pdf_path
+            print(json.dumps(result, ensure_ascii=False), flush=True)
+    elif args.pdf_path:
+        # Single file mode
+        result = extract_pdf(args.pdf_path, args.max_pages, args.language)
+        result['path'] = args.pdf_path
+        print(json.dumps(result, ensure_ascii=False))
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "specmem-hardwicksoftware",
-  "version": "3.7.34",
+  "version": "3.7.36",
   "type": "module",
   "description": "Your Claude Code sessions don't have to start from scratch anymore — SpecMem gives your AI real memory. It won't forget your conversations, your code, or your architecture decisions between sessions. That's the whole point. Semantic code indexing that actually works: TypeScript, JavaScript, Python, Go, Rust, Java, Kotlin, C, C++, HTML and more. It doesn't just track functions — it gets classes, methods, fields, constants, enums, macros, imports, structs, the whole codebase graph. There's chat memory too, powered by pgvector embeddings. You've also got token compression, team coordination, multi-agent comms, and file watching built in. 74+ MCP tools. Runs on PostgreSQL + Docker. It's kind of a big deal. justcalljon.pro",
   "main": "dist/index.js",

package/scripts/deploy-hooks.cjs

@@ -172,7 +172,7 @@ function getHookConfig() {
     ],
     "PreToolUse": [
       {
-        "matcher": "Task",
+        "matcher": "Agent",
         "hooks": [
           {
             "type": "command",
@@ -421,7 +421,7 @@ function getHookConfig() {
     ],
     "PostToolUse": [
       {
-        "matcher": "Task",
+        "matcher": "Agent",
         "hooks": [
           {
             "type": "command",

package/scripts/global-postinstall.cjs

@@ -1756,7 +1756,7 @@ function configureSettings() {
   // PreToolUse hooks - Agent loading with chooser - MERGE
   const specmemPreToolUseHooks = [
     {
-      matcher: 'Task',
+      matcher: 'Agent',
       hooks: [{
         type: 'command',
         command: `node ${path.join(CLAUDE_HOOKS_DIR, 'agent-loading-hook.js')}`,
@@ -1843,7 +1843,7 @@ function configureSettings() {
   // PostToolUse hooks - agent completion tracking - MERGE
   const specmemPostToolUseHooks = [
     {
-      matcher: 'Task',
+      matcher: 'Agent',
       hooks: [{
         type: 'command',
         command: `node ${path.join(CLAUDE_HOOKS_DIR, 'task-progress-hook.js')}`,