specmem-hardwicksoftware 3.7.29 → 3.7.31

package/scripts/specmem-init.cjs

@@ -3202,11 +3202,11 @@ const TIER_CONFIG = {
 const MAX_EMBED_CHARS = 8000;
 
 // ============================================================================
-// STAGE 1: PROJECT ANALYSIS
+// STAGE 2: PROJECT ANALYSIS
 // ============================================================================
 
 async function analyzeProject(projectPath, ui) {
-  ui.setStage(1, 'PROJECT ANALYSIS');
+  ui.setStage(2, 'PROJECT ANALYSIS');
 
   const results = {
     tier: 'small',
@@ -3386,11 +3386,11 @@ async function analyzeProject(projectPath, ui) {
 }
 
 // ============================================================================
-// STAGE 2: SCORCHED EARTH - Wipe everything and rebuild fresh
+// STAGE 3: SCORCHED EARTH - Wipe everything and rebuild fresh
 // ============================================================================
 
 async function scorchedEarth(projectPath, ui) {
-  ui.setStage(2, 'CLEANUP');
+  ui.setStage(3, 'CLEANUP');
 
   const specmemDir = path.join(projectPath, 'specmem');
   const projectDir = path.join(projectPath, '.claude');
@@ -3539,11 +3539,11 @@ async function scorchedEarth(projectPath, ui) {
 }
 
 // ============================================================================
-// STAGE 5: MODEL OPTIMIZATION
+// STAGE 4: MODEL OPTIMIZATION
 // ============================================================================
 
 async function optimizeModel(projectPath, analysis, ui) {
-  ui.setStage(3, 'BLAST OFF 🚀');
+  ui.setStage(4, 'BLAST OFF 🚀');
 
   const recommended = TIER_CONFIG[analysis.tier];
 
@@ -3750,7 +3750,7 @@ async function optimizeModel(projectPath, analysis, ui) {
  * This ensures the overflow queue is pre-populated before  launches.
  */
 async function coldStartEmbeddingDocker(projectPath, modelConfig, ui, codebaseResult) {
-  ui.setStage(4, 'EMBEDDING DOCKER');
+  ui.setStage(5, 'EMBEDDING DOCKER');
 
   // ═══════════════════════════════════════════════════════════════════════════
   // 🔒 PRE-FLIGHT ACK CHECK - Verify optimizations before starting embedding 🔒
@@ -4129,7 +4129,7 @@ async function waitForEmbeddingReady(sockPath, opts = {}) {
 }
 
 async function indexCodebase(projectPath, ui, embeddingResult) {
-  ui.setStage(5, 'CODEBASE INDEXING');
+  ui.setStage(6, 'CODEBASE INDEXING');
 
   const { Pool } = require('pg');
   const crypto = require('crypto');
@@ -5982,11 +5982,11 @@ async function extractDefinitions(content, filePath, language, fileId) {
 }
 
 // ============================================================================
-// STAGE 6: TOKEN COMPRESSION
+// STAGE 7: TOKEN COMPRESSION
 // ============================================================================
 
 async function compressTokens(projectPath, ui) {
-  ui.setStage(7, 'TOKEN COMPRESSION');
+  ui.setStage(8, 'TOKEN COMPRESSION');
 
   // Import compressor inline
   let compress;
@@ -6085,11 +6085,11 @@ async function compressTokens(projectPath, ui) {
 
 
 // ============================================================================
-// STAGE 7: COMMAND DEPLOYMENT
+// STAGE 8: COMMAND DEPLOYMENT
 // ============================================================================
 
 async function deployCommands(projectPath, ui) {
-  ui.setStage(8, 'COMMAND DEPLOYMENT');
+  ui.setStage(9, 'COMMAND DEPLOYMENT');
 
   const globalCmdsDir = path.join(os.homedir(), '.claude', 'commands');
   const projectCmdsDir = path.join(projectPath, '.claude', 'commands');
@@ -6167,7 +6167,7 @@ async function deployCommands(projectPath, ui) {
  * so they're searchable via find_memory from the start.
  */
 async function extractSessions(projectPath, ui, embeddingResult = null) {
-  ui.setStage(6, 'SESSION EXTRACTION');
+  ui.setStage(7, 'SESSION EXTRACTION');
 
   const claudeDir = path.join(os.homedir(), '.claude');
   const projectsDir = path.join(claudeDir, 'projects');
@@ -6812,11 +6812,11 @@ async function extractSessions(projectPath, ui, embeddingResult = null) {
 }
 
 // ============================================================================
-// STAGE 9: FINAL VERIFICATION
+// STAGE 10: FINAL VERIFICATION
 // ============================================================================
 
 async function finalVerification(projectPath, analysis, modelConfig, ui) {
-  ui.setStage(9, 'FINAL VERIFICATION');
+  ui.setStage(10, 'FINAL VERIFICATION');
 
   const checks = {
     modelConfig: false,
@@ -8799,9 +8799,14 @@ CREATE INDEX IF NOT EXISTS idx_embedding_queue_project ON embedding_queue (proje
         ? path.join(specmemPkg, 'mcp-proxy.cjs')
         : path.join(specmemPkg, 'bootstrap.cjs');
       // Container mode: postgres via unix socket in specmem/run/, user=specmem, trust auth
+      // Socket dir bind-mounted to /data/run in container — PG socket appears here after container starts
       // Legacy mode: postgres on localhost:5432, legacy credentials
       const isContainerMode = process.env.SPECMEM_CONTAINER_MODE === 'true';
       const runDir = path.join(projectPath, 'specmem', 'run');
+      if (isContainerMode) {
+        // Ensure socket directory exists on host — container bind-mounts dataDir:/data
+        try { fs.mkdirSync(runDir, { recursive: true }); } catch (e) { /* may already exist */ }
+      }
       const dbEnv = isContainerMode ? {
         SPECMEM_DB_HOST: runDir,
         SPECMEM_DB_PORT: "5432",
@@ -8955,32 +8960,44 @@ function registerProjectInRegistry(projectPath, globalDir) {
 }
 
 // ============================================================================
-// MODEL DOWNLOAD — ensures ML models exist before init proceeds
+// MODEL DOWNLOAD — Stage 1: ensures all ML models exist before init proceeds
 // ============================================================================
 
 /**
- * Checks if ML models are present in the install directory.
- * If missing (npm strips them), downloads from GitHub release.
- * Nohup mode: auto-accepts without prompting.
+ * Stage 1: MODEL DOWNLOAD
+ * Checks for all required ML models. If any are missing, auto-downloads
+ * the combined models tarball from GitHub release — no prompting.
+ *
+ * Models bundled in npm (always present after install):
+ *   - all-MiniLM-L6-v2 (embedding)
+ *   - minisbd (sentence boundary)
+ *
+ * Models NOT in npm (LFS-only, downloaded via release tarball):
+ *   - pythia-410m-onnx-quant (mini-COT)
+ *   - argos-translate (translation en↔zh/zt)
  */
-async function ensureModels() {
+async function ensureModels(ui) {
   const specmemRoot = path.resolve(__dirname, '..');
-  const sentinelFile = path.join(specmemRoot, 'embedding-sandbox', 'models', 'all-MiniLM-L6-v2', 'onnx', 'model_quint8_avx2.onnx');
+  const modelsDir = path.join(specmemRoot, 'embedding-sandbox', 'models');
+  const version = SPECMEM_VERSION;
 
-  if (fs.existsSync(sentinelFile)) {
-    initLog('[MODELS] Models present — skipping download');
-    console.log(`  ${c.green}✓${c.reset} ${c.dim}ML models OK${c.reset}`);
+  const sentinels = [
+    { name: 'Mini-COT (Pythia)', path: path.join(modelsDir, 'pythia-410m-onnx-quant', 'model_quantized.onnx') },
+    { name: 'Translation (Argos)', path: path.join(modelsDir, 'argos-translate', 'translate-en_zh-1_9', 'model', 'model.bin') },
+  ];
+
+  const missing = sentinels.filter(s => !fs.existsSync(s.path));
+
+  if (missing.length === 0) {
+    initLog('[MODELS] All models present — skipping download');
+    if (ui) ui.setSubStatus('✓ All ML models present');
     return;
   }
 
-  initLog('[MODELS] Sentinel file missing: ' + sentinelFile);
+  initLog(`[MODELS] Missing: ${missing.map(s => s.name).join(', ')}`);
+  if (ui) ui.setSubStatus(`↓ Downloading ML models (~570MB)...`);
 
-  const nohup = process.argv.includes('--nohup') || process.env.SPECMEM_NOHUP === '1';
-  const modelsDir = path.join(specmemRoot, 'embedding-sandbox', 'models');
-  const version = SPECMEM_VERSION;
-  const releaseUrl = `https://github.com/jonhardwick-spec/specmem/releases/download/v${version}/specmem-models-${version}.tar.gz`;
-
-  // Check if install path is globally owned (needs sudo for extraction)
+  // Check write perms
   let needsSudo = false;
   try {
     fs.accessSync(path.join(specmemRoot, 'embedding-sandbox'), fs.constants.W_OK);
@@ -8988,102 +9005,62 @@ async function ensureModels() {
     needsSudo = true;
   }
 
-  if (!nohup) {
-    console.log('');
-    console.log(drawBox([
-      `${c.bold}${c.yellow}ML Models Required${c.reset}`,
-      '',
-      `SpecMem needs to download ML models (~500MB).`,
-      `These are stripped from npm to keep installs fast.`,
-      '',
-      `${c.dim}Source: GitHub release v${version}${c.reset}`,
-      needsSudo ? `${c.yellow}Note: Install path requires sudo for extraction${c.reset}` : '',
-    ].filter(Boolean), { borderColor: c.yellow }));
-
-    const readline = require('readline');
-    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
-
-    const answer = await new Promise((resolve) => {
-      rl.question(`  ${c.bold}Download models?${c.reset} [Y/n] `, (ans) => {
-        rl.close();
-        resolve(ans.trim().toLowerCase());
-      });
-    });
-
-    if (answer === 'n' || answer === 'no') {
-      console.log(`  ${c.yellow}⚠${c.reset}  Skipped model download — embedding features will not work.`);
-      initLog('[MODELS] User declined model download');
-      return;
-    }
-  } else {
-    console.log(`  ${c.dim}[nohup] Auto-downloading ML models (~500MB)...${c.reset}`);
-  }
-
-  // Download
-  const tmpTarball = path.join(os.tmpdir(), `specmem-models-${version}.tar.gz`);
-
-  // Prefer curl, fall back to wget
   const hasCurl = (() => { try { execSync('which curl', { stdio: 'pipe' }); return true; } catch { return false; } })();
   const hasWget = (() => { try { execSync('which wget', { stdio: 'pipe' }); return true; } catch { return false; } })();
 
   if (!hasCurl && !hasWget) {
-    console.log(`  ${c.red}✗${c.reset} Neither curl nor wget found — cannot download models.`);
-    console.log(`  ${c.dim}Install curl or wget and re-run specmem init.${c.reset}`);
     initLog('[MODELS] ERROR: no curl or wget');
+    if (ui) ui.setSubStatus('⚠ curl/wget not found — ML models unavailable (mini-COT/translation disabled)');
     return;
   }
 
-  console.log(`  ${c.cyan}↓${c.reset} Downloading models from GitHub release v${version}...`);
-  initLog('[MODELS] Downloading: ' + releaseUrl);
+  const releaseUrl = `https://github.com/jonhardwick-spec/specmem/releases/download/v${version}/specmem-models-${version}.tar.gz`;
+  const tmpTarball = path.join(os.tmpdir(), `specmem-models-${version}.tar.gz`);
+
+  initLog(`[MODELS] Downloading: ${releaseUrl}`);
 
   try {
     const dlCmd = hasCurl
       ? `curl -fSL --progress-bar -o "${tmpTarball}" "${releaseUrl}"`
       : `wget --progress=bar:force -O "${tmpTarball}" "${releaseUrl}"`;
-
-    execSync(dlCmd, { stdio: 'inherit', timeout: 600000 });
+    execSync(dlCmd, { stdio: 'pipe', timeout: 900000 }); // 15 min timeout for ~570MB
   } catch (e) {
-    console.log(`  ${c.red}✗${c.reset} Download failed: ${e.message}`);
-    initLog('[MODELS] Download failed: ' + e.message);
+    initLog(`[MODELS] Download failed: ${e.message}`);
+    if (ui) ui.setSubStatus(`⚠ Model download failed — mini-COT/translation disabled`);
     try { fs.unlinkSync(tmpTarball); } catch {}
     return;
   }
 
-  // Extract
-  console.log(`  ${c.cyan}⤳${c.reset} Extracting models...`);
-  initLog('[MODELS] Extracting to: ' + modelsDir);
+  if (ui) ui.setSubStatus('⤳ Extracting ML models...');
+  initLog(`[MODELS] Extracting to: ${modelsDir}`);
 
   try {
-    // Ensure models directory exists
     const mkdirCmd = `mkdir -p "${modelsDir}"`;
     const extractCmd = `tar xzf "${tmpTarball}" -C "${modelsDir}"`;
-
     if (needsSudo) {
       execSync(`sudo ${mkdirCmd}`, { stdio: 'pipe' });
-      execSync(`sudo ${extractCmd}`, { stdio: 'inherit', timeout: 120000 });
-      // Fix ownership so specmem can read
+      execSync(`sudo ${extractCmd}`, { stdio: 'pipe', timeout: 180000 });
       execSync(`sudo chmod -R a+rX "${modelsDir}"`, { stdio: 'pipe' });
     } else {
       execSync(mkdirCmd, { stdio: 'pipe' });
-      execSync(extractCmd, { stdio: 'inherit', timeout: 120000 });
+      execSync(extractCmd, { stdio: 'pipe', timeout: 180000 });
     }
   } catch (e) {
-    console.log(`  ${c.red}✗${c.reset} Extraction failed: ${e.message}`);
-    initLog('[MODELS] Extraction failed: ' + e.message);
+    initLog(`[MODELS] Extraction failed: ${e.message}`);
+    if (ui) ui.setSubStatus('⚠ Model extraction failed');
     try { fs.unlinkSync(tmpTarball); } catch {}
     return;
   }
 
-  // Cleanup tarball
   try { fs.unlinkSync(tmpTarball); } catch {}
 
-  // Verify
-  if (fs.existsSync(sentinelFile)) {
-    console.log(`  ${c.green}✓${c.reset} Models downloaded and extracted successfully`);
-    initLog('[MODELS] Success — sentinel verified');
+  const stillMissing = sentinels.filter(s => !fs.existsSync(s.path));
+  if (stillMissing.length === 0) {
+    initLog('[MODELS] All models OK after download');
+    if (ui) ui.setSubStatus('✓ ML models downloaded and ready');
   } else {
-    console.log(`  ${c.yellow}⚠${c.reset}  Extraction completed but sentinel file not found — models may be incomplete`);
-    initLog('[MODELS] WARNING: sentinel missing after extraction');
+    initLog(`[MODELS] Still missing after download: ${stillMissing.map(s => s.name).join(', ')}`);
+    if (ui) ui.setSubStatus(`⚠ Still missing: ${stillMissing.map(s => s.name).join(', ')}`);
   }
 }
 
@@ -9193,9 +9170,6 @@ async function main() {
     }
   }
 
-  // ========== ENSURE ML MODELS ==========
-  await ensureModels();
-
   // Animated banner with sliding red highlight (screen already cleared at startup)
   const _dbg = (m) => { try { fs.appendFileSync('/tmp/init-trace.log', `${Date.now()} ${m}\n`); } catch {} };
   _dbg('PRE-BANNER');
@@ -9858,16 +9832,22 @@ ${lastOutput}
 
     // Adjust total stages based on mode
     if (!launchScreens) {
-      ui.totalStages = 9; // No screen sessions stage
+      ui.totalStages = 10; // No screen sessions stage
     } else if (skipScorchedEarth) {
-      ui.totalStages = 2; // Quick mode: just analyze + screens
+      ui.totalStages = 3; // Quick mode: models + analyze + screens
     } else {
-      ui.totalStages = 10; // Full mode with screen sessions
+      ui.totalStages = 11; // Full mode with screen sessions
     }
 
     ui.start();
     _dbg('POST-UI-START');
 
+    // ==========================================================================
+    // STAGE 1: MODEL DOWNLOAD — auto-fetch any missing ML models
+    // ==========================================================================
+    ui.setStage(1, 'MODEL DOWNLOAD');
+    await ensureModels(ui);
+
     // ==========================================================================
     // CONTAINER FAST-PATH: If podman/docker is available, use container mode
     // This replaces stages 2-9 with: pull image → start container → deploy hooks → verify
@@ -9953,14 +9933,14 @@ ${lastOutput}
     }
 
     if (useContainerMode && !skipScorchedEarth) {
-      // Container mode: 5 stages (+ optional codebase indexing + screens)
-      ui.totalStages = launchScreens ? 7 : 6;
+      // Container mode: Stage 1 = models, then container stages
+      ui.totalStages = launchScreens ? 8 : 7;
 
-      // Stage 1: Analyze project
+      // Stage 2: Analyze project
       const analysis = await analyzeProject(projectPath, ui);
 
-      // Stage 2: Container engine — pull image if needed
-      ui.setStage(2, 'CONTAINER ENGINE');
+      // Stage 3: Container engine — pull image if needed
+      ui.setStage(3, 'CONTAINER ENGINE');
       ui.setStatus('Pulling specmem-brain image...');
       const containerImage = process.env.SPECMEM_CONTAINER_IMAGE || 'ghcr.io/hardwicksoftware/specmem-brain:latest';
       const rtCmd = (() => {
@@ -10012,8 +9992,8 @@ ${lastOutput}
         console.log('');
         process.exit(1);
       } else {
-        // Stage 3: Start container
-        ui.setStage(3, 'CONTAINER START');
+        // Stage 4: Start container
+        ui.setStage(4, 'CONTAINER START');
         ui.setStatus('Starting specmem-brain...');
         const dirName = path.basename(projectPath).replace(/[^a-zA-Z0-9_-]/g, '_');
         const containerName = `specmem-brain-${dirName}`;
@@ -10412,8 +10392,8 @@ priority=999
           process.env.SPECMEM_TRANSLATE_SOCKET = path.join(runDir, 'translate.sock');
           process.env.SPECMEM_MINICOT_SOCKET = path.join(runDir, 'minicot.sock');
 
-          // Stage 4: PostgreSQL client tools (psql) — hooks shell out to psql for DB queries
-          ui.setStage(4, 'PSQL CLIENT');
+          // Stage 5: PostgreSQL client tools (psql) — hooks shell out to psql for DB queries
+          ui.setStage(5, 'PSQL CLIENT');
           ui.setStatus('Checking psql...');
 
           let hasPsqlClient = false;
@@ -10512,8 +10492,8 @@ priority=999
             console.log(`  ${c.dim}Hooks that query the DB will fail without it.${c.reset}`);
           }
 
-          // Stage 5: Deploy hooks + MCP config + verify
-          ui.setStage(5, 'HOOKS & CONFIG');
+          // Stage 6: Deploy hooks + MCP config + verify
+          ui.setStage(6, 'HOOKS & CONFIG');
           ui.setStatus('Deploying Claude hooks...');
           syncHooksAndSettings();
           ui.setStatus('Hooks deployed');
@@ -10619,9 +10599,9 @@ priority=999
             }, null, 2));
           } catch {}
 
-          // Stage 6: CODEBASE INDEXING
-          _dbg('STAGE-6-CODEBASE-INDEXING');
-          ui.setStage(6, 'CODEBASE INDEXING');
+          // Stage 7: CODEBASE INDEXING
+          _dbg('STAGE-7-CODEBASE-INDEXING');
+          ui.setStage(7, 'CODEBASE INDEXING');
           ui.enableFileFeed(true);
           ui.setStatus('Waiting for container services...');
 

package/specmem/model-config.json

@@ -13,9 +13,10 @@
     "cpus": 8
   },
   "embedding": {
-    "batchSize": 24,
+    "batchSize": 64,
     "maxConcurrent": 5,
-    "timeout": 45000
+    "timeout": 45000,
+    "throttleDelayMs": 50
   },
   "watcher": {
     "debounceMs": 750,
@@ -33,13 +34,13 @@
     "maxChunks": 75
   },
   "resources": {
-    "cpuMin": 20,
-    "cpuMax": 40,
+    "cpuMin": 10,
+    "cpuMax": 45,
     "cpuCoreMin": 1,
     "cpuCoreMax": 4,
     "ramMinMb": 4000,
-    "ramMaxMb": 11500,
-    "updatedAt": "2026-02-21T22:33:46.533Z"
+    "ramMaxMb": 13500,
+    "updatedAt": "2026-02-24T11:56:33.760Z"
   },
   "resourcePool": {
     "embedding": {
@@ -79,5 +80,24 @@
       "description": "Adaptive batch sizing based on CPU/RAM"
     },
     "enabledAt": "2026-02-12T23:19:17.948Z"
+  },
+  "powerMode": {
+    "level": "high",
+    "description": "Max Performance - for 16GB+ RAM systems",
+    "lazyLoading": false,
+    "diskCache": false,
+    "diskCacheMaxMb": 0,
+    "aggressiveCleanup": false,
+    "idleUnloadSeconds": 0,
+    "batchSize": 32,
+    "throttleDelayMs": 50,
+    "setAt": "2026-02-24T11:57:23.121Z"
+  },
+  "heavyOps": {
+    "enabled": true,
+    "enabledAt": "2026-02-24T11:57:35.508Z",
+    "originalBatchSize": 32,
+    "batchSizeMultiplier": 2,
+    "throttleReduction": 0.2
   }
 }

package/specmem/supervisord.conf

@@ -1,6 +1,6 @@
 ; ============================================
 ; SPECMEM BRAIN CONTAINER - DYNAMIC SUPERVISORD CONFIG
-; Generated by specmem-init at 2026-02-22T16:24:03.901Z
+; Generated by specmem-init at 2026-02-24T11:53:24.816Z
 ; Thread counts from model-config.json resourcePool
 ; ============================================
 

package/specmem/user-config.json

@@ -7,5 +7,17 @@
   "serviceMode": {
     "enabled": false,
     "disabledAt": "2026-02-18T21:38:50.526Z"
+  },
+  "powerMode": {
+    "level": "high",
+    "description": "Max Performance - for 16GB+ RAM systems",
+    "lazyLoading": false,
+    "diskCache": false,
+    "diskCacheMaxMb": 0,
+    "aggressiveCleanup": false,
+    "idleUnloadSeconds": 0,
+    "batchSize": 32,
+    "throttleDelayMs": 50,
+    "setAt": "2026-02-24T11:57:23.121Z"
   }
 }