seo-intel 1.1.6 → 1.1.7

package/cli.js

@@ -24,11 +24,11 @@ import { getNextCrawlTarget, needsAnalysis, getCrawlStatus, loadAllConfigs } fro
 import {
   getDb, upsertDomain, upsertPage, insertExtraction,
   insertKeywords, insertHeadings, insertLinks, insertPageSchemas,
-  upsertTechnical,
+  upsertTechnical, pruneStaleDomains,
   getCompetitorSummary, getKeywordMatrix, getHeadingStructure,
   getPageHash, getSchemasByProject
 } from './db/db.js';
-import { generateHtmlDashboard, generateMultiDashboard } from './reports/generate-html.js';
+import { generateMultiDashboard } from './reports/generate-html.js';
 import { buildTechnicalActions } from './exports/technical.js';
 import { buildCompetitiveActions } from './exports/competitive.js';
 import { buildSuggestiveActions } from './exports/suggestive.js';
@@ -393,6 +393,21 @@ program
       }
     }
 
+    // ── Prune stale domains (DB entries no longer in config) ─────────────
+    {
+      const configDomains = new Set([
+        config.target?.domain,
+        ...(config.owned || []).map(o => o.domain),
+        ...(config.competitors || []).map(c => c.domain),
+      ].filter(Boolean));
+
+      const pruned = pruneStaleDomains(db, project, configDomains);
+      if (pruned.length) {
+        console.log(chalk.yellow(`\n  🧹 Pruned ${pruned.length} stale domain(s) from DB (no longer in config):`));
+        for (const d of pruned) console.log(chalk.dim(`     − ${d}`));
+      }
+    }
+
     // ── Tier gate: Free tier = crawl-only, no AI extraction ──────────────
     if (opts.extract !== false && !isPro()) {
       console.log(chalk.dim('\n  ℹ  Free tier: crawl-only mode (AI extraction requires Solo/Agency)'));
@@ -488,6 +503,9 @@ program
           publishedDate: page.publishedDate || null,
           modifiedDate: page.modifiedDate || null,
           contentHash: page.contentHash || null,
+          title: page.title || null,
+          metaDesc: page.metaDesc || null,
+          bodyText: page.fullBodyText || page.bodyText || null,
         });
         const pageId = pageRes?.id;
 
@@ -579,9 +597,10 @@ program
     if (totalSkipped > 0) console.log(chalk.blue(`\n📊 Incremental: ${totalSkipped} unchanged pages skipped (same content hash)`));
     if (totalBlocked > 0) console.log(chalk.red(`\n⛔ ${totalBlocked} domain(s) blocked (rate-limited or WAF)`));
     const elapsed = ((Date.now() - crawlStart) / 1000).toFixed(1);
-    // Auto-regenerate dashboard so it never goes stale after a crawl
+    // Auto-regenerate dashboard (always multi-project so all projects stay current)
     try {
-      const dashPath = generateHtmlDashboard(db, project, config);
+      const allConfigs = loadAllConfigs();
+      const dashPath = generateMultiDashboard(db, allConfigs);
       console.log(chalk.dim(`  📊 Dashboard refreshed → ${dashPath}`));
     } catch (dashErr) {
       console.log(chalk.dim(`  ⚠  Dashboard refresh skipped: ${dashErr.message}`));
@@ -697,9 +716,10 @@ program
     // Print summary
     printAnalysisSummary(analysis, project);
 
-    // Auto-regenerate dashboard so it reflects the new analysis immediately
+    // Auto-regenerate dashboard (always multi-project so all projects stay current)
     try {
-      const dashPath = generateHtmlDashboard(db, project, config);
+      const allConfigs = loadAllConfigs();
+      const dashPath = generateMultiDashboard(db, allConfigs);
       console.log(chalk.dim(`  📊 Dashboard refreshed → ${dashPath}`));
     } catch (dashErr) {
       console.log(chalk.dim(`  ⚠  Dashboard refresh skipped: ${dashErr.message}`));
@@ -1074,6 +1094,9 @@ program
         publishedDate: page.publishedDate || null,
         modifiedDate: page.modifiedDate || null,
         contentHash: page.contentHash || null,
+        title: page.title || null,
+        metaDesc: page.metaDesc || null,
+        bodyText: page.fullBodyText || page.bodyText || null,
       });
       const pageId = pageRes?.id;
 
@@ -1379,6 +1402,7 @@ program
   .option('--add-owned <domain>', 'Add an owned subdomain')
   .option('--remove-owned <domain>', 'Remove an owned subdomain')
   .option('--set-target <domain>', 'Change the target domain')
+  .option('--prune', 'Remove DB data for domains no longer in config')
   .action((project, opts) => {
     const configPath = join(__dirname, `config/${project}.json`);
     let config;
@@ -1471,6 +1495,24 @@ program
       console.log(chalk.dim(`\n  Saved → config/${project}.json`));
     }
 
+    // ── Prune stale DB data (auto on remove, or manual --prune) ─────────
+    if (modified || opts.prune) {
+      const db = getDb();
+      const configDomains = new Set([
+        config.target?.domain,
+        ...(config.owned || []).map(o => o.domain),
+        ...(config.competitors || []).map(c => c.domain),
+      ].filter(Boolean));
+
+      const pruned = pruneStaleDomains(db, project, configDomains);
+      if (pruned.length) {
+        console.log(chalk.yellow(`\n  🧹 Pruned ${pruned.length} stale domain(s) from DB:`));
+        for (const d of pruned) console.log(chalk.dim(`     − ${d}`));
+      } else if (opts.prune) {
+        console.log(chalk.dim('\n  ✓ No stale domains to prune'));
+      }
+    }
+
     // ── Always show current config
     console.log(chalk.bold.cyan(`\n  📋 ${project} — Domain Configuration\n`));
     console.log(chalk.white('  Target:'));
@@ -1560,13 +1602,14 @@ async function runAnalysis(project, db) {
 program
   .command('extract <project>')
   .description('Run AI extraction on all crawled-but-not-yet-extracted pages (requires Solo/Agency)')
-  .option('--stealth', 'Advanced browser mode for JS-heavy and dynamic sites')
-  .action(async (project, opts) => {
+  .action(async (project) => {
     if (!requirePro('extract')) return;
     const db = getDb();
+
+    // Query pages that have body_text stored (from crawl) but no extraction yet
     const pendingPages = db.prepare(`
-      SELECT p.id, p.url, p.word_count,
-             e.id as extracted
+      SELECT p.id, p.url, p.word_count, p.title, p.meta_desc, p.body_text,
+             p.published_date, p.modified_date
       FROM pages p
       JOIN domains d ON d.id = p.domain_id
       LEFT JOIN extractions e ON e.page_id = p.id
@@ -1578,102 +1621,241 @@ program
       process.exit(0);
     }
 
-    const mode = opts.stealth ? chalk.magenta('STEALTH') : chalk.gray('standard');
-    console.log(chalk.bold.cyan(`\n⚙️  Extracting ${pendingPages.length} pages for ${project} via Qwen [${mode}]...\n`));
+    // Check how many have body_text stored vs need re-crawl
+    const withContent = pendingPages.filter(r => r.body_text);
+    const needsRecrawl = pendingPages.length - withContent.length;
+
+    console.log(chalk.bold.cyan(`\n⚙️  Extracting ${pendingPages.length} pages for ${project} via Qwen...\n`));
+    if (needsRecrawl > 0) {
+      console.log(chalk.yellow(`  ⚠  ${needsRecrawl} pages have no stored content (crawled before v1.1.6). Re-crawl to populate.\n`));
+    }
 
     const extractStart = Date.now();
-    let done = 0, failed = 0;
+    let done = 0, failed = 0, skipped = 0;
+
+    // ── Pre-extract template grouping: sample N per group, skip the rest ──
+    const SAMPLE_PER_GROUP = 5;
+    const MIN_GROUP_FOR_SAMPLING = 10;
+    let extractQueue = pendingPages.filter(r => r.body_text); // only pages with stored content
+
+    try {
+      const { clusterUrls } = await import('./analyses/templates/cluster.js');
+      const { groups } = clusterUrls(
+        extractQueue.map(r => ({ url: r.url })),
+        { minGroupSize: MIN_GROUP_FOR_SAMPLING }
+      );
+
+      if (groups.length > 0) {
+        const skipUrls = new Set();
 
-    // ── Stealth: single session across all pages (cookie accumulation) ──
-    let stealthSession = null;
-    if (opts.stealth) {
-      const { createStealthSession } = await import('./crawler/stealth.js');
-      stealthSession = await createStealthSession();
-      console.log(chalk.magenta('  🥷 Advanced mode — full browser rendering, persistent sessions\n'));
+        for (const group of groups) {
+          const urls = group.urls;
+          if (urls.length <= SAMPLE_PER_GROUP) continue;
+
+          const sampleSet = new Set();
+          sampleSet.add(urls[0]); sampleSet.add(urls[1]);
+          sampleSet.add(urls[urls.length - 1]); sampleSet.add(urls[urls.length - 2]);
+          sampleSet.add(urls[Math.floor(urls.length / 2)]);
+
+          const skippedCount = urls.length - sampleSet.size;
+          for (const u of urls) {
+            if (!sampleSet.has(u)) skipUrls.add(u);
+          }
+          console.log(chalk.yellow(`  [template] ${group.pattern} → ${urls.length} pages, sampling ${sampleSet.size}, skipping ${skippedCount}`));
+        }
+
+        if (skipUrls.size > 0) {
+          extractQueue = extractQueue.filter(r => !skipUrls.has(r.url));
+          skipped += skipUrls.size;
+          console.log(chalk.yellow(`  [template] ${withContent.length} extractable → ${extractQueue.length} to extract (${skipUrls.size} template-skipped)\n`));
+        }
+      }
+    } catch (e) {
+      console.log(chalk.gray(`  [template] Pattern detection skipped: ${e.message}`));
     }
 
-    // Register cleanup so SIGTERM closes the browser gracefully
-    onShutdown(async () => {
-      if (stealthSession) {
-        await stealthSession.close();
-        console.log(chalk.magenta('  🥷 Stealth session closed'));
+    // ── Consecutive failure tracking per URL pattern ──
+    const CONSEC_FAIL_THRESHOLD = 3;
+    const patternFailCounts = new Map();
+    const skippedPatterns = new Set();
+
+    function getPatternKey(url) {
+      try {
+        const u = new URL(url);
+        const parts = u.pathname.split('/').filter(Boolean);
+        return u.hostname + '/' + parts.map(p =>
+          (p.length > 20 || /^[0-9a-fA-F]{8,}$/.test(p) || /^0x/.test(p) || /[-_]/.test(p)) ? '{var}' : p
+        ).join('/');
+      } catch { return url; }
+    }
+
+    // ── Content similarity detection ──
+    const SIMILARITY_THRESHOLD = 0.80;
+    const SIMILARITY_SAMPLE_SIZE = 3;
+    const patternFingerprints = new Map();
+
+    function textToShingles(text, n = 3) {
+      const words = (text || '').toLowerCase().replace(/[^a-z0-9\s]/g, '').split(/\s+/).filter(Boolean);
+      const shingles = new Set();
+      for (let i = 0; i <= words.length - n; i++) {
+        shingles.add(words.slice(i, i + n).join(' '));
+      }
+      return shingles;
+    }
+
+    function jaccardSimilarity(a, b) {
+      if (!a.size || !b.size) return 0;
+      let intersection = 0;
+      for (const s of a) { if (b.has(s)) intersection++; }
+      return intersection / (a.size + b.size - intersection);
+    }
+
+    function checkPatternSimilarity(patKey, newShingles) {
+      if (!patternFingerprints.has(patKey)) patternFingerprints.set(patKey, []);
+      const fps = patternFingerprints.get(patKey);
+      fps.push(newShingles);
+      if (fps.length < SIMILARITY_SAMPLE_SIZE || fps.length > SIMILARITY_SAMPLE_SIZE) return false;
+      for (let i = 0; i < fps.length; i++) {
+        for (let j = i + 1; j < fps.length; j++) {
+          if (jaccardSimilarity(fps[i], fps[j]) < SIMILARITY_THRESHOLD) return false;
+        }
       }
-    });
+      return true;
+    }
 
-    try {
-      for (const row of pendingPages) {
-        process.stdout.write(chalk.gray(`  [${done + failed + 1}/${pendingPages.length}] ${row.url.slice(0, 65)} → `));
-        if (opts.stealth) process.stdout.write(chalk.magenta('stealth '));
-        process.stdout.write(chalk.gray('fetching...'));
-
-        writeProgress({
-          status: 'running', command: 'extract', project,
-          current_url: row.url,
-          page_index: done + failed + 1, total: pendingPages.length,
-          percent: Math.round(((done + failed) / pendingPages.length) * 100),
-          started_at: extractStart, failed,
-          stealth: !!opts.stealth,
-        });
+    // ── Prepare headings + schema queries (per-page lookups from DB) ──
+    const getHeadings = db.prepare('SELECT level, text FROM headings WHERE page_id = ? ORDER BY id');
+    const getSchemaTypes = db.prepare('SELECT DISTINCT schema_type FROM page_schemas WHERE page_id = ?');
 
-        try {
-          let pageData;
+    const totalToProcess = extractQueue.length;
+    console.log(chalk.gray(`  📖 Reading from DB — no network needed\n`));
 
-          if (stealthSession) {
-            // Stealth: reuse persistent browser session
-            pageData = await stealthSession.fetchPage(row.url);
-          } else {
-            // Standard: quick single-page crawl
-            const { crawlAll } = await import('./crawler/index.js');
-            const crawled = await crawlAll(row.url);
-            pageData = crawled[0] || null;
-          }
+    for (const row of extractQueue) {
+      const patKey = getPatternKey(row.url);
+      if (skippedPatterns.has(patKey)) {
+        skipped++;
+        continue;
+      }
 
-          if (!pageData || pageData.status >= 400) {
-            const reason = pageData ? `HTTP ${pageData.status}` : 'no data';
-            process.stdout.write(chalk.red(` ✗ ${reason}\n`));
-            failed++;
-            if (stealthSession) {
-              // Jittered delay even on failure — don't hammer a blocking site
-              await new Promise(r => setTimeout(r, 1500 + Math.random() * 2000));
-            }
-            continue;
-          }
+      const pos = done + failed + 1;
+      process.stdout.write(chalk.gray(`  [${pos}/${totalToProcess}] ${row.url.slice(0, 70)} → `));
+      process.stdout.write(chalk.gray('extracting...'));
 
-          process.stdout.write(chalk.gray(' extracting...'));
-          const extractFn = await getExtractPage();
-          const extraction = await extractFn(pageData);
-          insertExtraction(db, { pageId: row.id, data: extraction });
-          insertKeywords(db, row.id, extraction.keywords);
-
-          // Also update headings + links + schemas with fresh data from stealth fetch
-          if (stealthSession) {
-            insertHeadings(db, row.id, pageData.headings);
-            insertLinks(db, row.id, pageData.links);
-            if (pageData.parsedSchemas?.length) insertPageSchemas(db, row.id, pageData.parsedSchemas);
-          }
+      writeProgress({
+        status: 'running', command: 'extract', project,
+        current_url: row.url,
+        page_index: pos, total: totalToProcess,
+        percent: Math.round(((done + failed) / totalToProcess) * 100),
+        started_at: extractStart, failed, skipped,
+      });
+
+      let pageFailed = false;
 
-          process.stdout.write(chalk.green(` ✓${pageData.parsedSchemas?.length ? ` [${pageData.parsedSchemas.length} schema]` : ''}\n`));
+      try {
+        // Read headings + schema types from DB
+        const headings = getHeadings.all(row.id);
+        const schemaTypes = getSchemaTypes.all(row.id).map(r => r.schema_type);
+
+        const extractFn = await getExtractPage();
+        const extraction = await extractFn({
+          url: row.url,
+          title: row.title || '',
+          metaDesc: row.meta_desc || '',
+          headings,
+          bodyText: row.body_text,
+          schemaTypes,
+          publishedDate: row.published_date,
+          modifiedDate: row.modified_date,
+        });
+        insertExtraction(db, { pageId: row.id, data: extraction });
+        insertKeywords(db, row.id, extraction.keywords);
+
+        const isDegraded = extraction.extraction_source === 'degraded';
+        if (isDegraded) {
+          process.stdout.write(chalk.yellow(` ⚠ degraded\n`));
           done++;
-        } catch (err) {
-          process.stdout.write(chalk.red(` ✗ ${err.message}\n`));
-          failed++;
+          pageFailed = true;
+        } else {
+          process.stdout.write(chalk.green(` ✓\n`));
+          done++;
+          patternFailCounts.set(patKey, 0);
         }
 
-        // Jittered delay in stealth mode (2-5s) to mimic human browsing
-        if (stealthSession) {
-          await new Promise(r => setTimeout(r, 2000 + Math.random() * 3000));
+        // ── Content similarity detection ──
+        if (row.body_text.length > 50) {
+          const shingles = textToShingles(row.body_text);
+          if (checkPatternSimilarity(patKey, shingles) && !skippedPatterns.has(patKey)) {
+            const remaining = extractQueue.filter(r => getPatternKey(r.url) === patKey).length - (patternFingerprints.get(patKey)?.length || 0);
+            skippedPatterns.add(patKey);
+            if (remaining > 0) {
+              console.log(chalk.yellow(`  [similarity] 🔍 ${SIMILARITY_SAMPLE_SIZE} pages from ${patKey} are ${Math.round(SIMILARITY_THRESHOLD * 100)}%+ identical — skipping ${remaining} remaining`));
+            }
+          }
         }
+      } catch (err) {
+        process.stdout.write(chalk.red(` ✗ ${err.message}\n`));
+        failed++;
+        pageFailed = true;
       }
-    } finally {
-      // Always close stealth session
-      if (stealthSession) {
-        await stealthSession.close();
-        console.log(chalk.magenta(`\n  🥷 Stealth session closed (${stealthSession.getPageCount()} pages fetched)`));
+
+      // ── Track consecutive failures per pattern ──
+      if (pageFailed) {
+        const count = (patternFailCounts.get(patKey) || 0) + 1;
+        patternFailCounts.set(patKey, count);
+        if (count >= CONSEC_FAIL_THRESHOLD) {
+          const remaining = extractQueue.filter(r => !skippedPatterns.has(getPatternKey(r.url)) && getPatternKey(r.url) === patKey).length;
+          skippedPatterns.add(patKey);
+          console.log(chalk.yellow(`  [template] ⚡ ${count} consecutive failures for ${patKey} — skipping ~${remaining} remaining pages`));
+        }
       }
     }
 
-    writeProgress({ status: 'completed', command: 'extract', project, extracted: done, failed, total: pendingPages.length, started_at: extractStart, finished_at: Date.now() });
-    console.log(chalk.bold.green(`\n✅ Extraction complete: ${done} extracted, ${failed} failed\n`));
+    writeProgress({ status: 'completed', command: 'extract', project, extracted: done, failed, skipped, total: pendingPages.length, started_at: extractStart, finished_at: Date.now() });
+    const skipMsg = skipped > 0 ? chalk.yellow(`, ${skipped} template-skipped`) : '';
+    const recrawlMsg = needsRecrawl > 0 ? chalk.yellow(`, ${needsRecrawl} need re-crawl`) : '';
+    console.log(chalk.bold.green(`\n✅ Extraction complete: ${done} extracted, ${failed} failed${skipMsg}${recrawlMsg}\n`));
+  });
+
+// ── TEMPLATES ANALYSIS ────────────────────────────────────────────────────
+program
+  .command('templates <project>')
+  .description('Detect programmatic template pages — assess SEO value without crawling all of them')
+  .option('--min-group <n>', 'Minimum URLs to qualify as a template group', '10')
+  .option('--sample-size <n>', 'Pages to stealth-crawl per template group', '20')
+  .option('--skip-crawl', 'Skip sample crawl (pattern analysis + GSC only)')
+  .option('--skip-gsc', 'Skip GSC overlay phase')
+  .option('--skip-competitors', 'Skip competitor sitemap census')
+  .action(async (project, opts) => {
+    if (!requirePro('templates')) return;
+
+    console.log(chalk.bold.cyan(`\n🔍 SEO Intel — Template Analysis`));
+    console.log(chalk.dim(`  Project: ${project}`));
+
+    try {
+      const { runTemplatesAnalysis } = await import('./analyses/templates/index.js');
+      const report = await runTemplatesAnalysis(project, {
+        minGroupSize: parseInt(opts.minGroup) || 10,
+        sampleSize: parseInt(opts.sampleSize) || 20,
+        skipCrawl: !!opts.skipCrawl,
+        skipGsc: !!opts.skipGsc,
+        skipCompetitors: !!opts.skipCompetitors,
+        log: (msg) => console.log(chalk.gray(msg)),
+      });
+
+      if (report.groups.length === 0) {
+        console.log(chalk.yellow(`\n  No template patterns detected.\n`));
+        process.exit(0);
+      }
+
+      // Summary
+      console.log(chalk.bold.green(`\n✅ Template analysis complete`));
+      console.log(chalk.dim(`  ${report.stats.totalGroups} groups · ${report.stats.totalGrouped.toLocaleString()} URLs · ${(report.stats.coverage * 100).toFixed(0)}% of sitemap`));
+      console.log(chalk.dim(`  Run ${chalk.white('seo-intel html ' + project)} to see the full dashboard.\n`));
+    } catch (err) {
+      console.error(chalk.red(`\n  Error: ${err.message}\n`));
+      if (process.env.DEBUG) console.error(err.stack);
+      process.exit(1);
+    }
   });
 
 // ── HTML DASHBOARD ─────────────────────────────────────────────────────────
@@ -1741,10 +1923,10 @@ program
     }
   });
 
-// ── HTML ALL-PROJECTS DASHBOARD ──────────────────────────────────────────────
+// ── HTML ALL-PROJECTS DASHBOARD (alias for html — kept for backwards compat) ──
 program
   .command('html-all')
-  .description('Generate a single HTML dashboard with all projects (dropdown switcher)')
+  .description('Alias for "html" — generates the all-projects dashboard')
   .action(() => {
     const db = getDb();
     const configs = loadAllConfigs();

package/crawler/index.js

@@ -547,11 +547,16 @@ async function processPage(page, url, base, depth, queue, maxDepth) {
   // ── Quality gate — detect shells, blocked pages, empty content ──
   const quality = assessQuality({ wordCount, bodyText, title, status });
 
+  // Full body text for DB storage (extraction reads this); truncated for log output
+  const fullBodyText = sanitize(bodyText, 50000); // ~200K chars — enough for any real page
+  const shortBodyText = sanitize(bodyText, 2000);  // compact version for logging
+
   return {
     url, depth, status, loadMs, wordCount, isIndexable,
     title, metaDesc, headings,
     links: [...internalLinks, ...externalLinks],
-    bodyText: sanitize(bodyText, 2000),
+    bodyText: shortBodyText,
+    fullBodyText,
     schemaTypes, parsedSchemas, vitals, publishedDate, modifiedDate,
     contentHash: hash,
     quality: quality.ok, qualityReason: quality.reason,

package/db/db.js

@@ -21,6 +21,9 @@ export function getDb(dbPath = './seo-intel.db') {
   // Migrations for existing databases
   try { _db.exec('ALTER TABLE pages ADD COLUMN content_hash TEXT'); } catch { /* already exists */ }
   try { _db.exec('ALTER TABLE pages ADD COLUMN first_seen_at INTEGER'); } catch { /* already exists */ }
+  try { _db.exec('ALTER TABLE pages ADD COLUMN title TEXT'); } catch { /* already exists */ }
+  try { _db.exec('ALTER TABLE pages ADD COLUMN meta_desc TEXT'); } catch { /* already exists */ }
+  try { _db.exec('ALTER TABLE pages ADD COLUMN body_text TEXT'); } catch { /* already exists */ }
 
   // Backfill first_seen_at from crawled_at for existing rows
   _db.exec('UPDATE pages SET first_seen_at = crawled_at WHERE first_seen_at IS NULL');
@@ -42,11 +45,11 @@ export function upsertDomain(db, { domain, project, role }) {
   `).run(domain, project, role, now, now);
 }
 
-export function upsertPage(db, { domainId, url, statusCode, wordCount, loadMs, isIndexable, clickDepth = 0, publishedDate = null, modifiedDate = null, contentHash = null }) {
+export function upsertPage(db, { domainId, url, statusCode, wordCount, loadMs, isIndexable, clickDepth = 0, publishedDate = null, modifiedDate = null, contentHash = null, title = null, metaDesc = null, bodyText = null }) {
   const now = Date.now();
   db.prepare(`
-    INSERT INTO pages (domain_id, url, crawled_at, first_seen_at, status_code, word_count, load_ms, is_indexable, click_depth, published_date, modified_date, content_hash)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    INSERT INTO pages (domain_id, url, crawled_at, first_seen_at, status_code, word_count, load_ms, is_indexable, click_depth, published_date, modified_date, content_hash, title, meta_desc, body_text)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     ON CONFLICT(url) DO UPDATE SET
       crawled_at     = excluded.crawled_at,
       status_code    = excluded.status_code,
@@ -55,8 +58,11 @@ export function upsertPage(db, { domainId, url, statusCode, wordCount, loadMs, i
       click_depth    = excluded.click_depth,
       published_date = excluded.published_date,
       modified_date  = excluded.modified_date,
-      content_hash   = excluded.content_hash
-  `).run(domainId, url, now, now, statusCode, wordCount, loadMs, isIndexable ? 1 : 0, clickDepth, publishedDate, modifiedDate, contentHash);
+      content_hash   = excluded.content_hash,
+      title          = excluded.title,
+      meta_desc      = excluded.meta_desc,
+      body_text      = excluded.body_text
+  `).run(domainId, url, now, now, statusCode, wordCount, loadMs, isIndexable ? 1 : 0, clickDepth, publishedDate, modifiedDate, contentHash, title || null, metaDesc || null, bodyText || null);
   // first_seen_at is NOT in the ON CONFLICT UPDATE — it stays from original INSERT
   return db.prepare('SELECT id FROM pages WHERE url = ?').get(url);
 }
@@ -214,6 +220,147 @@ export function getKeywordMatrix(db, project) {
   `).all(project);
 }
 
+// ── Template analysis ─────────────────────────────────────────────────────
+
+export function upsertTemplateGroup(db, g) {
+  return db.prepare(`
+    INSERT INTO template_groups
+      (project, domain, pattern, url_count, sample_size,
+       avg_word_count, content_similarity, dom_similarity,
+       gsc_urls_with_impressions, gsc_total_clicks, gsc_total_impressions,
+       gsc_avg_position, indexation_efficiency, score, verdict, recommendation,
+       analyzed_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(project, domain, pattern) DO UPDATE SET
+      url_count = excluded.url_count,
+      sample_size = excluded.sample_size,
+      avg_word_count = excluded.avg_word_count,
+      content_similarity = excluded.content_similarity,
+      dom_similarity = excluded.dom_similarity,
+      gsc_urls_with_impressions = excluded.gsc_urls_with_impressions,
+      gsc_total_clicks = excluded.gsc_total_clicks,
+      gsc_total_impressions = excluded.gsc_total_impressions,
+      gsc_avg_position = excluded.gsc_avg_position,
+      indexation_efficiency = excluded.indexation_efficiency,
+      score = excluded.score,
+      verdict = excluded.verdict,
+      recommendation = excluded.recommendation,
+      analyzed_at = excluded.analyzed_at
+  `).run(
+    g.project, g.domain, g.pattern, g.urlCount, g.sampleSize || 0,
+    g.avgWordCount ?? null, g.contentSimilarity ?? null, g.domSimilarity ?? null,
+    g.gscUrlsWithImpressions || 0, g.gscTotalClicks || 0, g.gscTotalImpressions || 0,
+    g.gscAvgPosition ?? null, g.indexationEfficiency ?? null,
+    g.score ?? null, g.verdict || null, JSON.stringify(g.recommendation || []),
+    g.analyzedAt || Date.now()
+  );
+}
+
+export function getTemplateGroupId(db, project, domain, pattern) {
+  return db.prepare(
+    'SELECT id FROM template_groups WHERE project = ? AND domain = ? AND pattern = ?'
+  ).get(project, domain, pattern)?.id;
+}
+
+export function upsertTemplateSample(db, s) {
+  db.prepare(`
+    INSERT INTO template_samples
+      (group_id, url, sample_role, status_code, word_count,
+       title, meta_desc, has_canonical, has_schema, is_indexable,
+       dom_fingerprint, content_hash, body_text, crawled_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(group_id, url) DO UPDATE SET
+      sample_role = excluded.sample_role,
+      status_code = excluded.status_code,
+      word_count = excluded.word_count,
+      title = excluded.title,
+      meta_desc = excluded.meta_desc,
+      has_canonical = excluded.has_canonical,
+      has_schema = excluded.has_schema,
+      is_indexable = excluded.is_indexable,
+      dom_fingerprint = excluded.dom_fingerprint,
+      content_hash = excluded.content_hash,
+      body_text = excluded.body_text,
+      crawled_at = excluded.crawled_at
+  `).run(
+    s.groupId, s.url, s.sampleRole, s.statusCode ?? null, s.wordCount ?? null,
+    s.title || null, s.metaDesc || null,
+    s.hasCanonical ? 1 : 0, s.hasSchema ? 1 : 0, s.isIndexable ? 1 : 0,
+    s.domFingerprint || null, s.contentHash || null, s.bodyText || null,
+    s.crawledAt || Date.now()
+  );
+}
+
+export function getTemplateGroups(db, project) {
+  return db.prepare(
+    'SELECT * FROM template_groups WHERE project = ? ORDER BY url_count DESC'
+  ).all(project);
+}
+
+export function getTemplateSamples(db, groupId) {
+  return db.prepare(
+    'SELECT * FROM template_samples WHERE group_id = ? ORDER BY sample_role, url'
+  ).all(groupId);
+}
+
+// ── Domain sync / prune ───────────────────────────────────────────────────
+
+/**
+ * Remove DB domains (+ all child data) that no longer exist in config.
+ * Returns array of pruned domain names.
+ */
+export function pruneStaleDomains(db, project, configDomains) {
+  // configDomains = Set or array of domain strings currently in config
+  const validSet = new Set(configDomains);
+
+  const dbDomains = db.prepare(
+    'SELECT id, domain FROM domains WHERE project = ?'
+  ).all(project);
+
+  const stale = dbDomains.filter(d => !validSet.has(d.domain));
+  if (!stale.length) return [];
+
+  db.exec('PRAGMA foreign_keys = OFF');
+  db.exec('BEGIN');
+  try {
+    for (const { id, domain } of stale) {
+      // Delete all child tables referencing pages in this domain
+      const pageIds = db.prepare(
+        'SELECT id FROM pages WHERE domain_id = ?'
+      ).all(id).map(r => r.id);
+
+      if (pageIds.length) {
+        const placeholders = pageIds.map(() => '?').join(',');
+        db.prepare(`DELETE FROM links WHERE source_id IN (${placeholders})`).run(...pageIds);
+        db.prepare(`DELETE FROM technical WHERE page_id IN (${placeholders})`).run(...pageIds);
+        db.prepare(`DELETE FROM headings WHERE page_id IN (${placeholders})`).run(...pageIds);
+        db.prepare(`DELETE FROM page_schemas WHERE page_id IN (${placeholders})`).run(...pageIds);
+        db.prepare(`DELETE FROM extractions WHERE page_id IN (${placeholders})`).run(...pageIds);
+        db.prepare(`DELETE FROM keywords WHERE page_id IN (${placeholders})`).run(...pageIds);
+        db.prepare(`DELETE FROM pages WHERE domain_id = ?`).run(id);
+      }
+
+      // Template groups for this domain
+      db.prepare(
+        'DELETE FROM template_samples WHERE group_id IN (SELECT id FROM template_groups WHERE project = ? AND domain = ?)'
+      ).run(project, domain);
+      db.prepare(
+        'DELETE FROM template_groups WHERE project = ? AND domain = ?'
+      ).run(project, domain);
+
+      db.prepare('DELETE FROM domains WHERE id = ?').run(id);
+    }
+    db.exec('COMMIT');
+  } catch (e) {
+    db.exec('ROLLBACK');
+    throw e;
+  } finally {
+    db.exec('PRAGMA foreign_keys = ON');
+  }
+
+  return stale.map(d => d.domain);
+}
+
 export function getHeadingStructure(db, project) {
   return db.prepare(`
     SELECT d.domain, d.role, h.level, h.text

package/db/schema.sql

@@ -23,6 +23,9 @@ CREATE TABLE IF NOT EXISTS pages (
   published_date TEXT,               -- ISO string or null
   modified_date  TEXT,               -- ISO string or null
   content_hash   TEXT,               -- SHA-256 of body text for incremental crawling
+  title          TEXT,               -- page <title>
+  meta_desc      TEXT,               -- meta description
+  body_text      TEXT,               -- cleaned body text for extraction (stored at crawl time)
   FOREIGN KEY (domain_id) REFERENCES domains(id)
 );
 
@@ -110,6 +113,51 @@ CREATE TABLE IF NOT EXISTS page_schemas (
   extracted_at INTEGER NOT NULL
 );
 
+-- Template analysis tables
+CREATE TABLE IF NOT EXISTS template_groups (
+  id                        INTEGER PRIMARY KEY AUTOINCREMENT,
+  project                   TEXT NOT NULL,
+  domain                    TEXT NOT NULL,
+  pattern                   TEXT NOT NULL,
+  url_count                 INTEGER NOT NULL,
+  sample_size               INTEGER NOT NULL DEFAULT 0,
+  avg_word_count            REAL,
+  content_similarity        REAL,
+  dom_similarity            REAL,
+  gsc_urls_with_impressions INTEGER DEFAULT 0,
+  gsc_total_clicks          INTEGER DEFAULT 0,
+  gsc_total_impressions     INTEGER DEFAULT 0,
+  gsc_avg_position          REAL,
+  indexation_efficiency     REAL,
+  score                     INTEGER,
+  verdict                   TEXT,
+  recommendation            TEXT,
+  analyzed_at               INTEGER NOT NULL,
+  UNIQUE(project, domain, pattern)
+);
+
+CREATE TABLE IF NOT EXISTS template_samples (
+  id              INTEGER PRIMARY KEY AUTOINCREMENT,
+  group_id        INTEGER NOT NULL REFERENCES template_groups(id) ON DELETE CASCADE,
+  url             TEXT NOT NULL,
+  sample_role     TEXT NOT NULL,
+  status_code     INTEGER,
+  word_count      INTEGER,
+  title           TEXT,
+  meta_desc       TEXT,
+  has_canonical   INTEGER DEFAULT 0,
+  has_schema      INTEGER DEFAULT 0,
+  is_indexable    INTEGER DEFAULT 1,
+  dom_fingerprint TEXT,
+  content_hash    TEXT,
+  body_text       TEXT,
+  crawled_at      INTEGER,
+  UNIQUE(group_id, url)
+);
+
+CREATE INDEX IF NOT EXISTS idx_template_groups_project ON template_groups(project);
+CREATE INDEX IF NOT EXISTS idx_template_samples_group ON template_samples(group_id);
+
 -- Indexes
 CREATE INDEX IF NOT EXISTS idx_pages_domain ON pages(domain_id);
 CREATE INDEX IF NOT EXISTS idx_keywords_page ON keywords(page_id);

package/lib/gate.js

@@ -52,6 +52,7 @@ const FEATURE_NAMES = {
   'entities':         'Entity Coverage Analysis',
   'friction':         'Friction Point Analysis',
   'js-delta':         'JS Rendering Delta',
+  'templates':        'Programmatic Template Intelligence',
   'html':             'HTML Dashboard',
   'html-all':         'HTML Dashboard (All Projects)',
   'gsc-insights':     'GSC Intelligence & Insights',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "seo-intel",
-  "version": "1.1.6",
+  "version": "1.1.7",
   "description": "Local Ahrefs-style SEO competitor intelligence. Crawl → SQLite → cloud analysis.",
   "type": "module",
   "license": "SEE LICENSE IN LICENSE",

package/reports/generate-html.js

@@ -525,8 +525,13 @@ function buildHtmlTemplate(data, opts = {}) {
       white-space: nowrap;
     }
     .es-btn:hover { border-color: var(--accent-gold); color: var(--accent-gold); }
-    .es-btn-stop { border-color: rgba(220,80,80,0.3); color: #dc5050; }
-    .es-btn-stop:hover { border-color: #dc5050; color: #ff6b6b; background: rgba(220,80,80,0.08); }
+    .es-btn-stop { border-color: var(--border-card); color: var(--text-muted); }
+    .es-btn-stop:hover { border-color: var(--text-secondary); color: var(--text-secondary); }
+    .es-btn-stop.active { border-color: rgba(220,80,80,0.5); color: #dc5050; animation: stopPulse 2s ease-in-out infinite; }
+    .es-btn-stop.active:hover { border-color: #dc5050; color: #ff6b6b; background: rgba(220,80,80,0.08); }
+    @keyframes stopPulse { 0%,100% { border-color: rgba(220,80,80,0.3); } 50% { border-color: rgba(220,80,80,0.7); } }
+    .es-btn-restart { border-color: rgba(100,160,220,0.3); color: #6ca0dc; }
+    .es-btn-restart:hover { border-color: #6ca0dc; color: #8fc0f0; background: rgba(100,160,220,0.08); }
     .es-btn:disabled {
       opacity: 0.4; cursor: not-allowed;
       border-color: var(--border-card);
@@ -1950,9 +1955,12 @@ function buildHtmlTemplate(data, opts = {}) {
             <i class="fa-solid fa-brain"></i> Extract
           </button>`
       }
-      <button class="es-btn es-btn-stop" id="btnStop${suffix}" onclick="stopJob()" style="display:${extractionStatus.liveProgress?.status === 'running' ? 'inline-flex' : 'none'};">
+      <button class="es-btn es-btn-stop${extractionStatus.liveProgress?.status === 'running' ? ' active' : ''}" id="btnStop${suffix}" onclick="stopJob()">
         <i class="fa-solid fa-stop"></i> Stop
       </button>
+      <button class="es-btn es-btn-restart" id="btnRestart${suffix}" onclick="restartServer()">
+        <i class="fa-solid fa-rotate-right"></i> Restart
+      </button>
       <label class="es-stealth-toggle">
         <input type="checkbox" id="stealthToggle${suffix}"${extractionStatus.liveProgress?.stealth ? ' checked' : ''}>
         <i class="fa-solid fa-user-ninja"></i> Stealth
@@ -1981,7 +1989,8 @@ function buildHtmlTemplate(data, opts = {}) {
           ${pro ? `<button class="term-btn" data-cmd="extract" data-project="${project}"><i class="fa-solid fa-brain"></i> Extract</button>
           <button class="term-btn" data-cmd="analyze" data-project="${project}"><i class="fa-solid fa-chart-column"></i> Analyze</button>
           <button class="term-btn" data-cmd="brief" data-project="${project}"><i class="fa-solid fa-file-lines"></i> Brief</button>
-          <button class="term-btn" data-cmd="keywords" data-project="${project}"><i class="fa-solid fa-key"></i> Keywords</button>` : ''}
+          <button class="term-btn" data-cmd="keywords" data-project="${project}"><i class="fa-solid fa-key"></i> Keywords</button>
+          <button class="term-btn" data-cmd="templates" data-project="${project}"><i class="fa-solid fa-clone"></i> Templates</button>` : ''}
           <button class="term-btn" data-cmd="status" data-project=""><i class="fa-solid fa-circle-info"></i> Status</button>
           <button class="term-btn" data-cmd="guide" data-project="${project}"><i class="fa-solid fa-map"></i> Guide</button>
           <button class="term-btn" data-cmd="setup" data-project="" style="margin-left:auto;border-color:rgba(232,213,163,0.25);"><i class="fa-solid fa-gear"></i> Setup</button>
@@ -2084,7 +2093,10 @@ function buildHtmlTemplate(data, opts = {}) {
       if (extra?.stealth) params.set('stealth', 'true');
       if (extra?.format) params.set('format', extra.format);
 
-      appendLine('$ seo-intel ' + command + (proj ? ' ' + proj : '') + (extra?.scope ? ' --scope ' + extra.scope : ''), 'cmd');
+      var stealthFlag = extra?.stealth ? ' --stealth' : '';
+      appendLine('$ seo-intel ' + command + (proj ? ' ' + proj : '') + stealthFlag + (extra?.scope ? ' --scope ' + extra.scope : ''), 'cmd');
+
+      var isCrawlOrExtract = (command === 'crawl' || command === 'extract');
 
       eventSource = new EventSource('/api/terminal?' + params.toString());
       eventSource.onmessage = function(e) {
@@ -2101,29 +2113,56 @@ function buildHtmlTemplate(data, opts = {}) {
             status.style.color = code === 0 ? 'var(--color-success)' : 'var(--color-danger)';
             eventSource.close();
             eventSource = null;
+            // Update status bar when crawl/extract finishes
+            if (isCrawlOrExtract && window._setButtonsState) window._setButtonsState(false, null);
           }
         } catch (_) {}
       };
       eventSource.onerror = function() {
         if (running) {
-          appendLine('Connection lost.', 'error');
+          // SSE disconnected but crawl/extract continues server-side
+          if (isCrawlOrExtract) {
+            appendLine('Terminal disconnected — job continues in background.', 'stderr');
+          } else {
+            appendLine('Connection lost.', 'error');
+          }
           running = false;
-          status.textContent = 'disconnected';
-          status.style.color = 'var(--color-danger)';
+          status.textContent = isCrawlOrExtract ? 'backgrounded' : 'disconnected';
+          status.style.color = isCrawlOrExtract ? 'var(--text-muted)' : 'var(--color-danger)';
         }
         eventSource?.close();
         eventSource = null;
       };
     }
 
-    // Button clicks
+    // Expose terminal for status bar buttons
+    window._terminalRun = function(cmd, proj, extra) { runCommand(cmd, proj, extra); };
+    window._terminalStop = function() {
+      if (eventSource) { eventSource.close(); eventSource = null; }
+      if (running) {
+        appendLine('Stopped.', 'exit-err');
+        running = false;
+        status.textContent = 'stopped';
+        status.style.color = 'var(--color-warning)';
+      }
+    };
+
+    // Button clicks — crawl/extract read stealth toggle
     document.querySelectorAll('.terminal-panel .term-btn').forEach(function(btn) {
       if (btn.closest('.terminal-panel') !== output.closest('.terminal-panel')) return;
       btn.addEventListener('click', function() {
         const cmd = btn.getAttribute('data-cmd');
         const proj = btn.getAttribute('data-project');
         const scope = btn.getAttribute('data-scope');
-        runCommand(cmd, proj, scope ? { scope } : undefined);
+        var extra = scope ? { scope: scope } : {};
+        // Crawl/extract: read stealth toggle + update status bar
+        if (cmd === 'crawl' || cmd === 'extract') {
+          var stealthEl = document.querySelector('[id^="stealthToggle"]');
+          if (stealthEl?.checked) extra.stealth = true;
+          if (window._setButtonsState) window._setButtonsState(true, cmd);
+          if (window._startPolling) window._startPolling();
+        }
+        runCommand(cmd, proj, extra);
       });
     });
 
@@ -3558,41 +3597,55 @@ function buildHtmlTemplate(data, opts = {}) {
       let pollTimer = null;
 
       window.startJob = function(command, proj) {
-        const stealth = document.getElementById('stealthToggle' + sfx)?.checked || false;
-        const body = { project: proj, stealth: stealth };
+        var stealth = document.getElementById('stealthToggle' + sfx)?.checked || false;
+        var extra = {};
+        if (stealth) extra.stealth = true;
 
-        fetch('/api/' + command, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify(body),
-        })
-        .then(function(r) { return r.json(); })
-        .then(function(data) {
-          if (data.error) { alert('Cannot start: ' + data.error); return; }
-          setButtonsState(true, command);
-          startPolling();
-        })
-        .catch(function(err) { alert('Server error: ' + err.message); });
+        // Route through terminal for visible output
+        if (window._terminalRun) {
+          window._terminalRun(command, proj, extra);
+        }
+        setButtonsState(true, command);
+        startPolling();
       };
 
       window.stopJob = function() {
+        // Close terminal SSE (server detaches crawl/extract, so we also hit /api/stop)
+        if (window._terminalStop) window._terminalStop();
         fetch('/api/stop', { method: 'POST' })
           .then(function(r) { return r.json(); })
-          .then(function(data) {
-            if (data.stopped) {
-              setButtonsState(false, null);
-            }
+          .then(function() { setButtonsState(false, null); })
+          .catch(function() { setButtonsState(false, null); });
+      };
+
+      window.restartServer = function() {
+        if (!confirm('Restart SEO Intel? This will stop any running jobs and refresh the dashboard.')) return;
+        var btnR = document.getElementById('btnRestart' + sfx);
+        if (btnR) { btnR.disabled = true; btnR.innerHTML = '<i class="fa-solid fa-spinner fa-spin"></i> Restarting\u2026'; }
+        // Stop terminal SSE
+        if (window._terminalStop) window._terminalStop();
+        fetch('/api/restart', { method: 'POST' })
+          .then(function() {
+            // Server is restarting — wait a moment then reload
+            setTimeout(function() { window.location.reload(); }, 2000);
           })
-          .catch(function(err) { alert('Stop failed: ' + err.message); });
+          .catch(function() {
+            // Server might already be dead — try reloading anyway
+            setTimeout(function() { window.location.reload(); }, 2000);
+          });
       };
 
-      function setButtonsState(disabled, activeCmd) {
+      // Expose for terminal IIFE to call back
+      window._setButtonsState = setButtonsState;
+      window._startPolling = startPolling;
+
+      function setButtonsState(isRunning, activeCmd) {
         var btnC = document.getElementById('btnCrawl' + sfx);
         var btnE = document.getElementById('btnExtract' + sfx);
         var btnS = document.getElementById('btnStop' + sfx);
         if (btnC) {
-          btnC.disabled = disabled;
-          if (disabled && activeCmd === 'crawl') {
+          btnC.disabled = isRunning;
+          if (isRunning && activeCmd === 'crawl') {
             btnC.classList.add('running');
             btnC.innerHTML = '<i class="fa-solid fa-spinner fa-spin"></i> Crawling\u2026';
           } else {
@@ -3601,8 +3654,8 @@ function buildHtmlTemplate(data, opts = {}) {
           }
         }
         if (btnE) {
-          btnE.disabled = disabled;
-          if (disabled && activeCmd === 'extract') {
+          btnE.disabled = isRunning;
+          if (isRunning && activeCmd === 'extract') {
             btnE.classList.add('running');
             btnE.innerHTML = '<i class="fa-solid fa-spinner fa-spin"></i> Extracting\u2026';
           } else {
@@ -3611,7 +3664,12 @@ function buildHtmlTemplate(data, opts = {}) {
           }
         }
         if (btnS) {
-          btnS.style.display = disabled ? 'inline-flex' : 'none';
+          // Stop button always visible — turns red+pulsing when something is running
+          if (isRunning) {
+            btnS.classList.add('active');
+          } else {
+            btnS.classList.remove('active');
+          }
         }
       }
 
@@ -4762,50 +4820,57 @@ function buildMultiHtmlTemplate(allProjectData) {
       window.startJob = function(command, proj) {
         var sfx = '-' + proj;
         var stealth = document.getElementById('stealthToggle' + sfx)?.checked || false;
-        fetch('/api/' + command, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ project: proj, stealth: stealth }),
-        })
-        .then(function(r) { return r.json(); })
-        .then(function(data) {
-          if (data.error) { alert('Cannot start: ' + data.error); return; }
-          setButtonsState(true, command);
-          startPolling();
-        })
-        .catch(function(err) { alert('Server error: ' + err.message); });
+        var extra = {};
+        if (stealth) extra.stealth = true;
+
+        // Route through terminal for visible output
+        if (window._terminalRun) {
+          window._terminalRun(command, proj, extra);
+        }
+        setButtonsState(true, command);
+        startPolling();
       };
 
       window.stopJob = function() {
+        if (window._terminalStop) window._terminalStop();
         fetch('/api/stop', { method: 'POST' })
           .then(function(r) { return r.json(); })
-          .then(function(data) {
-            if (data.stopped) setButtonsState(false, null);
-          })
-          .catch(function(err) { alert('Stop failed: ' + err.message); });
+          .then(function() { setButtonsState(false, null); })
+          .catch(function() { setButtonsState(false, null); });
+      };
+
+      window.restartServer = function() {
+        if (!confirm('Restart SEO Intel? This will stop any running jobs and refresh the dashboard.')) return;
+        if (window._terminalStop) window._terminalStop();
+        fetch('/api/restart', { method: 'POST' })
+          .then(function() { setTimeout(function() { window.location.reload(); }, 2000); })
+          .catch(function() { setTimeout(function() { window.location.reload(); }, 2000); });
       };
 
-      function setButtonsState(disabled, activeCmd) {
+      window._setButtonsState = setButtonsState;
+      window._startPolling = startPolling;
+
+      function setButtonsState(isRunning, activeCmd) {
         var sfx = '-' + currentProject;
         var btnC = document.getElementById('btnCrawl' + sfx);
         var btnE = document.getElementById('btnExtract' + sfx);
         var btnS = document.getElementById('btnStop' + sfx);
         if (btnC) {
-          btnC.disabled = disabled;
-          btnC.classList.toggle('running', disabled && activeCmd === 'crawl');
-          btnC.innerHTML = disabled && activeCmd === 'crawl'
+          btnC.disabled = isRunning;
+          btnC.classList.toggle('running', isRunning && activeCmd === 'crawl');
+          btnC.innerHTML = isRunning && activeCmd === 'crawl'
             ? '<i class="fa-solid fa-spinner fa-spin"></i> Crawling\u2026'
             : '<i class="fa-solid fa-spider"></i> Crawl';
         }
         if (btnE) {
-          btnE.disabled = disabled;
-          btnE.classList.toggle('running', disabled && activeCmd === 'extract');
-          btnE.innerHTML = disabled && activeCmd === 'extract'
+          btnE.disabled = isRunning;
+          btnE.classList.toggle('running', isRunning && activeCmd === 'extract');
+          btnE.innerHTML = isRunning && activeCmd === 'extract'
             ? '<i class="fa-solid fa-spinner fa-spin"></i> Extracting\u2026'
             : '<i class="fa-solid fa-brain"></i> Extract';
         }
         if (btnS) {
-          btnS.style.display = disabled ? 'inline-flex' : 'none';
+          if (isRunning) { btnS.classList.add('active'); } else { btnS.classList.remove('active'); }
         }
       }
 

package/server.js

@@ -297,7 +297,7 @@ async function handleRequest(req, res) {
   if (req.method === 'POST' && path === '/api/extract') {
     try {
       const body = await readBody(req);
-      const { project, stealth } = body;
+      const { project } = body;
       if (!project) { json(res, 400, { error: 'Missing project' }); return; }
 
       // Conflict guard
@@ -308,7 +308,6 @@ async function handleRequest(req, res) {
       }
 
       const args = ['cli.js', 'extract', project];
-      if (stealth) args.push('--stealth');
 
       const child = spawn(process.execPath, args, {
         cwd: __dirname,
@@ -372,6 +371,38 @@ async function handleRequest(req, res) {
     return;
   }
 
+  // ─── API: Restart — kill running jobs + restart server ───
+  if (req.method === 'POST' && path === '/api/restart') {
+    try {
+      // 1. Kill any running job
+      const progress = readProgress();
+      if (progress?.status === 'running' && progress.pid) {
+        try { process.kill(progress.pid, 'SIGTERM'); } catch {}
+        try {
+          writeFileSync(PROGRESS_FILE, JSON.stringify({
+            ...progress, status: 'stopped', stopped_at: Date.now(), updated_at: Date.now(),
+          }, null, 2));
+        } catch {}
+      }
+      json(res, 200, { restarting: true });
+
+      // 2. Restart the server process after response is sent
+      setTimeout(() => {
+        const child = spawn(process.execPath, [fileURLToPath(import.meta.url), ...process.argv.slice(2)], {
+          cwd: __dirname,
+          detached: true,
+          stdio: 'ignore',
+          env: { ...process.env, SEO_INTEL_AUTO_OPEN: '0' },
+        });
+        child.unref();
+        process.exit(0);
+      }, 300);
+    } catch (e) {
+      json(res, 500, { error: e.message });
+    }
+    return;
+  }
+
   // ─── API: Export actions ───
   if (req.method === 'POST' && path === '/api/export-actions') {
     try {
@@ -538,7 +569,7 @@ async function handleRequest(req, res) {
     // Whitelist allowed commands
     const ALLOWED = ['crawl', 'extract', 'analyze', 'export-actions', 'competitive-actions',
       'suggest-usecases', 'html', 'status', 'brief', 'keywords', 'report', 'guide',
-      'schemas', 'headings-audit', 'orphans', 'entities', 'friction', 'shallow', 'decay', 'export'];
+      'schemas', 'headings-audit', 'orphans', 'entities', 'friction', 'shallow', 'decay', 'export', 'templates'];
 
     if (!command || !ALLOWED.includes(command)) {
       json(res, 400, { error: `Invalid command. Allowed: ${ALLOWED.join(', ')}` });
@@ -564,14 +595,21 @@ async function handleRequest(req, res) {
       res.write(`data: ${JSON.stringify({ type, data })}\n\n`);
     };
 
+    const isLongRunning = ['crawl', 'extract'].includes(command);
+
     send('start', { command, project, args: args.slice(1) });
 
     const child = spawn(process.execPath, args, {
       cwd: __dirname,
       env: { ...process.env, FORCE_COLOR: '0', NO_COLOR: '1' },
+      // Crawl/extract: detach so they survive SSE disconnect
+      ...(isLongRunning ? { detached: true } : {}),
     });
 
+    let clientClosed = false;
+
     child.stdout.on('data', chunk => {
+      if (clientClosed) return;
       const lines = chunk.toString().split('\n');
       for (const line of lines) {
         if (line) send('stdout', line);
@@ -579,6 +617,7 @@ async function handleRequest(req, res) {
     });
 
     child.stderr.on('data', chunk => {
+      if (clientClosed) return;
       const lines = chunk.toString().split('\n');
       for (const line of lines) {
         if (line) send('stderr', line);
@@ -586,18 +625,24 @@ async function handleRequest(req, res) {
     });
 
     child.on('error', err => {
-      send('error', err.message);
-      res.end();
+      if (!clientClosed) { send('error', err.message); res.end(); }
     });
 
     child.on('close', code => {
-      send('exit', { code });
-      res.end();
+      if (!clientClosed) { send('exit', { code }); res.end(); }
     });
 
-    // Kill child if client disconnects
+    // Client disconnect: kill short commands, let crawl/extract continue
     req.on('close', () => {
-      if (!child.killed) child.kill();
+      clientClosed = true;
+      if (isLongRunning) {
+        // Detach — crawl/extract keeps running, progress file tracks it
+        child.unref();
+        if (child.stdout) child.stdout.destroy();
+        if (child.stderr) child.stderr.destroy();
+      } else {
+        if (!child.killed) child.kill();
+      }
     });
 
     return;