seo-intel 1.4.1 → 1.4.3

package/cli.js

@@ -660,6 +660,25 @@ program
       console.log(chalk.dim(`  ⚠  Dashboard refresh skipped: ${dashErr.message}`));
     }
 
+    // Auto-run site watch after crawl
+    try {
+      const { runWatch } = await import('./analyses/watch/index.js');
+      const watchResult = runWatch(db, project, { log: () => {} });
+      if (watchResult.snapshot) {
+        if (watchResult.isBaseline) {
+          console.log(chalk.dim(`  👁  Site Watch: baseline snapshot saved (health: ${watchResult.healthScore}/100)`));
+        } else if (watchResult.events.length) {
+          const t = watchResult.trend;
+          const trendStr = t > 0 ? `▲ +${t}` : t < 0 ? `▼ ${t}` : '';
+          console.log(chalk.dim(`  👁  Site Watch: ${watchResult.events.length} changes detected (health: ${watchResult.healthScore}/100${trendStr ? ' ' + trendStr : ''})`));
+        } else {
+          console.log(chalk.dim(`  👁  Site Watch: no changes (health: ${watchResult.healthScore}/100)`));
+        }
+      }
+    } catch (e) {
+      console.log(chalk.dim(`  ⚠  Site Watch skipped: ${e.message}`));
+    }
+
     if (opts.extract === false && totalExtracted === 0) {
       console.log(chalk.bold.green(`\n✅ Crawl complete (${elapsed}s) — raw data collected.`));
       console.log(chalk.white('  Next steps:'));
@@ -723,9 +742,9 @@ program
     console.log(chalk.yellow('Sending to Gemini...\n'));
 
     // Save prompt for debugging (markdown for Obsidian/agent compatibility)
-    const promptTs = Date.now();
+    const promptTs = new Date().toISOString().slice(0, 10);
     const promptPath = join(__dirname, `reports/${project}-prompt-${promptTs}.md`);
-    const promptFrontmatter = `---\nproject: ${project}\ngenerated: ${new Date(promptTs).toISOString()}\ntype: analysis-prompt\nmodel: gemini\n---\n\n`;
+    const promptFrontmatter = `---\nproject: ${project}\ngenerated: ${new Date().toISOString()}\ntype: analysis-prompt\nmodel: gemini\n---\n\n`;
     writeFileSync(promptPath, promptFrontmatter + prompt, 'utf8');
     console.log(chalk.gray(`Prompt saved: ${promptPath}`));
 
@@ -745,13 +764,13 @@ program
       analysis = JSON.parse(jsonMatch[0]);
     } catch {
       console.error(chalk.red('Could not parse JSON from response. Saving raw output.'));
-      const rawPath = join(__dirname, `reports/${project}-raw-${Date.now()}.md`);
+      const rawPath = join(__dirname, `reports/${project}-raw-${new Date().toISOString().slice(0, 10)}.md`);
       writeFileSync(rawPath, result, 'utf8');
       process.exit(1);
     }
 
     // Save structured analysis to file
-    const outPath = join(__dirname, `reports/${project}-analysis-${Date.now()}.json`);
+    const outPath = join(__dirname, `reports/${project}-analysis-${new Date().toISOString().slice(0, 10)}.json`);
     writeFileSync(outPath, JSON.stringify(analysis, null, 2), 'utf8');
 
     // Save to DB (so HTML dashboard picks it up)
@@ -911,7 +930,7 @@ Respond ONLY with a single valid JSON object matching this exact schema. No expl
       data = JSON.parse(jsonMatch[0]);
     } catch {
       console.error(chalk.red('Could not parse JSON from Gemini response.'));
-      const rawPath = join(__dirname, `reports/${project}-keywords-raw-${Date.now()}.md`);
+      const rawPath = join(__dirname, `reports/${project}-keywords-raw-${new Date().toISOString().slice(0, 10)}.md`);
       writeFileSync(rawPath, result, 'utf8');
       console.error(chalk.gray(`Raw output saved: ${rawPath}`));
       process.exit(1);
@@ -972,7 +991,7 @@ Respond ONLY with a single valid JSON object matching this exact schema. No expl
     }
 
     if (opts.save) {
-      const outPath = join(__dirname, `reports/${project}-keywords-${Date.now()}.json`);
+      const outPath = join(__dirname, `reports/${project}-keywords-${new Date().toISOString().slice(0, 10)}.json`);
       writeFileSync(outPath, JSON.stringify(data, null, 2), 'utf8');
       console.log(chalk.bold.green(`✅ Report saved: ${outPath}\n`));
 
@@ -1740,7 +1759,7 @@ async function runAnalysis(project, db) {
   try {
     const jsonMatch = result.match(/\{[\s\S]*\}/);
     const analysis = JSON.parse(jsonMatch[0]);
-    const outPath = join(__dirname, `reports/${project}-analysis-${Date.now()}.json`);
+    const outPath = join(__dirname, `reports/${project}-analysis-${new Date().toISOString().slice(0, 10)}.json`);
     writeFileSync(outPath, JSON.stringify(analysis, null, 2), 'utf8');
 
     // Save to DB
@@ -2391,7 +2410,7 @@ program
       report += `> Analyze this heading structure from ${page.domain}. What H2/H3 sub-topics are logically missing? What would a user expect to find that isn't covered? Be specific.\n\n---\n\n`;
     }
 
-    const outPath = join(__dirname, `reports/${project}-headings-audit-${Date.now()}.md`);
+    const outPath = join(__dirname, `reports/${project}-headings-audit-${new Date().toISOString().slice(0, 10)}.md`);
     writeFileSync(outPath, report, 'utf8');
 
     console.log(chalk.bold.green(`\n✅ Full audit saved: ${outPath}`));
@@ -2653,7 +2672,7 @@ program
 
     // ── Save ──
     if (opts.save) {
-      const outPath = join(__dirname, `reports/${project}-entities-${Date.now()}.md`);
+      const outPath = join(__dirname, `reports/${project}-entities-${new Date().toISOString().slice(0, 10)}.md`);
       writeFileSync(outPath, mdOutput, 'utf8');
       console.log(chalk.bold.green(`  ✅ Entity map saved: ${outPath}\n`));
     }
@@ -3350,7 +3369,7 @@ program
 
     // ── Save ──
     if (opts.save) {
-      const outPath = join(__dirname, `reports/${project}-brief-${Date.now()}.md`);
+      const outPath = join(__dirname, `reports/${project}-brief-${new Date().toISOString().slice(0, 10)}.md`);
       writeFileSync(outPath, mdOutput, 'utf8');
       console.log(chalk.bold.green(`  ✅ Brief saved: ${outPath}\n`));
     }
@@ -3694,7 +3713,7 @@ program
     }
 
     if (opts.save) {
-      const outPath = join(__dirname, `reports/${project}-js-delta-${Date.now()}.md`);
+      const outPath = join(__dirname, `reports/${project}-js-delta-${new Date().toISOString().slice(0, 10)}.md`);
       writeFileSync(outPath, mdOutput, 'utf8');
       console.log(chalk.bold.green(`  ✅ Report saved: ${outPath}\n`));
     }
@@ -3840,7 +3859,7 @@ program
     }
 
     // Output
-    const timestamp = Date.now();
+    const timestamp = new Date().toISOString().slice(0, 10);
     const defaultPath = join(__dirname, `reports/${project}-export-${timestamp}.${format}`);
     const outPath = opts.output || defaultPath;
 
@@ -4218,12 +4237,134 @@ program
       for (const p of worst) {
         md += `- **${p.url}** — ${p.score}/100 (${p.tier})\n`;
       }
-      const outPath = join(__dirname, `reports/${project}-aeo-${Date.now()}.md`);
+      const outPath = join(__dirname, `reports/${project}-aeo-${new Date().toISOString().slice(0, 10)}.md`);
       writeFileSync(outPath, md, 'utf8');
       console.log(chalk.bold.green(`  ✅ Report saved: ${outPath}\n`));
     }
   });
 
+// ── SITE WATCH ──────────────────────────────────────────────────────────
+program
+  .command('watch <project>')
+  .description('Site health monitor — detect changes between crawl runs')
+  .option('--format <type>', 'Output format: brief or json', 'brief')
+  .action(async (project, opts) => {
+    const db = getDb();
+    const config = loadConfig(project);
+    if (!config) return;
+    const isJson = opts.format === 'json';
+
+    if (!isJson) {
+      printAttackHeader('Site Watch', project);
+    }
+
+    const { runWatch } = await import('./analyses/watch/index.js');
+
+    const result = runWatch(db, project, {
+      log: isJson ? () => {} : (msg) => console.log(chalk.gray(msg)),
+    });
+
+    if (!result.snapshot) {
+      if (isJson) console.log(JSON.stringify({ ok: false, error: 'No crawled pages found' }));
+      return;
+    }
+
+    if (isJson) {
+      console.log(JSON.stringify({
+        snapshot: result.snapshot,
+        events: result.events,
+        healthScore: result.healthScore,
+        previousHealthScore: result.previousHealthScore,
+        trend: result.trend,
+        isBaseline: result.isBaseline,
+      }, null, 2));
+    } else {
+      // ── Health Score ──
+      const score = result.healthScore;
+      const scoreFmt = score >= 80 ? chalk.bold.green(score + '/100')
+        : score >= 60 ? chalk.bold.yellow(score + '/100')
+        : chalk.bold.red(score + '/100');
+
+      console.log('');
+      console.log(`    Health Score:  ${scoreFmt}`);
+
+      if (result.previousHealthScore !== null) {
+        const t = result.trend;
+        const trendStr = t > 0 ? chalk.green(`▲ +${t}`)
+          : t < 0 ? chalk.red(`▼ ${t}`)
+          : chalk.gray('— unchanged');
+        console.log(`                  ${trendStr} (was ${result.previousHealthScore}/100)`);
+      }
+      console.log('');
+
+      if (result.isBaseline) {
+        console.log(chalk.bold.green(`    ✅ Baseline captured — ${result.snapshot.total_pages} pages`));
+        console.log(chalk.gray('    Run another crawl to see changes.\n'));
+      } else {
+        // ── Severity summary ──
+        const { errors_count: e, warnings_count: w, notices_count: n } = result.snapshot;
+        const prev = result.previousHealthScore !== null ? {
+          e: 0, w: 0, n: 0, // We'll compute deltas from events
+        } : null;
+
+        console.log(`    ${chalk.red('●')} Critical:  ${e}`);
+        console.log(`    ${chalk.yellow('●')} Warning:   ${w}`);
+        console.log(`    ${chalk.gray('●')} Notice:    ${n}`);
+        console.log('');
+
+        // ── Event list ──
+        if (result.events.length) {
+          console.log(chalk.bold('    What\'s New:'));
+          console.log('');
+
+          const shown = result.events.slice(0, 25);
+          for (const ev of shown) {
+            const icon = ev.severity === 'critical' ? chalk.red('●')
+              : ev.severity === 'warning' ? chalk.yellow('▲')
+              : chalk.gray('○');
+            const sev = ev.severity.toUpperCase().padEnd(8);
+            const path = ev.url.replace(/https?:\/\/[^/]+/, '') || '/';
+
+            let desc = '';
+            switch (ev.event_type) {
+              case 'page_added':          desc = `${path} — new page`; break;
+              case 'page_removed':        desc = `${path} — removed`; break;
+              case 'new_error':           desc = `${path} → ${ev.new_value} (was ${ev.old_value})`; break;
+              case 'status_changed':      desc = `${path} status ${ev.old_value} → ${ev.new_value}`; break;
+              case 'title_changed':       desc = `${path} title: "${(ev.old_value || '').slice(0, 30)}" → "${(ev.new_value || '').slice(0, 30)}"`; break;
+              case 'h1_changed':          desc = `${path} H1 changed`; break;
+              case 'meta_desc_changed':   desc = `${path} meta description changed`; break;
+              case 'word_count_changed':  desc = `${path} word count ${ev.old_value} → ${ev.new_value}`; break;
+              case 'indexability_changed': desc = `${path} became ${ev.new_value}`; break;
+              case 'content_changed':     desc = `${path} content updated`; break;
+              default:                    desc = `${path} — ${ev.event_type.replace(/_/g, ' ')}`;
+            }
+
+            console.log(`    ${icon} ${chalk.dim(sev)} ${desc}`);
+          }
+
+          if (result.events.length > 25) {
+            console.log(chalk.gray(`    ... and ${result.events.length - 25} more`));
+          }
+          console.log('');
+        } else {
+          console.log(chalk.green('    ✅ No changes detected since last crawl.\n'));
+        }
+      }
+    }
+
+    // ── Regenerate dashboard ──
+    if (!isJson) {
+      try {
+        const configs = loadAllConfigs();
+        generateMultiDashboard(db, configs);
+        console.log(chalk.green('  ✅ Dashboard updated with Site Watch card\n'));
+      } catch (e) {
+        console.log(chalk.gray(`  (Dashboard not updated: ${e.message})\n`));
+      }
+    }
+  });
+
 // ── GAP INTEL ────────────────────────────────────────────────────────────
 
 program
@@ -4380,7 +4521,7 @@ program
       const slug = opts.topic
         ? opts.topic.toLowerCase().replace(/[^a-z0-9]+/g, '-').slice(0, 50)
         : 'auto';
-      const filename = `${project}-blog-draft-${slug}-${Date.now()}.md`;
+      const filename = `${project}-blog-draft-${slug}-${new Date().toISOString().slice(0, 10)}.md`;
       const outPath = join(__dirname, 'reports', filename);
       writeFileSync(outPath, draft, 'utf8');
       console.log(chalk.bold.green(`  ✅ Draft saved: ${outPath}`));

package/db/db.js

@@ -29,6 +29,51 @@ export function getDb(dbPath = './seo-intel.db') {
   // Backfill first_seen_at from crawled_at for existing rows
   _db.exec('UPDATE pages SET first_seen_at = crawled_at WHERE first_seen_at IS NULL');
 
+  // Site Watch tables
+  try {
+    _db.exec(`
+      CREATE TABLE IF NOT EXISTS watch_snapshots (
+        id             INTEGER PRIMARY KEY AUTOINCREMENT,
+        project        TEXT NOT NULL,
+        created_at     INTEGER NOT NULL,
+        total_pages    INTEGER NOT NULL DEFAULT 0,
+        health_score   INTEGER,
+        errors_count   INTEGER NOT NULL DEFAULT 0,
+        warnings_count INTEGER NOT NULL DEFAULT 0,
+        notices_count  INTEGER NOT NULL DEFAULT 0
+      );
+      CREATE INDEX IF NOT EXISTS idx_watch_snapshots_project ON watch_snapshots(project, created_at DESC);
+
+      CREATE TABLE IF NOT EXISTS watch_page_states (
+        id           INTEGER PRIMARY KEY AUTOINCREMENT,
+        snapshot_id  INTEGER NOT NULL REFERENCES watch_snapshots(id) ON DELETE CASCADE,
+        url          TEXT NOT NULL,
+        status_code  INTEGER,
+        title        TEXT,
+        h1           TEXT,
+        meta_desc    TEXT,
+        word_count   INTEGER,
+        is_indexable INTEGER DEFAULT 1,
+        content_hash TEXT,
+        UNIQUE(snapshot_id, url)
+      );
+      CREATE INDEX IF NOT EXISTS idx_watch_page_states_snapshot ON watch_page_states(snapshot_id);
+
+      CREATE TABLE IF NOT EXISTS watch_events (
+        id           INTEGER PRIMARY KEY AUTOINCREMENT,
+        snapshot_id  INTEGER NOT NULL REFERENCES watch_snapshots(id) ON DELETE CASCADE,
+        event_type   TEXT NOT NULL,
+        severity     TEXT NOT NULL,
+        url          TEXT NOT NULL,
+        old_value    TEXT,
+        new_value    TEXT,
+        details      TEXT
+      );
+      CREATE INDEX IF NOT EXISTS idx_watch_events_snapshot ON watch_events(snapshot_id);
+      CREATE INDEX IF NOT EXISTS idx_watch_events_type ON watch_events(event_type);
+    `);
+  } catch { /* tables already exist */ }
+
   // Migrate existing analyses → insights (one-time)
   _migrateAnalysesToInsights(_db);
 
@@ -49,6 +94,7 @@ function _insightFingerprint(type, item) {
     case 'positioning':       raw = 'positioning'; break;
     case 'keyword_inventor':  raw = item.phrase || ''; break;
     case 'citability_gap':    raw = item.url || ''; break;
+    case 'site_watch':        raw = `${item.url || ''}::${item.event_type || ''}`; break;
     default:                  raw = JSON.stringify(item);
   }
   return raw.toLowerCase().replace(/[^a-z0-9\s]/g, '').replace(/\s+/g, ' ').trim();
@@ -201,6 +247,7 @@ export function getActiveInsights(db, project) {
     technical_gaps: grouped.technical_gap || [],
     positioning: grouped.positioning?.[0] || null,
     keyword_inventor: grouped.keyword_inventor || [],
+    site_watch: grouped.site_watch || [],
     generated_at: rows.length ? Math.max(...rows.map(r => r.last_seen)) : null,
   };
 }
@@ -548,3 +595,29 @@ export function getHeadingStructure(db, project) {
     ORDER BY d.domain, h.level
   `).all(project);
 }
+
+// ── Site Watch ────────────────────────────────────────────────────────────
+
+export function getLatestWatchSnapshot(db, project) {
+  return db.prepare(
+    'SELECT * FROM watch_snapshots WHERE project = ? ORDER BY created_at DESC LIMIT 1'
+  ).get(project) || null;
+}
+
+export function getWatchPageStates(db, snapshotId) {
+  return db.prepare(
+    'SELECT * FROM watch_page_states WHERE snapshot_id = ?'
+  ).all(snapshotId);
+}
+
+export function getWatchEvents(db, snapshotId) {
+  return db.prepare(
+    'SELECT * FROM watch_events WHERE snapshot_id = ? ORDER BY CASE severity WHEN \'critical\' THEN 0 WHEN \'warning\' THEN 1 ELSE 2 END, event_type'
+  ).all(snapshotId);
+}
+
+export function getWatchHistory(db, project, limit = 10) {
+  return db.prepare(
+    'SELECT * FROM watch_snapshots WHERE project = ? ORDER BY created_at DESC LIMIT ?'
+  ).all(project, limit);
+}

package/lib/export-zip.js

@@ -0,0 +1,102 @@
+/**
+ * Minimal ZIP archive builder — zero dependencies.
+ * Creates valid ZIP files (uncompressed / STORE method) from an array of entries.
+ * Sufficient for text-based exports (MD, JSON, CSV) where deflate adds little value.
+ *
+ * Usage:
+ *   import { createZip } from './lib/export-zip.js';
+ *   const buf = createZip([{ name: 'report.md', content: '# Hello' }]);
+ *   res.end(buf);
+ */
+
+/**
+ * @param {{ name: string, content: string | Buffer }[]} entries
+ * @returns {Buffer} Valid ZIP archive
+ */
+export function createZip(entries) {
+  const localHeaders = [];
+  const centralHeaders = [];
+  let offset = 0;
+
+  for (const entry of entries) {
+    const nameBytes = Buffer.from(entry.name, 'utf8');
+    const data = typeof entry.content === 'string' ? Buffer.from(entry.content, 'utf8') : entry.content;
+    const crc = crc32(data);
+
+    // Local file header (30 + nameLen + dataLen)
+    const local = Buffer.alloc(30 + nameBytes.length);
+    local.writeUInt32LE(0x04034b50, 0);   // signature
+    local.writeUInt16LE(20, 4);            // version needed (2.0)
+    local.writeUInt16LE(0, 6);             // flags
+    local.writeUInt16LE(0, 8);             // compression: STORE
+    local.writeUInt16LE(0, 10);            // mod time
+    local.writeUInt16LE(0, 12);            // mod date
+    local.writeUInt32LE(crc, 14);          // crc-32
+    local.writeUInt32LE(data.length, 18);  // compressed size
+    local.writeUInt32LE(data.length, 22);  // uncompressed size
+    local.writeUInt16LE(nameBytes.length, 26); // filename length
+    local.writeUInt16LE(0, 28);            // extra field length
+    nameBytes.copy(local, 30);
+
+    localHeaders.push(Buffer.concat([local, data]));
+
+    // Central directory header (46 + nameLen)
+    const central = Buffer.alloc(46 + nameBytes.length);
+    central.writeUInt32LE(0x02014b50, 0);  // signature
+    central.writeUInt16LE(20, 4);          // version made by
+    central.writeUInt16LE(20, 6);          // version needed
+    central.writeUInt16LE(0, 8);           // flags
+    central.writeUInt16LE(0, 10);          // compression: STORE
+    central.writeUInt16LE(0, 12);          // mod time
+    central.writeUInt16LE(0, 14);          // mod date
+    central.writeUInt32LE(crc, 16);        // crc-32
+    central.writeUInt32LE(data.length, 20); // compressed size
+    central.writeUInt32LE(data.length, 24); // uncompressed size
+    central.writeUInt16LE(nameBytes.length, 28); // filename length
+    central.writeUInt16LE(0, 30);          // extra field length
+    central.writeUInt16LE(0, 32);          // comment length
+    central.writeUInt16LE(0, 34);          // disk number start
+    central.writeUInt16LE(0, 36);          // internal attrs
+    central.writeUInt32LE(0, 38);          // external attrs
+    central.writeUInt32LE(offset, 42);     // local header offset
+    nameBytes.copy(central, 46);
+
+    centralHeaders.push(central);
+    offset += local.length + data.length;
+  }
+
+  const centralDir = Buffer.concat(centralHeaders);
+  const centralDirOffset = offset;
+
+  // End of central directory record (22 bytes)
+  const eocd = Buffer.alloc(22);
+  eocd.writeUInt32LE(0x06054b50, 0);                // signature
+  eocd.writeUInt16LE(0, 4);                          // disk number
+  eocd.writeUInt16LE(0, 6);                          // disk with central dir
+  eocd.writeUInt16LE(entries.length, 8);              // entries on this disk
+  eocd.writeUInt16LE(entries.length, 10);             // total entries
+  eocd.writeUInt32LE(centralDir.length, 12);          // central dir size
+  eocd.writeUInt32LE(centralDirOffset, 16);           // central dir offset
+  eocd.writeUInt16LE(0, 20);                          // comment length
+
+  return Buffer.concat([...localHeaders, centralDir, eocd]);
+}
+
+// ─── CRC-32 (IEEE 802.3) ────────────────────────────────────────────────────
+
+const CRC_TABLE = new Uint32Array(256);
+for (let n = 0; n < 256; n++) {
+  let c = n;
+  for (let k = 0; k < 8; k++) {
+    c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
+  }
+  CRC_TABLE[n] = c;
+}
+
+function crc32(buf) {
+  let crc = 0xFFFFFFFF;
+  for (let i = 0; i < buf.length; i++) {
+    crc = CRC_TABLE[(crc ^ buf[i]) & 0xFF] ^ (crc >>> 8);
+  }
+  return (crc ^ 0xFFFFFFFF) >>> 0;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "seo-intel",
-  "version": "1.4.1",
+  "version": "1.4.3",
   "description": "Local Ahrefs-style SEO competitor intelligence. Crawl → SQLite → cloud analysis.",
   "type": "module",
   "license": "SEE LICENSE IN LICENSE",