seo-intel 1.4.1 → 1.4.2

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 1.4.2 (2026-04-05)
+
+### New Feature: Site Watch
+- `seo-intel watch <project>` — detect changes between crawl runs and track site health
+- Health Score (0-100) based on page errors, missing titles, missing H1s
+- Diff engine detects 10 event types: new/removed pages, status changes, title/H1/meta changes, word count shifts, indexability flips, content updates
+- Events classified by severity: critical, warning, notice — with trend arrows
+- Auto-runs after every crawl with a one-liner summary
+- Dashboard card: health score gauge, severity counts with deltas, "What's New" event table
+- Significant changes (critical/warning) feed into Intelligence Ledger as `site_watch` insights
+- Available via CLI, dashboard terminal, and froggo.js API
+- Free tier — no license required
+
 ## 1.4.1 (2026-04-03)
 
 ### Fixes

package/analyses/watch/diff.js

@@ -0,0 +1,158 @@
+/**
+ * Site Watch — Diff Engine
+ *
+ * Pure function: compares two page snapshots and returns change events.
+ * Zero I/O, zero side effects — deterministic and testable.
+ */
+
+/**
+ * @param {object[]} currentPages  - { url, status_code, title, h1, meta_desc, word_count, is_indexable, content_hash }
+ * @param {object[]} previousPages - same shape, from previous snapshot
+ * @returns {object[]} Array of { event_type, severity, url, old_value, new_value, details }
+ */
+export function diffPages(currentPages, previousPages) {
+  const events = [];
+
+  const currentMap = new Map(currentPages.map(p => [p.url, p]));
+  const previousMap = new Map(previousPages.map(p => [p.url, p]));
+
+  // ── Pages added ──────────────────────────────────────────────────────────
+  for (const [url, page] of currentMap) {
+    if (!previousMap.has(url)) {
+      events.push({
+        event_type: 'page_added',
+        severity: 'notice',
+        url,
+        old_value: null,
+        new_value: String(page.status_code || 200),
+        details: JSON.stringify({ title: page.title, word_count: page.word_count }),
+      });
+    }
+  }
+
+  // ── Pages removed ────────────────────────────────────────────────────────
+  for (const [url, page] of previousMap) {
+    if (!currentMap.has(url)) {
+      events.push({
+        event_type: 'page_removed',
+        severity: 'warning',
+        url,
+        old_value: String(page.status_code || 200),
+        new_value: null,
+        details: JSON.stringify({ title: page.title, word_count: page.word_count }),
+      });
+    }
+  }
+
+  // ── Per-page field comparisons ───────────────────────────────────────────
+  for (const [url, curr] of currentMap) {
+    const prev = previousMap.get(url);
+    if (!prev) continue;
+
+    // Status code change
+    if (curr.status_code !== prev.status_code) {
+      const isNewError = prev.status_code < 400 && curr.status_code >= 400;
+      const isRecovery = prev.status_code >= 400 && curr.status_code < 400;
+      const severity = isNewError ? 'critical'
+        : curr.status_code >= 400 ? 'critical'
+        : isRecovery ? 'notice'
+        : 'warning';
+
+      events.push({
+        event_type: isNewError ? 'new_error' : 'status_changed',
+        severity,
+        url,
+        old_value: String(prev.status_code),
+        new_value: String(curr.status_code),
+        details: null,
+      });
+    }
+
+    // Title change
+    if (normalise(curr.title) !== normalise(prev.title)) {
+      events.push({
+        event_type: 'title_changed',
+        severity: 'notice',
+        url,
+        old_value: prev.title || '',
+        new_value: curr.title || '',
+        details: null,
+      });
+    }
+
+    // H1 change
+    if (normalise(curr.h1) !== normalise(prev.h1)) {
+      events.push({
+        event_type: 'h1_changed',
+        severity: 'notice',
+        url,
+        old_value: prev.h1 || '',
+        new_value: curr.h1 || '',
+        details: null,
+      });
+    }
+
+    // Meta description change
+    if (normalise(curr.meta_desc) !== normalise(prev.meta_desc)) {
+      events.push({
+        event_type: 'meta_desc_changed',
+        severity: 'notice',
+        url,
+        old_value: prev.meta_desc || '',
+        new_value: curr.meta_desc || '',
+        details: null,
+      });
+    }
+
+    // Word count significant change (>20%)
+    const prevWc = prev.word_count || 0;
+    const currWc = curr.word_count || 0;
+    if (prevWc > 0 && Math.abs(currWc - prevWc) / prevWc > 0.2) {
+      events.push({
+        event_type: 'word_count_changed',
+        severity: 'notice',
+        url,
+        old_value: String(prevWc),
+        new_value: String(currWc),
+        details: JSON.stringify({ delta_pct: Math.round(((currWc - prevWc) / prevWc) * 100) }),
+      });
+    }
+
+    // Indexability change
+    const prevIdx = prev.is_indexable ? 1 : 0;
+    const currIdx = curr.is_indexable ? 1 : 0;
+    if (currIdx !== prevIdx) {
+      events.push({
+        event_type: 'indexability_changed',
+        severity: currIdx === 0 ? 'critical' : 'notice',
+        url,
+        old_value: prevIdx ? 'indexable' : 'non-indexable',
+        new_value: currIdx ? 'indexable' : 'non-indexable',
+        details: null,
+      });
+    }
+
+    // Content hash change (body text changed)
+    if (curr.content_hash && prev.content_hash && curr.content_hash !== prev.content_hash) {
+      events.push({
+        event_type: 'content_changed',
+        severity: 'notice',
+        url,
+        old_value: prev.content_hash?.slice(0, 8) || '',
+        new_value: curr.content_hash?.slice(0, 8) || '',
+        details: null,
+      });
+    }
+  }
+
+  // Sort: critical first, then warning, then notice
+  const severityOrder = { critical: 0, warning: 1, notice: 2 };
+  events.sort((a, b) => (severityOrder[a.severity] ?? 9) - (severityOrder[b.severity] ?? 9));
+
+  return events;
+}
+
+/** Normalise a string for comparison (null-safe, trimmed, lowercased). */
+function normalise(s) {
+  return (s || '').trim().toLowerCase();
+}

package/analyses/watch/health.js

@@ -0,0 +1,78 @@
+/**
+ * Site Watch — Health Score Calculator
+ *
+ * Pure function: evaluates site health from current page data.
+ * Zero I/O, zero side effects.
+ */
+
+/**
+ * @param {object[]} pages - { url, status_code, title, h1, meta_desc, word_count, is_indexable }
+ * @returns {object} { score, errors, warnings, notices, details }
+ */
+export function calculateHealthScore(pages) {
+  if (!pages.length) return { score: 100, errors: 0, warnings: 0, notices: 0, details: [] };
+
+  const details = [];
+  let errorPages = 0;
+
+  // Track duplicates
+  const titleCounts = new Map();
+  for (const p of pages) {
+    if (p.title && p.status_code < 400) {
+      const t = p.title.trim().toLowerCase();
+      titleCounts.set(t, (titleCounts.get(t) || 0) + 1);
+    }
+  }
+
+  for (const p of pages) {
+    let hasError = false;
+
+    // ── Errors (reduce health score) ───────────────────────────────────────
+    if (p.status_code >= 400) {
+      details.push({ url: p.url, severity: 'error', issue: `${p.status_code} error` });
+      hasError = true;
+    }
+
+    if (p.status_code < 400 && (!p.title || !p.title.trim())) {
+      details.push({ url: p.url, severity: 'error', issue: 'Missing title' });
+      hasError = true;
+    }
+
+    if (p.status_code < 400 && (!p.h1 || !p.h1.trim())) {
+      details.push({ url: p.url, severity: 'error', issue: 'Missing H1' });
+      hasError = true;
+    }
+
+    if (hasError) errorPages++;
+
+    // ── Warnings (tracked, don't reduce score) ────────────────────────────
+    if (p.status_code >= 300 && p.status_code < 400) {
+      details.push({ url: p.url, severity: 'warning', issue: `${p.status_code} redirect` });
+    }
+
+    if (p.status_code < 400 && (!p.meta_desc || !p.meta_desc.trim())) {
+      details.push({ url: p.url, severity: 'warning', issue: 'Missing meta description' });
+    }
+
+    if (p.title && p.status_code < 400) {
+      const t = p.title.trim().toLowerCase();
+      if (titleCounts.get(t) > 1) {
+        details.push({ url: p.url, severity: 'warning', issue: 'Duplicate title' });
+      }
+    }
+
+    // ── Notices ────────────────────────────────────────────────────────────
+    if (p.status_code < 400 && (p.word_count || 0) < 100 && (p.word_count || 0) > 0) {
+      details.push({ url: p.url, severity: 'notice', issue: 'Thin content (<100 words)' });
+    }
+  }
+
+  const errors = details.filter(d => d.severity === 'error').length;
+  const warnings = details.filter(d => d.severity === 'warning').length;
+  const notices = details.filter(d => d.severity === 'notice').length;
+
+  // Health score = % of pages without errors
+  const score = Math.round(((pages.length - errorPages) / pages.length) * 100);
+
+  return { score, errors, warnings, notices, details };
+}

package/analyses/watch/index.js

@@ -0,0 +1,215 @@
+/**
+ * Site Watch — Orchestrator
+ *
+ * Gathers current page state, diffs against previous snapshot,
+ * persists results, and feeds significant changes into the Intelligence Ledger.
+ */
+
+import { diffPages } from './diff.js';
+import { calculateHealthScore } from './health.js';
+import {
+  getLatestWatchSnapshot,
+  getWatchPageStates,
+  getWatchEvents,
+  getWatchHistory,
+} from '../../db/db.js';
+
+// ═══════════════════════════════════════════════════════════════════════════
+// MAIN RUNNER
+// ═══════════════════════════════════════════════════════════════════════════
+
+/**
+ * Run site watch analysis for a project.
+ *
+ * @param {import('node:sqlite').DatabaseSync} db
+ * @param {string} project
+ * @param {object} [opts] - { log: function }
+ * @returns {object} { snapshot, events, healthScore, previousHealthScore, trend, isBaseline }
+ */
+export function runWatch(db, project, opts = {}) {
+  const log = opts.log || console.log;
+
+  // ── Gather current page state ──────────────────────────────────────────
+  const currentPages = db.prepare(`
+    SELECT
+      p.url, p.status_code, p.title, p.meta_desc, p.word_count,
+      p.is_indexable, p.content_hash,
+      (SELECT text FROM headings WHERE page_id = p.id AND level = 1 ORDER BY id LIMIT 1) as h1
+    FROM pages p
+    JOIN domains d ON d.id = p.domain_id
+    WHERE d.project = ? AND d.role IN ('target', 'owned')
+    ORDER BY p.url
+  `).all(project);
+
+  if (!currentPages.length) {
+    log('  No crawled pages found. Run crawl first.');
+    return { snapshot: null, events: [], healthScore: 0, previousHealthScore: null, trend: 0, isBaseline: false };
+  }
+
+  // ── Health score ───────────────────────────────────────────────────────
+  const health = calculateHealthScore(currentPages);
+
+  // ── Load previous snapshot ─────────────────────────────────────────────
+  const prevSnapshot = getLatestWatchSnapshot(db, project);
+  let events = [];
+  let isBaseline = false;
+
+  if (prevSnapshot) {
+    const prevPages = getWatchPageStates(db, prevSnapshot.id);
+    events = diffPages(currentPages, prevPages);
+    log(`  Compared ${currentPages.length} pages against snapshot from ${new Date(prevSnapshot.created_at).toLocaleDateString()}`);
+  } else {
+    isBaseline = true;
+    log(`  Baseline snapshot — ${currentPages.length} pages captured`);
+  }
+
+  // ── Persist new snapshot ───────────────────────────────────────────────
+  const now = Date.now();
+  const criticalCount = events.filter(e => e.severity === 'critical').length;
+  const warningCount = events.filter(e => e.severity === 'warning').length;
+  const noticeCount = events.filter(e => e.severity === 'notice').length;
+
+  const snapshotResult = db.prepare(`
+    INSERT INTO watch_snapshots (project, created_at, total_pages, health_score, errors_count, warnings_count, notices_count)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+  `).run(project, now, currentPages.length, health.score, criticalCount, warningCount, noticeCount);
+
+  const snapshotId = Number(db.prepare('SELECT last_insert_rowid() as id').get().id);
+
+  // ── Persist page states ────────────────────────────────────────────────
+  const stateStmt = db.prepare(`
+    INSERT INTO watch_page_states (snapshot_id, url, status_code, title, h1, meta_desc, word_count, is_indexable, content_hash)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+
+  db.exec('BEGIN');
+  try {
+    for (const p of currentPages) {
+      stateStmt.run(snapshotId, p.url, p.status_code, p.title, p.h1, p.meta_desc, p.word_count, p.is_indexable ? 1 : 0, p.content_hash);
+    }
+    db.exec('COMMIT');
+  } catch (e) {
+    db.exec('ROLLBACK');
+    throw e;
+  }
+
+  // ── Persist events ─────────────────────────────────────────────────────
+  if (events.length) {
+    const eventStmt = db.prepare(`
+      INSERT INTO watch_events (snapshot_id, event_type, severity, url, old_value, new_value, details)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    db.exec('BEGIN');
+    try {
+      for (const e of events) {
+        eventStmt.run(snapshotId, e.event_type, e.severity, e.url, e.old_value, e.new_value, e.details);
+      }
+      db.exec('COMMIT');
+    } catch (e) {
+      db.exec('ROLLBACK');
+      throw e;
+    }
+  }
+
+  // ── Feed Intelligence Ledger (critical + warning only) ─────────────────
+  const significant = events.filter(e => e.severity === 'critical' || e.severity === 'warning');
+  if (significant.length) {
+    _upsertWatchInsights(db, project, significant, now);
+  }
+
+  const previousHealthScore = prevSnapshot?.health_score ?? null;
+  const trend = previousHealthScore !== null ? health.score - previousHealthScore : 0;
+
+  const snapshot = {
+    id: snapshotId,
+    project,
+    created_at: now,
+    total_pages: currentPages.length,
+    health_score: health.score,
+    errors_count: criticalCount,
+    warnings_count: warningCount,
+    notices_count: noticeCount,
+  };
+
+  return {
+    snapshot,
+    events,
+    healthScore: health.score,
+    healthDetails: health,
+    previousHealthScore,
+    trend,
+    isBaseline,
+  };
+}
+
+// ═══════════════════════════════════════════════════════════════════════════
+// DASHBOARD DATA
+// ═══════════════════════════════════════════════════════════════════════════
+
+/**
+ * Get watch data for dashboard rendering.
+ *
+ * @param {import('node:sqlite').DatabaseSync} db
+ * @param {string} project
+ * @returns {object|null} { current, previous, events, trend }
+ */
+export function getWatchData(db, project) {
+  const history = getWatchHistory(db, project, 2);
+  if (!history.length) return null;
+
+  const current = history[0];
+  const previous = history[1] || null;
+  const events = getWatchEvents(db, current.id);
+  const trend = previous ? current.health_score - previous.health_score : 0;
+
+  return { current, previous, events, trend };
+}
+
+// ═══════════════════════════════════════════════════════════════════════════
+// INTERNAL HELPERS
+// ═══════════════════════════════════════════════════════════════════════════
+
+function _upsertWatchInsights(db, project, events, timestamp) {
+  const upsertStmt = db.prepare(`
+    INSERT INTO insights (project, type, status, fingerprint, first_seen, last_seen, source_analysis_id, data)
+    VALUES (?, 'site_watch', 'active', ?, ?, ?, NULL, ?)
+    ON CONFLICT(project, type, fingerprint) DO UPDATE SET
+      last_seen = excluded.last_seen,
+      data = excluded.data
+  `);
+
+  db.exec('BEGIN');
+  try {
+    for (const e of events) {
+      const raw = `${e.url || ''}::${e.event_type || ''}`;
+      const fp = raw.toLowerCase().replace(/[^a-z0-9\s]/g, '').replace(/\s+/g, ' ').trim();
+      if (!fp) continue;
+
+      const data = {
+        url: e.url,
+        event_type: e.event_type,
+        severity: e.severity,
+        old_value: e.old_value,
+        new_value: e.new_value,
+        summary: _eventSummary(e),
+      };
+
+      upsertStmt.run(project, fp, timestamp, timestamp, JSON.stringify(data));
+    }
+    db.exec('COMMIT');
+  } catch (err) {
+    db.exec('ROLLBACK');
+    console.error('[watch] insight upsert failed:', err.message);
+  }
+}
+
+function _eventSummary(e) {
+  switch (e.event_type) {
+    case 'new_error':            return `${e.url} returned ${e.new_value} (was ${e.old_value})`;
+    case 'status_changed':       return `${e.url} status ${e.old_value} → ${e.new_value}`;
+    case 'page_removed':         return `${e.url} disappeared from crawl`;
+    case 'indexability_changed':  return `${e.url} became ${e.new_value}`;
+    default:                     return `${e.event_type.replace(/_/g, ' ')} on ${e.url}`;
+  }
+}

package/cli.js

@@ -660,6 +660,25 @@ program
       console.log(chalk.dim(`  ⚠  Dashboard refresh skipped: ${dashErr.message}`));
     }
 
+    // Auto-run site watch after crawl
+    try {
+      const { runWatch } = await import('./analyses/watch/index.js');
+      const watchResult = runWatch(db, project, { log: () => {} });
+      if (watchResult.snapshot) {
+        if (watchResult.isBaseline) {
+          console.log(chalk.dim(`  👁  Site Watch: baseline snapshot saved (health: ${watchResult.healthScore}/100)`));
+        } else if (watchResult.events.length) {
+          const t = watchResult.trend;
+          const trendStr = t > 0 ? `▲ +${t}` : t < 0 ? `▼ ${t}` : '';
+          console.log(chalk.dim(`  👁  Site Watch: ${watchResult.events.length} changes detected (health: ${watchResult.healthScore}/100${trendStr ? ' ' + trendStr : ''})`));
+        } else {
+          console.log(chalk.dim(`  👁  Site Watch: no changes (health: ${watchResult.healthScore}/100)`));
+        }
+      }
+    } catch (e) {
+      console.log(chalk.dim(`  ⚠  Site Watch skipped: ${e.message}`));
+    }
+
     if (opts.extract === false && totalExtracted === 0) {
       console.log(chalk.bold.green(`\n✅ Crawl complete (${elapsed}s) — raw data collected.`));
       console.log(chalk.white('  Next steps:'));
@@ -4224,6 +4243,128 @@ program
     }
   });
 
+// ── SITE WATCH ──────────────────────────────────────────────────────────
+program
+  .command('watch <project>')
+  .description('Site health monitor — detect changes between crawl runs')
+  .option('--format <type>', 'Output format: brief or json', 'brief')
+  .action(async (project, opts) => {
+    const db = getDb();
+    const config = loadConfig(project);
+    if (!config) return;
+    const isJson = opts.format === 'json';
+
+    if (!isJson) {
+      printAttackHeader('Site Watch', project);
+    }
+
+    const { runWatch } = await import('./analyses/watch/index.js');
+
+    const result = runWatch(db, project, {
+      log: isJson ? () => {} : (msg) => console.log(chalk.gray(msg)),
+    });
+
+    if (!result.snapshot) {
+      if (isJson) console.log(JSON.stringify({ ok: false, error: 'No crawled pages found' }));
+      return;
+    }
+
+    if (isJson) {
+      console.log(JSON.stringify({
+        snapshot: result.snapshot,
+        events: result.events,
+        healthScore: result.healthScore,
+        previousHealthScore: result.previousHealthScore,
+        trend: result.trend,
+        isBaseline: result.isBaseline,
+      }, null, 2));
+    } else {
+      // ── Health Score ──
+      const score = result.healthScore;
+      const scoreFmt = score >= 80 ? chalk.bold.green(score + '/100')
+        : score >= 60 ? chalk.bold.yellow(score + '/100')
+        : chalk.bold.red(score + '/100');
+
+      console.log('');
+      console.log(`    Health Score:  ${scoreFmt}`);
+
+      if (result.previousHealthScore !== null) {
+        const t = result.trend;
+        const trendStr = t > 0 ? chalk.green(`▲ +${t}`)
+          : t < 0 ? chalk.red(`▼ ${t}`)
+          : chalk.gray('— unchanged');
+        console.log(`                  ${trendStr} (was ${result.previousHealthScore}/100)`);
+      }
+      console.log('');
+
+      if (result.isBaseline) {
+        console.log(chalk.bold.green(`    ✅ Baseline captured — ${result.snapshot.total_pages} pages`));
+        console.log(chalk.gray('    Run another crawl to see changes.\n'));
+      } else {
+        // ── Severity summary ──
+        const { errors_count: e, warnings_count: w, notices_count: n } = result.snapshot;
+        const prev = result.previousHealthScore !== null ? {
+          e: 0, w: 0, n: 0, // We'll compute deltas from events
+        } : null;
+
+        console.log(`    ${chalk.red('●')} Critical:  ${e}`);
+        console.log(`    ${chalk.yellow('●')} Warning:   ${w}`);
+        console.log(`    ${chalk.gray('●')} Notice:    ${n}`);
+        console.log('');
+
+        // ── Event list ──
+        if (result.events.length) {
+          console.log(chalk.bold('    What\'s New:'));
+          console.log('');
+
+          const shown = result.events.slice(0, 25);
+          for (const ev of shown) {
+            const icon = ev.severity === 'critical' ? chalk.red('●')
+              : ev.severity === 'warning' ? chalk.yellow('▲')
+              : chalk.gray('○');
+            const sev = ev.severity.toUpperCase().padEnd(8);
+            const path = ev.url.replace(/https?:\/\/[^/]+/, '') || '/';
+
+            let desc = '';
+            switch (ev.event_type) {
+              case 'page_added':          desc = `${path} — new page`; break;
+              case 'page_removed':        desc = `${path} — removed`; break;
+              case 'new_error':           desc = `${path} → ${ev.new_value} (was ${ev.old_value})`; break;
+              case 'status_changed':      desc = `${path} status ${ev.old_value} → ${ev.new_value}`; break;
+              case 'title_changed':       desc = `${path} title: "${(ev.old_value || '').slice(0, 30)}" → "${(ev.new_value || '').slice(0, 30)}"`; break;
+              case 'h1_changed':          desc = `${path} H1 changed`; break;
+              case 'meta_desc_changed':   desc = `${path} meta description changed`; break;
+              case 'word_count_changed':  desc = `${path} word count ${ev.old_value} → ${ev.new_value}`; break;
+              case 'indexability_changed': desc = `${path} became ${ev.new_value}`; break;
+              case 'content_changed':     desc = `${path} content updated`; break;
+              default:                    desc = `${path} — ${ev.event_type.replace(/_/g, ' ')}`;
+            }
+
+            console.log(`    ${icon} ${chalk.dim(sev)} ${desc}`);
+          }
+
+          if (result.events.length > 25) {
+            console.log(chalk.gray(`    ... and ${result.events.length - 25} more`));
+          }
+          console.log('');
+        } else {
+          console.log(chalk.green('    ✅ No changes detected since last crawl.\n'));
+        }
+      }
+    }
+
+    // ── Regenerate dashboard ──
+    if (!isJson) {
+      try {
+        const configs = loadAllConfigs();
+        generateMultiDashboard(db, configs);
+        console.log(chalk.green('  ✅ Dashboard updated with Site Watch card\n'));
+      } catch (e) {
+        console.log(chalk.gray(`  (Dashboard not updated: ${e.message})\n`));
+      }
+    }
+  });
+
 // ── GAP INTEL ────────────────────────────────────────────────────────────
 
 program