seo-intel 1.4.0 → 1.4.2

package/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## 1.4.2 (2026-04-05)
+
+### New Feature: Site Watch
+- `seo-intel watch <project>` — detect changes between crawl runs and track site health
+- Health Score (0-100) based on page errors, missing titles, missing H1s
+- Diff engine detects 10 event types: new/removed pages, status changes, title/H1/meta changes, word count shifts, indexability flips, content updates
+- Events classified by severity: critical, warning, notice — with trend arrows
+- Auto-runs after every crawl with a one-liner summary
+- Dashboard card: health score gauge, severity counts with deltas, "What's New" event table
+- Significant changes (critical/warning) feed into Intelligence Ledger as `site_watch` insights
+- Available via CLI, dashboard terminal, and froggo.js API
+- Free tier — no license required
+
+## 1.4.1 (2026-04-03)
+
+### Fixes
+- **CLI JSON output** — all 11 commands now produce clean JSON with zero chalk/ANSI leakage
+- **Brief `--format json`** — full rich data (keyword gaps, schema gaps, actions) instead of lean subset
+- **Templates `--format json`** — suppressed chalk header and log output in JSON mode
+- **JS-Delta `--format json`** — suppressed per-page progress chalk in JSON mode
+
+### Agent Integration
+- Model selection hints (`modelHint`, `modelNote`) on extract, gap-intel, blog-draft capabilities
+- AGENT_GUIDE.md — added Model Selection Guidance table (light-local vs cloud-medium per phase)
+- GitHub Releases now auto-created on tag push via CI
+
 ## 1.4.0 (2026-04-03)
 
 ### New Feature: Gap Intelligence
@@ -17,6 +43,19 @@
 - Qwen models remain fully supported as alternatives
 - Setup wizard, model recommendations, and VRAM tiers updated for Gemma 4
 
+### Agent-Ready JSON Output
+- All 11 analysis commands support `--format json` for clean, parseable output
+- JSON output is chalk-free — no ANSI escape codes mixed into structured data
+- Commands: shallow, decay, headings-audit, orphans, entities, schemas, friction, brief, velocity, templates, js-delta
+
+### Programmatic API (`seo-intel/froggo`)
+- Unified agent runner: `run(command, project, opts)` returns `{ ok, command, project, timestamp, data }`
+- 18 capabilities with machine-readable manifest (inputs, outputs, dependencies, tier)
+- Pipeline dependency graph for orchestration
+- Model selection hints per capability (light-local vs cloud-medium)
+- Deep imports: `seo-intel/aeo`, `seo-intel/crawler`, `seo-intel/db`, etc.
+- Agent Guide (`AGENT_GUIDE.md`) with orchestration patterns and model guidance
+
 ### Server
 - Added `gap-intel` to terminal command whitelist
 - Forward `--vs`, `--type`, `--limit`, `--raw`, `--out` params from dashboard to CLI

package/analyses/watch/diff.js

@@ -0,0 +1,158 @@
+/**
+ * Site Watch — Diff Engine
+ *
+ * Pure function: compares two page snapshots and returns change events.
+ * Zero I/O, zero side effects — deterministic and testable.
+ */
+
+/**
+ * @param {object[]} currentPages  - { url, status_code, title, h1, meta_desc, word_count, is_indexable, content_hash }
+ * @param {object[]} previousPages - same shape, from previous snapshot
+ * @returns {object[]} Array of { event_type, severity, url, old_value, new_value, details }
+ */
+export function diffPages(currentPages, previousPages) {
+  const events = [];
+
+  const currentMap = new Map(currentPages.map(p => [p.url, p]));
+  const previousMap = new Map(previousPages.map(p => [p.url, p]));
+
+  // ── Pages added ──────────────────────────────────────────────────────────
+  for (const [url, page] of currentMap) {
+    if (!previousMap.has(url)) {
+      events.push({
+        event_type: 'page_added',
+        severity: 'notice',
+        url,
+        old_value: null,
+        new_value: String(page.status_code || 200),
+        details: JSON.stringify({ title: page.title, word_count: page.word_count }),
+      });
+    }
+  }
+
+  // ── Pages removed ────────────────────────────────────────────────────────
+  for (const [url, page] of previousMap) {
+    if (!currentMap.has(url)) {
+      events.push({
+        event_type: 'page_removed',
+        severity: 'warning',
+        url,
+        old_value: String(page.status_code || 200),
+        new_value: null,
+        details: JSON.stringify({ title: page.title, word_count: page.word_count }),
+      });
+    }
+  }
+
+  // ── Per-page field comparisons ───────────────────────────────────────────
+  for (const [url, curr] of currentMap) {
+    const prev = previousMap.get(url);
+    if (!prev) continue;
+
+    // Status code change
+    if (curr.status_code !== prev.status_code) {
+      const isNewError = prev.status_code < 400 && curr.status_code >= 400;
+      const isRecovery = prev.status_code >= 400 && curr.status_code < 400;
+      const severity = isNewError ? 'critical'
+        : curr.status_code >= 400 ? 'critical'
+        : isRecovery ? 'notice'
+        : 'warning';
+
+      events.push({
+        event_type: isNewError ? 'new_error' : 'status_changed',
+        severity,
+        url,
+        old_value: String(prev.status_code),
+        new_value: String(curr.status_code),
+        details: null,
+      });
+    }
+
+    // Title change
+    if (normalise(curr.title) !== normalise(prev.title)) {
+      events.push({
+        event_type: 'title_changed',
+        severity: 'notice',
+        url,
+        old_value: prev.title || '',
+        new_value: curr.title || '',
+        details: null,
+      });
+    }
+
+    // H1 change
+    if (normalise(curr.h1) !== normalise(prev.h1)) {
+      events.push({
+        event_type: 'h1_changed',
+        severity: 'notice',
+        url,
+        old_value: prev.h1 || '',
+        new_value: curr.h1 || '',
+        details: null,
+      });
+    }
+
+    // Meta description change
+    if (normalise(curr.meta_desc) !== normalise(prev.meta_desc)) {
+      events.push({
+        event_type: 'meta_desc_changed',
+        severity: 'notice',
+        url,
+        old_value: prev.meta_desc || '',
+        new_value: curr.meta_desc || '',
+        details: null,
+      });
+    }
+
+    // Word count significant change (>20%)
+    const prevWc = prev.word_count || 0;
+    const currWc = curr.word_count || 0;
+    if (prevWc > 0 && Math.abs(currWc - prevWc) / prevWc > 0.2) {
+      events.push({
+        event_type: 'word_count_changed',
+        severity: 'notice',
+        url,
+        old_value: String(prevWc),
+        new_value: String(currWc),
+        details: JSON.stringify({ delta_pct: Math.round(((currWc - prevWc) / prevWc) * 100) }),
+      });
+    }
+
+    // Indexability change
+    const prevIdx = prev.is_indexable ? 1 : 0;
+    const currIdx = curr.is_indexable ? 1 : 0;
+    if (currIdx !== prevIdx) {
+      events.push({
+        event_type: 'indexability_changed',
+        severity: currIdx === 0 ? 'critical' : 'notice',
+        url,
+        old_value: prevIdx ? 'indexable' : 'non-indexable',
+        new_value: currIdx ? 'indexable' : 'non-indexable',
+        details: null,
+      });
+    }
+
+    // Content hash change (body text changed)
+    if (curr.content_hash && prev.content_hash && curr.content_hash !== prev.content_hash) {
+      events.push({
+        event_type: 'content_changed',
+        severity: 'notice',
+        url,
+        old_value: prev.content_hash?.slice(0, 8) || '',
+        new_value: curr.content_hash?.slice(0, 8) || '',
+        details: null,
+      });
+    }
+  }
+
+  // Sort: critical first, then warning, then notice
+  const severityOrder = { critical: 0, warning: 1, notice: 2 };
+  events.sort((a, b) => (severityOrder[a.severity] ?? 9) - (severityOrder[b.severity] ?? 9));
+
+  return events;
+}
+
+/** Normalise a string for comparison (null-safe, trimmed, lowercased). */
+function normalise(s) {
+  return (s || '').trim().toLowerCase();
+}

package/analyses/watch/health.js

@@ -0,0 +1,78 @@
+/**
+ * Site Watch — Health Score Calculator
+ *
+ * Pure function: evaluates site health from current page data.
+ * Zero I/O, zero side effects.
+ */
+
+/**
+ * @param {object[]} pages - { url, status_code, title, h1, meta_desc, word_count, is_indexable }
+ * @returns {object} { score, errors, warnings, notices, details }
+ */
+export function calculateHealthScore(pages) {
+  if (!pages.length) return { score: 100, errors: 0, warnings: 0, notices: 0, details: [] };
+
+  const details = [];
+  let errorPages = 0;
+
+  // Track duplicates
+  const titleCounts = new Map();
+  for (const p of pages) {
+    if (p.title && p.status_code < 400) {
+      const t = p.title.trim().toLowerCase();
+      titleCounts.set(t, (titleCounts.get(t) || 0) + 1);
+    }
+  }
+
+  for (const p of pages) {
+    let hasError = false;
+
+    // ── Errors (reduce health score) ───────────────────────────────────────
+    if (p.status_code >= 400) {
+      details.push({ url: p.url, severity: 'error', issue: `${p.status_code} error` });
+      hasError = true;
+    }
+
+    if (p.status_code < 400 && (!p.title || !p.title.trim())) {
+      details.push({ url: p.url, severity: 'error', issue: 'Missing title' });
+      hasError = true;
+    }
+
+    if (p.status_code < 400 && (!p.h1 || !p.h1.trim())) {
+      details.push({ url: p.url, severity: 'error', issue: 'Missing H1' });
+      hasError = true;
+    }
+
+    if (hasError) errorPages++;
+
+    // ── Warnings (tracked, don't reduce score) ────────────────────────────
+    if (p.status_code >= 300 && p.status_code < 400) {
+      details.push({ url: p.url, severity: 'warning', issue: `${p.status_code} redirect` });
+    }
+
+    if (p.status_code < 400 && (!p.meta_desc || !p.meta_desc.trim())) {
+      details.push({ url: p.url, severity: 'warning', issue: 'Missing meta description' });
+    }
+
+    if (p.title && p.status_code < 400) {
+      const t = p.title.trim().toLowerCase();
+      if (titleCounts.get(t) > 1) {
+        details.push({ url: p.url, severity: 'warning', issue: 'Duplicate title' });
+      }
+    }
+
+    // ── Notices ────────────────────────────────────────────────────────────
+    if (p.status_code < 400 && (p.word_count || 0) < 100 && (p.word_count || 0) > 0) {
+      details.push({ url: p.url, severity: 'notice', issue: 'Thin content (<100 words)' });
+    }
+  }
+
+  const errors = details.filter(d => d.severity === 'error').length;
+  const warnings = details.filter(d => d.severity === 'warning').length;
+  const notices = details.filter(d => d.severity === 'notice').length;
+
+  // Health score = % of pages without errors
+  const score = Math.round(((pages.length - errorPages) / pages.length) * 100);
+
+  return { score, errors, warnings, notices, details };
+}

package/analyses/watch/index.js

@@ -0,0 +1,215 @@
+/**
+ * Site Watch — Orchestrator
+ *
+ * Gathers current page state, diffs against previous snapshot,
+ * persists results, and feeds significant changes into the Intelligence Ledger.
+ */
+
+import { diffPages } from './diff.js';
+import { calculateHealthScore } from './health.js';
+import {
+  getLatestWatchSnapshot,
+  getWatchPageStates,
+  getWatchEvents,
+  getWatchHistory,
+} from '../../db/db.js';
+
+// ═══════════════════════════════════════════════════════════════════════════
+// MAIN RUNNER
+// ═══════════════════════════════════════════════════════════════════════════
+
+/**
+ * Run site watch analysis for a project.
+ *
+ * @param {import('node:sqlite').DatabaseSync} db
+ * @param {string} project
+ * @param {object} [opts] - { log: function }
+ * @returns {object} { snapshot, events, healthScore, previousHealthScore, trend, isBaseline }
+ */
+export function runWatch(db, project, opts = {}) {
+  const log = opts.log || console.log;
+
+  // ── Gather current page state ──────────────────────────────────────────
+  const currentPages = db.prepare(`
+    SELECT
+      p.url, p.status_code, p.title, p.meta_desc, p.word_count,
+      p.is_indexable, p.content_hash,
+      (SELECT text FROM headings WHERE page_id = p.id AND level = 1 ORDER BY id LIMIT 1) as h1
+    FROM pages p
+    JOIN domains d ON d.id = p.domain_id
+    WHERE d.project = ? AND d.role IN ('target', 'owned')
+    ORDER BY p.url
+  `).all(project);
+
+  if (!currentPages.length) {
+    log('  No crawled pages found. Run crawl first.');
+    return { snapshot: null, events: [], healthScore: 0, previousHealthScore: null, trend: 0, isBaseline: false };
+  }
+
+  // ── Health score ───────────────────────────────────────────────────────
+  const health = calculateHealthScore(currentPages);
+
+  // ── Load previous snapshot ─────────────────────────────────────────────
+  const prevSnapshot = getLatestWatchSnapshot(db, project);
+  let events = [];
+  let isBaseline = false;
+
+  if (prevSnapshot) {
+    const prevPages = getWatchPageStates(db, prevSnapshot.id);
+    events = diffPages(currentPages, prevPages);
+    log(`  Compared ${currentPages.length} pages against snapshot from ${new Date(prevSnapshot.created_at).toLocaleDateString()}`);
+  } else {
+    isBaseline = true;
+    log(`  Baseline snapshot — ${currentPages.length} pages captured`);
+  }
+
+  // ── Persist new snapshot ───────────────────────────────────────────────
+  const now = Date.now();
+  const criticalCount = events.filter(e => e.severity === 'critical').length;
+  const warningCount = events.filter(e => e.severity === 'warning').length;
+  const noticeCount = events.filter(e => e.severity === 'notice').length;
+
+  const snapshotResult = db.prepare(`
+    INSERT INTO watch_snapshots (project, created_at, total_pages, health_score, errors_count, warnings_count, notices_count)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+  `).run(project, now, currentPages.length, health.score, criticalCount, warningCount, noticeCount);
+
+  const snapshotId = Number(db.prepare('SELECT last_insert_rowid() as id').get().id);
+
+  // ── Persist page states ────────────────────────────────────────────────
+  const stateStmt = db.prepare(`
+    INSERT INTO watch_page_states (snapshot_id, url, status_code, title, h1, meta_desc, word_count, is_indexable, content_hash)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+
+  db.exec('BEGIN');
+  try {
+    for (const p of currentPages) {
+      stateStmt.run(snapshotId, p.url, p.status_code, p.title, p.h1, p.meta_desc, p.word_count, p.is_indexable ? 1 : 0, p.content_hash);
+    }
+    db.exec('COMMIT');
+  } catch (e) {
+    db.exec('ROLLBACK');
+    throw e;
+  }
+
+  // ── Persist events ─────────────────────────────────────────────────────
+  if (events.length) {
+    const eventStmt = db.prepare(`
+      INSERT INTO watch_events (snapshot_id, event_type, severity, url, old_value, new_value, details)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    db.exec('BEGIN');
+    try {
+      for (const e of events) {
+        eventStmt.run(snapshotId, e.event_type, e.severity, e.url, e.old_value, e.new_value, e.details);
+      }
+      db.exec('COMMIT');
+    } catch (e) {
+      db.exec('ROLLBACK');
+      throw e;
+    }
+  }
+
+  // ── Feed Intelligence Ledger (critical + warning only) ─────────────────
+  const significant = events.filter(e => e.severity === 'critical' || e.severity === 'warning');
+  if (significant.length) {
+    _upsertWatchInsights(db, project, significant, now);
+  }
+
+  const previousHealthScore = prevSnapshot?.health_score ?? null;
+  const trend = previousHealthScore !== null ? health.score - previousHealthScore : 0;
+
+  const snapshot = {
+    id: snapshotId,
+    project,
+    created_at: now,
+    total_pages: currentPages.length,
+    health_score: health.score,
+    errors_count: criticalCount,
+    warnings_count: warningCount,
+    notices_count: noticeCount,
+  };
+
+  return {
+    snapshot,
+    events,
+    healthScore: health.score,
+    healthDetails: health,
+    previousHealthScore,
+    trend,
+    isBaseline,
+  };
+}
+
+// ═══════════════════════════════════════════════════════════════════════════
+// DASHBOARD DATA
+// ═══════════════════════════════════════════════════════════════════════════
+
+/**
+ * Get watch data for dashboard rendering.
+ *
+ * @param {import('node:sqlite').DatabaseSync} db
+ * @param {string} project
+ * @returns {object|null} { current, previous, events, trend }
+ */
+export function getWatchData(db, project) {
+  const history = getWatchHistory(db, project, 2);
+  if (!history.length) return null;
+
+  const current = history[0];
+  const previous = history[1] || null;
+  const events = getWatchEvents(db, current.id);
+  const trend = previous ? current.health_score - previous.health_score : 0;
+
+  return { current, previous, events, trend };
+}
+
+// ═══════════════════════════════════════════════════════════════════════════
+// INTERNAL HELPERS
+// ═══════════════════════════════════════════════════════════════════════════
+
+function _upsertWatchInsights(db, project, events, timestamp) {
+  const upsertStmt = db.prepare(`
+    INSERT INTO insights (project, type, status, fingerprint, first_seen, last_seen, source_analysis_id, data)
+    VALUES (?, 'site_watch', 'active', ?, ?, ?, NULL, ?)
+    ON CONFLICT(project, type, fingerprint) DO UPDATE SET
+      last_seen = excluded.last_seen,
+      data = excluded.data
+  `);
+
+  db.exec('BEGIN');
+  try {
+    for (const e of events) {
+      const raw = `${e.url || ''}::${e.event_type || ''}`;
+      const fp = raw.toLowerCase().replace(/[^a-z0-9\s]/g, '').replace(/\s+/g, ' ').trim();
+      if (!fp) continue;
+
+      const data = {
+        url: e.url,
+        event_type: e.event_type,
+        severity: e.severity,
+        old_value: e.old_value,
+        new_value: e.new_value,
+        summary: _eventSummary(e),
+      };
+
+      upsertStmt.run(project, fp, timestamp, timestamp, JSON.stringify(data));
+    }
+    db.exec('COMMIT');
+  } catch (err) {
+    db.exec('ROLLBACK');
+    console.error('[watch] insight upsert failed:', err.message);
+  }
+}
+
+function _eventSummary(e) {
+  switch (e.event_type) {
+    case 'new_error':            return `${e.url} returned ${e.new_value} (was ${e.old_value})`;
+    case 'status_changed':       return `${e.url} status ${e.old_value} → ${e.new_value}`;
+    case 'page_removed':         return `${e.url} disappeared from crawl`;
+    case 'indexability_changed':  return `${e.url} became ${e.new_value}`;
+    default:                     return `${e.event_type.replace(/_/g, ' ')} on ${e.url}`;
+  }
+}