seo-intel 1.4.1 → 1.4.3

package/reports/generate-html.js

@@ -20,16 +20,23 @@ import { loadGscData } from './gsc-loader.js';
 import { isPro } from '../lib/license.js';
 import { getActiveInsights } from '../db/db.js';
 import { getCitabilityScores } from '../analyses/aeo/index.js';
+import { getWatchData } from '../analyses/watch/index.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
-/**
- * Generate HTML dashboard from database
- * @param {import('node:sqlite').DatabaseSync} db - SQLite database
- * @param {string} project - Project name (e.g., 'mysite')
- * @param {object} config - Project config
- * @returns {string} Path to generated HTML file
- */
+/** Build per-card export dropdown HTML */
+function cardExportHtml(section, project) {
+  return `<div class="card-export" data-project="${project}" data-section="${section}">
+    <button class="card-export-btn" title="Export"><i class="fa-solid fa-download"></i></button>
+    <div class="card-export-dropdown">
+      <button data-fmt="md"><i class="fa-solid fa-file-lines"></i> Markdown</button>
+      <button data-fmt="json"><i class="fa-solid fa-code"></i> JSON</button>
+      <button data-fmt="csv"><i class="fa-solid fa-table"></i> CSV</button>
+      <button data-fmt="zip"><i class="fa-solid fa-file-zipper"></i> ZIP (all)</button>
+    </div>
+  </div>`;
+}
+
 /**
  * Gather all dashboard data for a single project
  */
@@ -108,6 +115,10 @@ function gatherProjectData(db, project, config) {
   let citabilityData = null;
   try { citabilityData = getCitabilityScores(db, project); } catch { /* table may not exist yet */ }
 
+  // Site Watch data
+  let watchData = null;
+  try { watchData = getWatchData(db, project); } catch { /* tables may not exist yet */ }
+
   // Extraction status
   const extractionStatus = getExtractionStatus(db, project, config);
 
@@ -131,7 +142,7 @@ function gatherProjectData(db, project, config) {
     ctaLandscape, entityTopicMap, schemaBreakdown,
     gravityMap, contentTerrain, keywordVenn, performanceBubbles,
     headingFlow, territoryTreemap, topicClusters, linkDna, linkRadarPulse,
-    keywordsReport, extractionStatus, gscData, domainArch, gscInsights, citabilityData,
+    keywordsReport, extractionStatus, gscData, domainArch, gscInsights, citabilityData, watchData,
   };
 
   // Rollback the owned→target merge so the actual DB is unchanged
@@ -194,7 +205,7 @@ function buildHtmlTemplate(data, opts = {}) {
     ctaLandscape, entityTopicMap, schemaBreakdown,
     gravityMap, contentTerrain, keywordVenn, performanceBubbles,
     headingFlow, territoryTreemap, topicClusters, linkDna, linkRadarPulse,
-    keywordsReport, extractionStatus, gscData, domainArch, gscInsights, citabilityData,
+    keywordsReport, extractionStatus, gscData, domainArch, gscInsights, citabilityData, watchData,
   } = data;
 
   const totalPages = domains.reduce((sum, d) => sum + d.page_count, 0);
@@ -414,6 +425,32 @@ function buildHtmlTemplate(data, opts = {}) {
     .card.full-width {
       grid-column: 1 / -1;
     }
+    .card { position: relative; }
+    .card-export {
+      position: absolute; top: 10px; right: 10px; z-index: 5;
+    }
+    .card-export-btn {
+      background: var(--bg-elevated); border: 1px solid var(--border-subtle);
+      color: var(--text-muted); cursor: pointer; border-radius: var(--radius);
+      width: 28px; height: 28px; display: flex; align-items: center; justify-content: center;
+      font-size: 0.7rem; transition: all .15s;
+    }
+    .card-export-btn:hover { color: var(--accent-gold); border-color: var(--accent-gold); }
+    .card-export-dropdown {
+      display: none; position: absolute; right: 0; top: 32px;
+      background: var(--bg-card); border: 1px solid var(--border-card);
+      border-radius: var(--radius); min-width: 150px; padding: 4px 0;
+      box-shadow: 0 8px 24px rgba(0,0,0,.4);
+    }
+    .card-export.open .card-export-dropdown { display: block; }
+    .card-export-dropdown button {
+      display: flex; align-items: center; gap: 8px; width: 100%;
+      background: none; border: none; color: var(--text-secondary);
+      padding: 7px 14px; font-size: 0.72rem; cursor: pointer; text-align: left;
+      font-family: var(--font-body);
+    }
+    .card-export-dropdown button:hover { background: var(--bg-elevated); color: var(--text-primary); }
+    .card-export-dropdown button i { width: 14px; text-align: center; color: var(--text-muted); font-size: 0.65rem; }
 
     /* ─── Extraction Status Bar ─────────────────────────────────────────── */
     .extraction-status {
@@ -1792,6 +1829,9 @@ function buildHtmlTemplate(data, opts = {}) {
         Crawled <strong>${totalPages}</strong> pages — no major structural issues detected.
       </div>`}
 
+      <!-- SITE WATCH -->
+      ${watchData?.current ? buildWatchCard(watchData, escapeHtml, project) : ''}
+
       <!-- PAGE INVENTORY -->
       <div class="card" style="margin-bottom:16px;">
         <h2><span class="icon"><i class="fa-solid fa-table-list"></i></span> Page Inventory — ${targetDomain}</h2>
@@ -2149,6 +2189,12 @@ function buildHtmlTemplate(data, opts = {}) {
         </div>
         <button class="export-btn" data-export-cmd="aeo" data-export-project="${project}"><i class="fa-solid fa-robot"></i> AI Citability Audit</button>
       </div>
+      <div class="export-sidebar-header" style="margin-top:12px;">
+        <i class="fa-solid fa-download"></i> Download
+      </div>
+      <div class="export-sidebar-btns">
+        <button class="export-btn download-all-btn" data-project="${project}"><i class="fa-solid fa-file-zipper"></i> Download All Reports (ZIP)</button>
+      </div>
       <div style="position:relative;">
         <div id="exportSaveStatus${suffix}" style="display:none;padding:4px 10px;font-size:.6rem;color:var(--color-success);background:rgba(80,200,120,0.06);border-bottom:1px solid rgba(80,200,120,0.15);font-family:'SF Mono',monospace;">
           <i class="fa-solid fa-check" style="margin-right:4px;"></i><span></span>
@@ -2508,6 +2554,49 @@ function buildHtmlTemplate(data, opts = {}) {
       });
     }
 
+    // ── Card export dropdowns (global — run once, capture phase) ──
+    if (!window._cardExportBound) {
+      window._cardExportBound = true;
+      document.addEventListener('click', function(e) {
+        var btn = e.target.closest('.card-export-btn');
+        var fmtBtn = e.target.closest('[data-fmt]');
+        if (btn) {
+          var wrap = btn.closest('.card-export');
+          if (wrap) {
+            e.stopImmediatePropagation();
+            var wasOpen = wrap.classList.contains('open');
+            document.querySelectorAll('.card-export.open').forEach(function(el) { el.classList.remove('open'); });
+            if (!wasOpen) wrap.classList.add('open');
+            return;
+          }
+        }
+        if (fmtBtn) {
+          var wrap2 = fmtBtn.closest('.card-export');
+          if (wrap2) {
+            e.stopImmediatePropagation();
+            wrap2.classList.remove('open');
+            var sec = wrap2.getAttribute('data-section');
+            var proj2 = wrap2.getAttribute('data-project');
+            var fmt = fmtBtn.getAttribute('data-fmt');
+            if (window.location.protocol.startsWith('http')) {
+              window.location = '/api/export/download?project=' + encodeURIComponent(proj2) + '&section=' + encodeURIComponent(sec) + '&format=' + encodeURIComponent(fmt);
+            }
+            return;
+          }
+        }
+        var dlBtn = e.target.closest('.download-all-btn');
+        if (dlBtn) {
+          var proj3 = dlBtn.getAttribute('data-project');
+          if (window.location.protocol.startsWith('http')) {
+            window.location = '/api/export/download?project=' + encodeURIComponent(proj3) + '&section=all&format=zip';
+          }
+          return;
+        }
+        // Outside click — close all open dropdowns
+        document.querySelectorAll('.card-export.open').forEach(function(el) { el.classList.remove('open'); });
+      }, true);
+    }
+
     // Input enter
     input.addEventListener('keydown', function(e) {
       if (e.key !== 'Enter') return;
@@ -2770,6 +2859,9 @@ function buildHtmlTemplate(data, opts = {}) {
     </div>`;
     })() : ''}
 
+    <!-- ═══ SITE WATCH ═══ -->
+    ${watchData?.current ? buildWatchCard(watchData, escapeHtml, project) : ''}
+
     <div class="section-divider">
       <div class="section-divider-line right"></div>
       <span class="section-divider-label"><i class="fa-solid fa-radar"></i> Competitive Landscape</span>
@@ -2824,6 +2916,7 @@ function buildHtmlTemplate(data, opts = {}) {
 
     <!-- ═══ HEADING DEPTH FLOW ═══ -->
     <div class="card full-width" id="heading-flow">
+      ${cardExportHtml('headings', project)}
       <h2><span class="icon"><i class="fa-solid fa-water"></i></span> Heading Depth Flow</h2>
       <canvas id="headingFlowCanvas${suffix}" width="1100" height="320"></canvas>
     </div>
@@ -2851,6 +2944,7 @@ function buildHtmlTemplate(data, opts = {}) {
     <!-- ═══ ENTITY TOPIC MAP ═══ -->
     ${pro && entityTopicMap.hasData ? `
     <div class="card full-width" id="entity-map">
+      ${cardExportHtml('insights', project)}
       <h2><span class="icon"><i class="fa-solid fa-map"></i></span> Entity Topic Map</h2>
       <div class="entity-map-grid">
         ${Object.entries(entityTopicMap.domainEntities).map(([domain, data]) => `
@@ -2875,6 +2969,7 @@ function buildHtmlTemplate(data, opts = {}) {
     <!-- ═══ KEYWORD BATTLEGROUND ═══ -->
     ${pro ? `
     <div class="card full-width" id="keyword-heatmap">
+      ${cardExportHtml('keywords', project)}
       <h2><span class="icon"><i class="fa-solid fa-shield-halved"></i></span> Keyword Battleground</h2>
       ${keywordHeatmap.keywords.length ? `
       <div class="table-wrapper">
@@ -2919,6 +3014,7 @@ function buildHtmlTemplate(data, opts = {}) {
 
     <!-- ═══ TECHNICAL SEO SCORECARD ═══ -->
     <div class="card full-width" id="technical-seo">
+      ${cardExportHtml('technical', project)}
       <h2><span class="icon"><i class="fa-solid fa-gear"></i></span> Technical SEO Scorecard</h2>
       <div class="scorecard-grid">
         ${technicalScores.map(ts => {
@@ -2987,6 +3083,7 @@ function buildHtmlTemplate(data, opts = {}) {
     <!-- ═══ TECHNICAL SEO GAPS ═══ -->
     ${pro && latestAnalysis?.technical_gaps?.length ? `
     <div class="card full-width" id="technical-gaps">
+      ${cardExportHtml('technical', project)}
       <h2><span class="icon"><i class="fa-solid fa-wrench"></i></span> Technical SEO Gaps</h2>
       <div class="analysis-table-wrap">
         <table class="analysis-table">
@@ -3015,6 +3112,7 @@ function buildHtmlTemplate(data, opts = {}) {
     <!-- ═══ QUICK WINS ═══ -->
     ${pro && latestAnalysis?.quick_wins?.length ? `
     <div class="card" id="quick-wins">
+      ${cardExportHtml('insights', project)}
       <h2><span class="icon"><i class="fa-solid fa-bolt"></i></span> Quick Wins</h2>
       <div class="analysis-table-wrap">
         <table class="analysis-table">
@@ -3037,6 +3135,7 @@ function buildHtmlTemplate(data, opts = {}) {
     <!-- ═══ NEW PAGES TO CREATE ═══ -->
     ${pro && latestAnalysis?.new_pages?.length ? `
     <div class="card" id="new-pages">
+      ${cardExportHtml('pages', project)}
       <h2><span class="icon"><i class="fa-solid fa-file-circle-plus"></i></span> New Pages to Create</h2>
       <div class="new-pages-grid" style="grid-template-columns: 1fr;">
         ${(latestAnalysis.new_pages).map(np => `
@@ -3089,6 +3188,7 @@ function buildHtmlTemplate(data, opts = {}) {
     <!-- ═══ CONTENT GAPS ═══ -->
     ${pro && latestAnalysis?.content_gaps?.length ? `
     <div class="card full-width" id="content-gaps">
+      ${cardExportHtml('insights', project)}
       <h2><span class="icon"><i class="fa-solid fa-magnifying-glass-minus"></i></span> Content Gaps</h2>
       <div class="insights-grid">
         ${(latestAnalysis.content_gaps).map(gap => `
@@ -3297,6 +3397,7 @@ function buildHtmlTemplate(data, opts = {}) {
     <!-- ═══ SCHEMA TYPE BREAKDOWN ═══ -->
     ${schemaBreakdown.hasData ? `
     <div class="card" id="schema-breakdown">
+      ${cardExportHtml('schemas', project)}
       <h2><span class="icon"><i class="fa-solid fa-code"></i></span> Schema Markup Breakdown</h2>
       <div class="table-wrapper">
         <table>
@@ -3323,6 +3424,7 @@ function buildHtmlTemplate(data, opts = {}) {
     <!-- ═══ TOP KEYWORDS ═══ -->
     ${pro ? `
     <div class="card" id="top-keywords">
+      ${cardExportHtml('keywords', project)}
       <h2><span class="icon"><i class="fa-solid fa-key"></i></span> Top Keywords (${targetDomain})</h2>
       ${keywords.length ? `
       <div class="table-wrapper" style="max-height: 400px; overflow-y: auto;">
@@ -3358,6 +3460,7 @@ function buildHtmlTemplate(data, opts = {}) {
 
     <!-- ═══ INTERNAL LINK ANALYSIS ═══ -->
     <div class="card" id="internal-links">
+      ${cardExportHtml('links', project)}
       <h2><span class="icon"><i class="fa-solid fa-link"></i></span> Internal Link Analysis</h2>
       <div class="stat-row">
         <div class="stat-box">
@@ -3396,7 +3499,7 @@ function buildHtmlTemplate(data, opts = {}) {
     </div>` : ''}
 
     <!-- ═══ AEO / AI CITABILITY AUDIT ═══ -->
-    ${pro && citabilityData?.length ? buildAeoCard(citabilityData, escapeHtml) : ''}
+    ${pro && citabilityData?.length ? buildAeoCard(citabilityData, escapeHtml, project) : ''}
 
     <!-- ═══ LONG-TAIL OPPORTUNITIES ═══ -->
     ${pro && latestAnalysis?.long_tails?.length ? `
@@ -5221,7 +5324,7 @@ function buildMultiHtmlTemplate(allProjectData) {
 
 // ─── AEO Card Builder ────────────────────────────────────────────────────────
 
-function buildAeoCard(citabilityData, escapeHtml) {
+function buildAeoCard(citabilityData, escapeHtml, project) {
   const targetScores = citabilityData.filter(s => s.role === 'target' || s.role === 'owned');
   const compScores = citabilityData.filter(s => s.role === 'competitor');
   if (!targetScores.length) return '';
@@ -5285,6 +5388,7 @@ function buildAeoCard(citabilityData, escapeHtml) {
 
   return `
     <div class="card full-width" id="aeo-citability">
+      ${cardExportHtml('aeo', project)}
       <h2><span class="icon"><i class="fa-solid fa-robot"></i></span> AI Citability Audit</h2>
       <div class="ki-stat-bar">
         <div class="ki-stat"><span class="ki-stat-number" style="color:${scoreColor(avgTarget)}">${avgTarget}</span><span class="ki-stat-label">Target Avg</span></div>
@@ -6870,3 +6974,141 @@ function getGscInsights(gscData, db, project) {
 
   return insights.length ? insights : null;
 }
+
+// ═══════════════════════════════════════════════════════════════════════════
+// SITE WATCH CARD
+// ═══════════════════════════════════════════════════════════════════════════
+
+function buildWatchCard(watchData, escapeHtml, project) {
+  const { current, previous, events, trend } = watchData;
+  const score = current.health_score ?? 0;
+  const scoreColor = score >= 80 ? 'var(--color-success)' : score >= 60 ? 'var(--color-warning)' : 'var(--color-danger)';
+  const trendArrow = trend > 0 ? `<span style="color:var(--color-success);">▲ +${trend}</span>`
+    : trend < 0 ? `<span style="color:var(--color-danger);">▼ ${trend}</span>`
+    : previous ? '<span style="color:var(--text-muted);">— unchanged</span>' : '';
+  const prevText = previous ? ` <span style="color:var(--text-muted);font-size:0.7rem;">(was ${previous.health_score}/100)</span>` : '';
+
+  const timestamp = new Date(current.created_at).toLocaleDateString('en-US', {
+    month: 'short', day: 'numeric', year: 'numeric', hour: '2-digit', minute: '2-digit',
+  });
+
+  // Severity counts with deltas
+  const eCurr = current.errors_count || 0;
+  const wCurr = current.warnings_count || 0;
+  const nCurr = current.notices_count || 0;
+  const ePrev = previous?.errors_count || 0;
+  const wPrev = previous?.warnings_count || 0;
+  const nPrev = previous?.notices_count || 0;
+
+  function delta(curr, prev) {
+    if (!previous) return '';
+    const d = curr - prev;
+    if (d > 0) return `<span style="color:var(--color-danger);font-size:0.65rem;"> +${d}</span>`;
+    if (d < 0) return `<span style="color:var(--color-success);font-size:0.65rem;"> ${d}</span>`;
+    return '';
+  }
+
+  // Event type labels
+  const eventLabel = (type) => {
+    const labels = {
+      page_added: 'New page',
+      page_removed: 'Page removed',
+      new_error: 'New error',
+      status_changed: 'Status changed',
+      title_changed: 'Title changed',
+      h1_changed: 'H1 changed',
+      meta_desc_changed: 'Meta description changed',
+      word_count_changed: 'Word count changed',
+      indexability_changed: 'Indexability changed',
+      content_changed: 'Content updated',
+    };
+    return labels[type] || type.replace(/_/g, ' ');
+  };
+
+  const severityIcon = (s) => {
+    if (s === 'critical') return '<span style="color:var(--color-danger);">●</span>';
+    if (s === 'warning') return '<span style="color:var(--color-warning);">▲</span>';
+    return '<span style="color:var(--text-muted);">○</span>';
+  };
+
+  const shownEvents = events.slice(0, 20);
+
+  return `
+  <div class="card" style="margin-bottom:16px;">
+    ${cardExportHtml('watch', project)}
+    <h2><span class="icon"><i class="fa-solid fa-eye"></i></span> Site Watch</h2>
+
+    <!-- Health Score + Summary -->
+    <div style="display:flex;align-items:center;gap:24px;margin-bottom:16px;">
+      <div style="text-align:center;">
+        <div style="font-size:2rem;font-weight:700;color:${scoreColor};line-height:1;">${score}</div>
+        <div style="font-size:0.65rem;color:var(--text-muted);">Health Score</div>
+      </div>
+      <div style="font-size:0.75rem;">
+        <div>${trendArrow}${prevText}</div>
+        <div style="color:var(--text-muted);font-size:0.65rem;margin-top:4px;">${current.total_pages} pages · ${timestamp}</div>
+      </div>
+      <div style="display:flex;gap:16px;margin-left:auto;font-size:0.75rem;">
+        <div style="text-align:center;">
+          <div style="font-size:1.1rem;font-weight:600;color:var(--color-danger);">${eCurr}</div>
+          <div style="font-size:0.6rem;color:var(--text-muted);">Critical${delta(eCurr, ePrev)}</div>
+        </div>
+        <div style="text-align:center;">
+          <div style="font-size:1.1rem;font-weight:600;color:var(--color-warning);">${wCurr}</div>
+          <div style="font-size:0.6rem;color:var(--text-muted);">Warning${delta(wCurr, wPrev)}</div>
+        </div>
+        <div style="text-align:center;">
+          <div style="font-size:1.1rem;font-weight:600;color:var(--text-secondary);">${nCurr}</div>
+          <div style="font-size:0.6rem;color:var(--text-muted);">Notice${delta(nCurr, nPrev)}</div>
+        </div>
+      </div>
+    </div>
+
+    ${shownEvents.length ? `
+    <!-- What's New -->
+    <div style="border-top:1px solid var(--border-card);padding-top:12px;">
+      <div style="font-size:0.72rem;font-weight:600;color:var(--text-primary);margin-bottom:8px;">What's New</div>
+      <div class="table-wrapper" style="max-height:320px;overflow-y:auto;">
+        <table>
+          <thead>
+            <tr>
+              <th style="width:30px;"></th>
+              <th>Type</th>
+              <th>URL</th>
+              <th>Change</th>
+            </tr>
+          </thead>
+          <tbody>
+            ${shownEvents.map(ev => {
+              const path = ev.url.replace(/^https?:\/\/[^/]+/, '') || '/';
+              let changeText = '';
+              if (ev.old_value && ev.new_value) {
+                changeText = `${escapeHtml((ev.old_value || '').slice(0, 40))} → ${escapeHtml((ev.new_value || '').slice(0, 40))}`;
+              } else if (ev.new_value) {
+                changeText = escapeHtml((ev.new_value || '').slice(0, 60));
+              } else if (ev.old_value) {
+                changeText = `was: ${escapeHtml((ev.old_value || '').slice(0, 60))}`;
+              }
+              return `<tr>
+                <td style="text-align:center;">${severityIcon(ev.severity)}</td>
+                <td style="font-size:0.7rem;white-space:nowrap;">${eventLabel(ev.event_type)}</td>
+                <td style="font-size:0.7rem;max-width:200px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="${escapeHtml(ev.url)}">${escapeHtml(path)}</td>
+                <td style="font-size:0.68rem;color:var(--text-secondary);max-width:250px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">${changeText}</td>
+              </tr>`;
+            }).join('')}
+          </tbody>
+        </table>
+      </div>
+      ${events.length > 20 ? `<div style="font-size:0.65rem;color:var(--text-muted);margin-top:4px;">Showing 20 of ${events.length} changes. Run <code>seo-intel watch</code> for full report.</div>` : ''}
+    </div>
+    ` : previous ? `
+    <div style="border-top:1px solid var(--border-card);padding-top:12px;font-size:0.75rem;color:var(--color-success);">
+      <i class="fa-solid fa-check-circle" style="margin-right:6px;"></i> No changes detected since last crawl.
+    </div>
+    ` : `
+    <div style="border-top:1px solid var(--border-card);padding-top:12px;font-size:0.75rem;color:var(--text-muted);">
+      <i class="fa-solid fa-info-circle" style="margin-right:6px;"></i> Baseline captured. Run another crawl to see changes.
+    </div>
+    `}
+  </div>`;
+}

package/reports/gsc-loader.js

@@ -1,7 +1,7 @@
 /**
  * Google Search Console CSV data loader
  * Reads GSC export folders from seo-intel/gsc/<project>*/
-import { readdirSync, readFileSync, existsSync } from 'fs';
+import { readdirSync, readFileSync, existsSync, statSync } from 'fs';
 import { join, dirname } from 'path';
 import { fileURLToPath } from 'url';
 
@@ -87,8 +87,18 @@ export function loadGscData(project) {
   );
   if (!folders.length) return null;
 
-  // Use most recent folder (alphabetically last)
-  const folder = join(GSC_DIR, folders.sort().pop());
+  // Use most recently modified matching folder.
+  // This avoids stale picks like "carbium-2" winning over a freshly uploaded "carbium".
+  const selectedFolder = [...folders]
+    .map(name => {
+      const path = join(GSC_DIR, name);
+      let mtimeMs = 0;
+      try { mtimeMs = statSync(path).mtimeMs; } catch { /* ignore */ }
+      return { name, path, mtimeMs };
+    })
+    .sort((a, b) => b.mtimeMs - a.mtimeMs || a.name.localeCompare(b.name))[0];
+
+  const folder = selectedFolder.path;
 
   function loadCSV(filename) {
     const filepath = join(folder, filename);
@@ -185,6 +195,6 @@ export function loadGscData(project) {
     countries,
     devices,
     summary: { totalClicks, totalImpressions, avgPosition, avgCtr, dateRange },
-    folder: folders.sort().pop(),
+    folder: selectedFolder.name,
   };
 }