seo-intel 1.2.5 → 1.3.0

package/analyses/blog-draft/index.js

@@ -0,0 +1,227 @@
+/**
+ * AEO Blog Draft Generator — Data Gathering & Prompt Builder
+ *
+ * Pulls intelligence from the Ledger (keyword gaps, long-tails, citability gaps,
+ * entities, positioning) and builds a prompt that produces a publish-ready,
+ * AEO-optimised blog post in .md format with YAML frontmatter.
+ */
+
+import { getActiveInsights } from '../../db/db.js';
+
+// ── Data Gathering ──────────────────────────────────────────────────────────
+
+export function gatherBlogDraftContext(db, project, topic = null) {
+  const insights = getActiveInsights(db, project);
+
+  // citability_gap insights — not in getActiveInsights grouped return
+  let citabilityGaps = [];
+  try {
+    citabilityGaps = db.prepare(
+      `SELECT data FROM insights WHERE project = ? AND type = 'citability_gap' AND status = 'active' ORDER BY last_seen DESC LIMIT 15`
+    ).all(project).map(r => JSON.parse(r.data));
+  } catch { /* table may not exist yet */ }
+
+  // Top entities across target pages
+  let entityRows = [];
+  try {
+    entityRows = db.prepare(`
+      SELECT e.primary_entities, p.title, p.url
+      FROM extractions e
+      JOIN pages p ON p.id = e.page_id
+      JOIN domains d ON d.id = p.domain_id
+      WHERE d.project = ? AND (d.role = 'target' OR d.role = 'owned')
+        AND e.primary_entities IS NOT NULL AND e.primary_entities != '[]'
+      ORDER BY p.word_count DESC LIMIT 20
+    `).all(project);
+  } catch { /* extraction may not have run */ }
+
+  // Best AEO-scoring pages (content to emulate)
+  let topCitablePages = [];
+  try {
+    topCitablePages = db.prepare(`
+      SELECT p.url, p.title, cs.total_score as score, cs.ai_intents, cs.tier
+      FROM citability_scores cs
+      JOIN pages p ON p.id = cs.page_id
+      JOIN domains d ON d.id = p.domain_id
+      WHERE d.project = ? AND (d.role = 'target' OR d.role = 'owned') AND cs.total_score >= 55
+      ORDER BY cs.total_score DESC LIMIT 5
+    `).all(project);
+  } catch { /* AEO may not have run */ }
+
+  // Filter by topic if given
+  const matchesTopic = (text) => {
+    if (!topic || !text) return true;
+    return text.toLowerCase().includes(topic.toLowerCase());
+  };
+
+  const kwInventor = insights.keyword_inventor
+    .filter(k => matchesTopic(k.phrase) || matchesTopic(k.cluster))
+    .slice(0, 30);
+
+  const longTails = topic
+    ? [
+        ...insights.long_tails.filter(lt => matchesTopic(lt.phrase)).slice(0, 20),
+        ...insights.long_tails.filter(lt => !matchesTopic(lt.phrase)).slice(0, 10),
+      ]
+    : insights.long_tails.slice(0, 30);
+
+  const keywordGaps = topic
+    ? [
+        ...insights.keyword_gaps.filter(kg => matchesTopic(kg.keyword)).slice(0, 15),
+        ...insights.keyword_gaps.filter(kg => !matchesTopic(kg.keyword)).slice(0, 10),
+      ]
+    : insights.keyword_gaps.filter(kg => kg.priority === 'high').slice(0, 25);
+
+  const contentGaps = (insights.content_gaps || []).slice(0, 8);
+
+  return {
+    insights,
+    citabilityGaps,
+    entityRows,
+    topCitablePages,
+    kwInventor,
+    longTails,
+    keywordGaps,
+    contentGaps,
+    topic,
+  };
+}
+
+// ── Prompt Builder ──────────────────────────────────────────────────────────
+
+export function buildBlogDraftPrompt(context, { config, lang = 'en', topic = null }) {
+  const { longTails, keywordGaps, citabilityGaps, entityRows, topCitablePages, kwInventor, contentGaps, insights } = context;
+  const isFi = lang === 'fi';
+
+  // Extract unique entities from extraction data
+  const allEntities = new Set();
+  for (const row of entityRows) {
+    try {
+      const ents = JSON.parse(row.primary_entities);
+      if (Array.isArray(ents)) ents.forEach(e => allEntities.add(typeof e === 'string' ? e : e.name || e));
+    } catch { /* skip */ }
+  }
+  const topEntities = [...allEntities].slice(0, 15);
+
+  // ── Section 1: Role ──
+  let prompt = `You are an expert content strategist and copywriter specialising in AEO (Answer Engine Optimisation).
+
+Your task: write a complete, publish-ready blog post draft in ${isFi ? 'Finnish' : 'English'}.
+The post must score 70+ on the AEO citability scale (entity authority, structured claims, answer density, Q&A proximity, freshness signals, schema coverage).
+
+`;
+
+  // ── Section 2: Site intelligence ──
+  prompt += `## Site Context
+
+- **Site:** ${config.context?.siteName || config.target?.domain} (${config.target?.url})
+- **Industry:** ${config.context?.industry || 'N/A'}
+- **Audience:** ${config.context?.audience || 'N/A'}
+- **Goal:** ${config.context?.goal || 'N/A'}
+`;
+
+  if (insights.positioning) {
+    prompt += `- **Positioning:** ${typeof insights.positioning === 'string' ? insights.positioning : JSON.stringify(insights.positioning)}\n`;
+  }
+
+  if (topEntities.length) {
+    prompt += `- **Core entities:** ${topEntities.join(', ')}\n`;
+  }
+
+  if (topCitablePages.length) {
+    prompt += `\n### Highest-scoring pages on the site (emulate their structure)\n`;
+    for (const p of topCitablePages) {
+      prompt += `- ${p.url} — AEO score: ${p.score}/100 (${p.tier})\n`;
+    }
+  }
+
+  // ── Section 3: Topic focus ──
+  prompt += `\n## Topic\n\n`;
+  if (topic) {
+    prompt += `Primary focus: **${topic}**. All keyword and gap data below has been filtered to this topic. Build the entire post around this subject.\n`;
+  } else {
+    prompt += `Select the highest-opportunity topic from the gaps below. Choose the gap that: (a) has the most keyword_gap entries or (b) is flagged as a high priority long-tail. Explain your topic choice in the frontmatter \`topic_selection_rationale\` field.\n`;
+  }
+
+  // ── Section 4: Intelligence data ──
+  if (keywordGaps.length) {
+    prompt += `\n## Keyword Gaps to Target (include these as primary/secondary keywords)\n\n`;
+    prompt += `| Keyword | Priority | Notes |\n|---|---|---|\n`;
+    for (const kg of keywordGaps) {
+      prompt += `| ${kg.keyword || kg.phrase || '—'} | ${kg.priority || 'medium'} | ${(kg.notes || '').slice(0, 80)} |\n`;
+    }
+  }
+
+  if (longTails.length) {
+    prompt += `\n## Long-tail Phrases to Answer (each should have a direct answer in the post)\n\n`;
+    prompt += `| Phrase | Intent | Priority |\n|---|---|---|\n`;
+    for (const lt of longTails) {
+      prompt += `| ${lt.phrase || '—'} | ${lt.intent || '—'} | ${lt.priority || 'medium'} |\n`;
+    }
+  }
+
+  if (kwInventor.length) {
+    prompt += `\n## Keyword Inventor Phrases (weave these naturally into headings/body)\n\n`;
+    for (const kw of kwInventor.slice(0, 20)) {
+      prompt += `- "${kw.phrase}" (${kw.type || 'traditional'}, ${kw.intent || '—'})\n`;
+    }
+  }
+
+  if (citabilityGaps.length) {
+    prompt += `\n## Citability Gaps (pages scoring <60 on AEO — model the fix in this post)\n\n`;
+    prompt += `| URL | Score | Weakest Signals |\n|---|---|---|\n`;
+    for (const cg of citabilityGaps) {
+      prompt += `| ${cg.url || '—'} | ${cg.score || '—'} | ${cg.weakest || cg.weakest_signal || '—'} |\n`;
+    }
+  }
+
+  if (contentGaps.length) {
+    prompt += `\n## Content Gaps (topics competitors cover that you don't)\n\n`;
+    for (const cg of contentGaps) {
+      const desc = typeof cg === 'string' ? cg : (cg.topic || cg.description || cg.gap || JSON.stringify(cg));
+      prompt += `- ${desc}\n`;
+    }
+  }
+
+  // ── Section 5: AEO structural requirements ──
+  prompt += `
+## AEO Structural Requirements
+
+The draft MUST include:
+1. YAML frontmatter with: title, slug, description (155 chars max), primary_keyword, secondary_keywords[], date (${new Date().toISOString().slice(0, 10)}), updated (same), lang (${lang}), tags[]${!topic ? ', topic_selection_rationale' : ''}
+2. An H1 that contains the primary keyword
+3. A 2-3 sentence summary immediately after the H1 (answer-first structure — inverted pyramid). This paragraph will be cited by AI assistants.
+4. Minimum 6 H2 subheadings
+5. At least 3 H2s phrased as direct questions (What is / How to / Why / When)
+6. At least one numbered or bulleted list with 4+ items
+7. At least one "X is Y because Z" definitional sentence per major concept
+8. A FAQ section at the end with minimum 4 Q&A pairs (### H3 questions, 2-4 sentence answers)
+9. A closing CTA paragraph referencing ${config.context?.siteName || config.target?.domain}
+10. Word count: 1,200-2,000 words
+11. Internal link suggestions: include 2-3 \`[anchor text](URL)\` links back to the site where natural
+`;
+
+  // ── Section 6: Language ──
+  if (isFi) {
+    prompt += `
+## Language: Finnish
+
+Write in Finnish. Use informal, direct register (sinuttelu where natural). Avoid marketing clichés common in Finnish B2B copy. Prefer short sentences. Finnish SEO keywords must appear in their exact searched base form in headings — Finnish inflection reduces exact-match keyword presence.
+`;
+  } else {
+    prompt += `
+## Language: English
+
+Write in clear, direct international English. No filler phrases. No "in today's digital landscape" or "it's no secret that" openers. Every sentence should contain a fact, insight, or actionable point.
+`;
+  }
+
+  // ── Section 7: Output format ──
+  prompt += `
+## Output Format
+
+Respond with ONLY the complete markdown document. Start with --- (YAML frontmatter open fence). End with the FAQ section and CTA. No explanation before or after. No triple backticks wrapping the response.
+`;
+
+  return prompt;
+}

package/analyses/blog-draft/prescorer.js

@@ -0,0 +1,60 @@
+/**
+ * AEO Pre-Scorer — scores a generated markdown draft against citability signals
+ *
+ * Uses the same scorePage() function as the full AEO audit, but constructs
+ * synthetic inputs from the markdown text instead of reading from the DB.
+ *
+ * Freshness always scores 0 (no publish date yet) — the reported score
+ * accounts for this by adding +10 for "what it will score once published."
+ */
+
+import { scorePage } from '../aeo/scorer.js';
+
+export function prescore(markdownText) {
+  // Strip YAML frontmatter
+  const bodyMatch = markdownText.match(/^---[\s\S]*?---\n([\s\S]*)$/);
+  const body = bodyMatch ? bodyMatch[1] : markdownText;
+
+  // Extract headings
+  const headings = [];
+  for (const line of body.split('\n')) {
+    const m = line.match(/^(#{1,6})\s+(.+)$/);
+    if (m) headings.push({ level: m[1].length, text: m[2].trim() });
+  }
+
+  // Word count
+  const wordCount = body.split(/\s+/).filter(Boolean).length;
+
+  // Extract frontmatter fields
+  let fmSchemaType = null;
+  const fmMatch = markdownText.match(/^---([\s\S]*?)---/);
+  if (fmMatch) {
+    const schemaLine = fmMatch[1].match(/schema_type:\s*(.+)/);
+    if (schemaLine) fmSchemaType = schemaLine[1].trim();
+  }
+
+  // Build synthetic page object
+  const syntheticPage = {
+    body_text: body,
+    word_count: wordCount,
+    published_date: null,   // not published yet — freshness = 0
+    modified_date: null,
+  };
+
+  // Extract entity candidates from headings (capitalised noun phrases)
+  const entityCandidates = headings
+    .filter(h => h.level <= 3)
+    .flatMap(h => h.text.match(/\b[A-ZÄÖÅ][a-zäöå]+(?:\s+[A-ZÄÖÅ][a-zäöå]+)*/g) || []);
+  const entities = [...new Set(entityCandidates)].slice(0, 8);
+
+  const schemaTypes = fmSchemaType ? [fmSchemaType] : [];
+  const schemas = [];
+
+  const result = scorePage(syntheticPage, headings, entities, schemaTypes, schemas, 'Informational');
+
+  return {
+    ...result,
+    wordCount,
+    headingCount: headings.length,
+  };
+}

package/analyses/templates/cluster.js

@@ -0,0 +1,209 @@
+/**
+ * URL Pattern Clustering — Phase 1
+ *
+ * Takes sitemap URLs, detects parametric patterns, groups them.
+ * Pure function — no I/O, no side effects.
+ */
+
+/**
+ * Is this path segment a "variable" (one of N possible values)
+ * vs a "constant" (structural path like 'swap', 'docs', 'blog')?
+ */
+function isVariable(segment) {
+  // Version prefixes stay constant: v1, v2, v3...
+  if (/^v\d+$/.test(segment)) return false;
+
+  // Common structural words stay constant
+  const STRUCTURAL = new Set([
+    'api', 'docs', 'blog', 'news', 'about', 'pricing', 'features',
+    'help', 'support', 'contact', 'legal', 'terms', 'privacy',
+    'login', 'signup', 'register', 'dashboard', 'settings',
+    'token', 'tokens', 'swap', 'trade', 'perps', 'perpetuals',
+    'pool', 'pools', 'stake', 'staking', 'bridge', 'earn',
+    'governance', 'vote', 'proposals', 'stats', 'analytics',
+    'markets', 'pairs', 'explorer', 'episodes', 'categories',
+    'tags', 'products', 'collections', 'pages', 'posts',
+  ]);
+  if (STRUCTURAL.has(segment.toLowerCase())) return false;
+
+  // Purely numeric → variable (IDs, dates)
+  if (/^\d+$/.test(segment)) return true;
+
+  // UUID or hash-like
+  if (/^[0-9a-f-]{8,}$/i.test(segment)) return true;
+
+  // Hex address (0x...)
+  if (/^0x[0-9a-fA-F]+$/.test(segment)) return true;
+
+  // Contains separator characters typical of slugs/pairs: SOL-USDC, my-blog-post
+  if (/[-_.]/.test(segment) && segment.length > 2) return true;
+
+  // All uppercase short string → likely a ticker: SOL, BONK, USDT, ETH
+  if (/^[A-Z0-9]{2,10}$/.test(segment)) return true;
+
+  // Mixed case with digits → product codes, IDs
+  if (/[A-Z]/.test(segment) && /\d/.test(segment)) return true;
+
+  // Very long segments are likely slugs or IDs
+  if (segment.length > 30) return true;
+
+  return false;
+}
+
+/**
+ * Infer a semantic name for a param position based on observed values.
+ */
+function inferParamName(values, position) {
+  const sample = values.slice(0, 100);
+
+  // Crypto pairs: X-Y format where both parts are short uppercase
+  const pairCount = sample.filter(v => /^[A-Za-z0-9]+-[A-Za-z0-9]+$/.test(v)).length;
+  if (pairCount > sample.length * 0.6) return 'pair';
+
+  // Token tickers: 2-10 uppercase chars
+  const tickerCount = sample.filter(v => /^[A-Z0-9]{2,10}$/.test(v)).length;
+  if (tickerCount > sample.length * 0.6) return 'symbol';
+
+  // Slugs: lowercase with hyphens
+  const slugCount = sample.filter(v => /^[a-z0-9]+(-[a-z0-9]+)+$/.test(v)).length;
+  if (slugCount > sample.length * 0.6) return 'slug';
+
+  // Numeric IDs
+  const numCount = sample.filter(v => /^\d+$/.test(v)).length;
+  if (numCount > sample.length * 0.6) return 'id';
+
+  // Hex hashes/addresses
+  const hexCount = sample.filter(v => /^(0x)?[0-9a-f]{8,}$/i.test(v)).length;
+  if (hexCount > sample.length * 0.6) return 'hash';
+
+  return `param${position}`;
+}
+
+/**
+ * Cluster sitemap URLs into template groups.
+ *
+ * @param {Array<{url: string, lastmod?: string}>} sitemapEntries
+ * @param {object} opts
+ * @param {number} opts.minGroupSize — min URLs to qualify as template (default 10)
+ * @param {number} opts.maxSegments — max path depth to consider (default 8)
+ * @returns {{ groups: TemplateGroup[], ungrouped: string[], stats: object }}
+ */
+export function clusterUrls(sitemapEntries, opts = {}) {
+  const minGroupSize = opts.minGroupSize || 10;
+  const maxSegments = opts.maxSegments || 8;
+
+  // patternKey → { pattern parts, urls[], paramValues by position }
+  const clusters = new Map();
+
+  for (const entry of sitemapEntries) {
+    let pathname;
+    try {
+      pathname = new URL(entry.url).pathname;
+    } catch { continue; }
+
+    // Normalize
+    pathname = pathname.replace(/\/+$/, '') || '/';
+
+    // Homepage is always unique
+    if (pathname === '/') continue;
+
+    const segments = pathname.split('/').filter(Boolean).slice(0, maxSegments);
+    const patternParts = [];
+    const paramPositions = {};
+    let paramIdx = 0;
+
+    for (let i = 0; i < segments.length; i++) {
+      if (isVariable(segments[i])) {
+        const key = `p${paramIdx}`;
+        patternParts.push(`{${key}}`);
+        if (!paramPositions[key]) paramPositions[key] = [];
+        paramPositions[key].push(segments[i]);
+        paramIdx++;
+      } else {
+        patternParts.push(segments[i].toLowerCase());
+      }
+    }
+
+    const patternKey = '/' + patternParts.join('/');
+
+    if (!clusters.has(patternKey)) {
+      clusters.set(patternKey, {
+        patternKey,
+        patternParts,
+        urls: [],
+        paramPositions: {},
+        lastmods: [],
+      });
+    }
+
+    const cluster = clusters.get(patternKey);
+    cluster.urls.push(entry.url);
+    if (entry.lastmod) cluster.lastmods.push(entry.lastmod);
+
+    // Collect param values (cap at 200 for memory)
+    for (const [key, values] of Object.entries(paramPositions)) {
+      if (!cluster.paramPositions[key]) cluster.paramPositions[key] = [];
+      if (cluster.paramPositions[key].length < 200) {
+        cluster.paramPositions[key].push(...values);
+      }
+    }
+  }
+
+  // Separate into template groups (>= minGroupSize) and ungrouped
+  const groups = [];
+  const ungrouped = [];
+
+  for (const [patternKey, cluster] of clusters) {
+    if (cluster.urls.length >= minGroupSize) {
+      // Rename params to semantic names
+      const params = {};
+      const renamedParts = [...cluster.patternParts];
+
+      let paramIdx = 0;
+      for (let i = 0; i < renamedParts.length; i++) {
+        const match = renamedParts[i].match(/^\{(p\d+)\}$/);
+        if (match) {
+          const key = match[1];
+          const name = inferParamName(cluster.paramPositions[key] || [], paramIdx);
+          renamedParts[i] = `{${name}}`;
+          params[name] = (cluster.paramPositions[key] || []).slice(0, 50);
+          paramIdx++;
+        }
+      }
+
+      const pattern = '/' + renamedParts.join('/');
+      const sortedLastmods = cluster.lastmods.sort();
+
+      groups.push({
+        pattern,
+        patternKey,
+        params,
+        urls: cluster.urls,
+        urlCount: cluster.urls.length,
+        depth: cluster.patternParts.length,
+        firstSeen: sortedLastmods[0] || null,
+        lastSeen: sortedLastmods[sortedLastmods.length - 1] || null,
+      });
+    } else {
+      ungrouped.push(...cluster.urls);
+    }
+  }
+
+  // Sort by URL count descending
+  groups.sort((a, b) => b.urlCount - a.urlCount);
+
+  const totalGrouped = groups.reduce((sum, g) => sum + g.urlCount, 0);
+  const totalUrls = sitemapEntries.length;
+
+  return {
+    groups,
+    ungrouped,
+    stats: {
+      totalUrls,
+      totalGroups: groups.length,
+      totalGrouped,
+      largestGroup: groups[0]?.urlCount || 0,
+      coverage: totalUrls > 0 ? totalGrouped / totalUrls : 0,
+    },
+  };
+}

package/analyses/templates/gsc-overlay.js

@@ -0,0 +1,93 @@
+/**
+ * GSC Overlay — Phase 3
+ *
+ * Cross-references template groups against Google Search Console per-URL data.
+ * Pure computation — no I/O.
+ */
+
+/**
+ * Normalize a URL for GSC matching.
+ * GSC reports URLs inconsistently — trailing slashes, www, http vs https.
+ */
+function normalizeUrl(url) {
+  try {
+    const u = new URL(url);
+    return (u.protocol + '//' + u.hostname + u.pathname).replace(/\/+$/, '').toLowerCase();
+  } catch {
+    return url.toLowerCase().replace(/\/+$/, '');
+  }
+}
+
+/**
+ * Cross-reference template groups with GSC pages data.
+ *
+ * @param {TemplateGroup[]} groups — from cluster.js
+ * @param {Array<{url: string, clicks: number, impressions: number, ctr: number, position: number}>|null} gscPages
+ * @returns {GscOverlayResult[]}
+ */
+export function overlayGsc(groups, gscPages) {
+  if (!gscPages || gscPages.length === 0) {
+    // No GSC data — return groups with null GSC fields
+    return groups.map(g => ({
+      ...g,
+      gscUrlsWithImpressions: null,
+      gscTotalClicks: null,
+      gscTotalImpressions: null,
+      gscAvgPosition: null,
+      indexationEfficiency: null,
+      topGscUrls: [],
+    }));
+  }
+
+  // Build normalized URL → GSC entry lookup
+  const gscMap = new Map();
+  for (const entry of gscPages) {
+    const key = normalizeUrl(entry.url);
+    // Keep the one with more impressions if dupes
+    const existing = gscMap.get(key);
+    if (!existing || entry.impressions > existing.impressions) {
+      gscMap.set(key, entry);
+    }
+  }
+
+  return groups.map(group => {
+    let urlsWithImpressions = 0;
+    let totalClicks = 0;
+    let totalImpressions = 0;
+    let positionSum = 0;
+    let positionCount = 0;
+    const topUrls = [];
+
+    for (const url of group.urls) {
+      const gscEntry = gscMap.get(normalizeUrl(url));
+      if (gscEntry && gscEntry.impressions > 0) {
+        urlsWithImpressions++;
+        totalClicks += gscEntry.clicks || 0;
+        totalImpressions += gscEntry.impressions || 0;
+        if (gscEntry.position > 0) {
+          positionSum += gscEntry.position;
+          positionCount++;
+        }
+        topUrls.push({
+          url: gscEntry.url,
+          clicks: gscEntry.clicks,
+          impressions: gscEntry.impressions,
+          position: gscEntry.position,
+        });
+      }
+    }
+
+    // Sort top URLs by impressions desc, take top 10
+    topUrls.sort((a, b) => b.impressions - a.impressions);
+
+    return {
+      ...group,
+      gscUrlsWithImpressions: urlsWithImpressions,
+      gscTotalClicks: totalClicks,
+      gscTotalImpressions: totalImpressions,
+      gscAvgPosition: positionCount > 0 ? Math.round((positionSum / positionCount) * 10) / 10 : null,
+      indexationEfficiency: group.urlCount > 0 ? urlsWithImpressions / group.urlCount : 0,
+      topGscUrls: topUrls.slice(0, 10),
+    };
+  });
+}