seo-intel 1.2.5 → 1.3.0

package/CHANGELOG.md

@@ -1,5 +1,34 @@
 # Changelog
 
+## 1.3.0 (2026-04-01)
+
+### New Feature: AEO Blog Draft Generator
+- `seo-intel blog-draft <project>` — generate AEO-optimised blog post drafts from Intelligence Ledger data
+- Gathers keyword gaps, long-tails, citability insights, entities, and top citable pages
+- Builds structured prompt with 10 AEO signal rules for maximum AI citability
+- Pre-scores generated draft against AEO signals before publishing
+- Options: `--topic`, `--lang en|fi`, `--model gemini|claude|gpt|deepseek`, `--save`
+- Pro feature gated via Lemon Squeezy license
+
+### Dashboard
+- New "Create" section in export sidebar with interactive draft generator
+- "Create a Draft" dropdown: select type (Blog Post / Documentation), topic, language, then generate
+- "AI Citability Audit" button added to export sidebar — run AEO from dashboard
+- Both `aeo` and `blog-draft` commands now available via dashboard terminal
+
+### Server
+- Added `aeo` and `blog-draft` to terminal command whitelist
+- Forward `--topic`, `--lang`, `--model`, `--save` params from dashboard to CLI
+
+## 1.2.6 (2026-03-31)
+
+### Critical Fix
+- **Ship analysis, extraction, and AEO modules in npm package** — these were gitignored as "proprietary" from the Froggo era but are required for `extract`, `analyze`, `aeo`, `templates`, and dashboard generation
+- npm users can now run the full pipeline without missing module errors
+- Files added to git: `analyses/aeo/`, `analyses/templates/`, `analysis/`, `extractor/`
+- Removed stale "NOT shipped in free npm package" comment from cli.js
+- Deleted local `froggo-package/` directory
+
 ## 1.2.5 (2026-03-31)
 
 ### Skill / OpenClaw

package/analyses/aeo/index.js

@@ -0,0 +1,252 @@
+/**
+ * AEO / AI Citability Analysis — Orchestrator
+ *
+ * Reads crawled pages from DB, scores each for AI citability,
+ * stores results, and optionally feeds low-scoring pages into the Intelligence Ledger.
+ */
+
+import { scorePage } from './scorer.js';
+
+/**
+ * Run AEO analysis for a project.
+ *
+ * @param {import('node:sqlite').DatabaseSync} db
+ * @param {string} project
+ * @param {object} opts - { includeCompetitors: boolean, log: function }
+ * @returns {object} { target: PageScore[], competitors: Map<domain, PageScore[]>, summary }
+ */
+export function runAeoAnalysis(db, project, opts = {}) {
+  const log = opts.log || console.log;
+  const includeCompetitors = opts.includeCompetitors ?? true;
+
+  // ── Gather pages with body_text ─────────────────────────────────────────
+  const roleFilter = includeCompetitors
+    ? ''
+    : `AND d.role IN ('target', 'owned')`;
+
+  const pages = db.prepare(`
+    SELECT
+      p.id, p.url, p.title, p.body_text, p.word_count,
+      p.published_date, p.modified_date,
+      d.domain, d.role,
+      e.primary_entities, e.search_intent, e.schema_types
+    FROM pages p
+    JOIN domains d ON d.id = p.domain_id
+    LEFT JOIN extractions e ON e.page_id = p.id
+    WHERE d.project = ?
+      AND p.body_text IS NOT NULL AND p.body_text != ''
+      AND p.is_indexable = 1
+      ${roleFilter}
+    ORDER BY d.role ASC, p.url ASC
+  `).all(project);
+
+  if (!pages.length) {
+    return { target: [], competitors: new Map(), summary: null };
+  }
+
+  // ── Gather headings + schemas per page ──────────────────────────────────
+  const headingsStmt = db.prepare(
+    'SELECT level, text FROM headings WHERE page_id = ? ORDER BY id'
+  );
+  const schemasStmt = db.prepare(
+    'SELECT schema_type, date_published, date_modified FROM page_schemas WHERE page_id = ?'
+  );
+
+  // ── Score each page ─────────────────────────────────────────────────────
+  const targetResults = [];
+  const competitorResults = new Map();
+  let scored = 0;
+
+  for (const page of pages) {
+    const headings = headingsStmt.all(page.id);
+    const pageSchemas = schemasStmt.all(page.id);
+    const schemaTypes = pageSchemas.map(s => s.schema_type);
+
+    // Also merge extraction schema_types if page_schemas is empty
+    if (!schemaTypes.length && page.schema_types) {
+      try {
+        const ext = JSON.parse(page.schema_types);
+        if (Array.isArray(ext)) schemaTypes.push(...ext);
+      } catch { /* ignore */ }
+    }
+
+    let entities = [];
+    try {
+      entities = JSON.parse(page.primary_entities || '[]');
+    } catch { /* ignore */ }
+
+    const result = scorePage(
+      page, headings, entities, schemaTypes, pageSchemas, page.search_intent
+    );
+
+    const pageScore = {
+      pageId: page.id,
+      url: page.url,
+      title: page.title,
+      domain: page.domain,
+      role: page.role,
+      wordCount: page.word_count,
+      ...result,
+    };
+
+    if (page.role === 'target' || page.role === 'owned') {
+      targetResults.push(pageScore);
+    } else {
+      if (!competitorResults.has(page.domain)) competitorResults.set(page.domain, []);
+      competitorResults.get(page.domain).push(pageScore);
+    }
+
+    scored++;
+  }
+
+  // Sort by score ascending (worst first — actionable)
+  targetResults.sort((a, b) => a.score - b.score);
+  for (const [, arr] of competitorResults) arr.sort((a, b) => a.score - b.score);
+
+  // ── Summary stats ────────────────────────────────────────────────────────
+  const targetScores = targetResults.map(r => r.score);
+  const avgTarget = targetScores.length
+    ? Math.round(targetScores.reduce((a, b) => a + b, 0) / targetScores.length)
+    : 0;
+
+  const compScores = [...competitorResults.values()].flat().map(r => r.score);
+  const avgComp = compScores.length
+    ? Math.round(compScores.reduce((a, b) => a + b, 0) / compScores.length)
+    : 0;
+
+  const tierCounts = { excellent: 0, good: 0, needs_work: 0, poor: 0 };
+  for (const r of targetResults) tierCounts[r.tier]++;
+
+  const summary = {
+    totalScored: scored,
+    targetPages: targetResults.length,
+    competitorPages: compScores.length,
+    avgTargetScore: avgTarget,
+    avgCompetitorScore: avgComp,
+    scoreDelta: avgTarget - avgComp,
+    tierCounts,
+    weakestSignals: getWeakestSignals(targetResults),
+  };
+
+  log(`  Scored ${scored} pages (${targetResults.length} target, ${compScores.length} competitor)`);
+  log(`  Target avg: ${avgTarget}/100 | Competitor avg: ${avgComp}/100 | Delta: ${summary.scoreDelta > 0 ? '+' : ''}${summary.scoreDelta}`);
+
+  return { target: targetResults, competitors: competitorResults, summary };
+}
+
+/**
+ * Persist AEO scores to citability_scores table
+ */
+export function persistAeoScores(db, results) {
+  const stmt = db.prepare(`
+    INSERT OR REPLACE INTO citability_scores
+      (page_id, score, entity_authority, structured_claims, answer_density,
+       qa_proximity, freshness, schema_coverage, ai_intents, tier, scored_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+
+  const allResults = [
+    ...results.target,
+    ...[...results.competitors.values()].flat(),
+  ];
+
+  db.exec('BEGIN');
+  try {
+    for (const r of allResults) {
+      stmt.run(
+        r.pageId, r.score,
+        r.breakdown.entity_authority, r.breakdown.structured_claims,
+        r.breakdown.answer_density, r.breakdown.qa_proximity,
+        r.breakdown.freshness, r.breakdown.schema_coverage,
+        JSON.stringify(r.aiIntents), r.tier, Date.now()
+      );
+    }
+    db.exec('COMMIT');
+  } catch (e) {
+    db.exec('ROLLBACK');
+    throw e;
+  }
+}
+
+/**
+ * Feed low-scoring pages into Intelligence Ledger as citability_gap insights
+ */
+export function upsertCitabilityInsights(db, project, targetResults) {
+  const upsertStmt = db.prepare(`
+    INSERT INTO insights (project, type, status, fingerprint, first_seen, last_seen, source_analysis_id, data)
+    VALUES (?, 'citability_gap', 'active', ?, ?, ?, NULL, ?)
+    ON CONFLICT(project, type, fingerprint) DO UPDATE SET
+      last_seen = excluded.last_seen,
+      data = excluded.data
+  `);
+
+  const ts = Date.now();
+  db.exec('BEGIN');
+  try {
+    for (const r of targetResults) {
+      if (r.score >= 60) continue; // only flag pages that need work
+
+      const fp = r.url.toLowerCase().replace(/[^a-z0-9/]/g, '').trim();
+      const weakest = Object.entries(r.breakdown)
+        .sort(([, a], [, b]) => a - b)
+        .slice(0, 2)
+        .map(([k]) => k.replace(/_/g, ' '));
+
+      const data = {
+        url: r.url,
+        title: r.title,
+        score: r.score,
+        tier: r.tier,
+        weakest_signals: weakest,
+        ai_intents: r.aiIntents,
+        recommendation: `Improve ${weakest.join(' and ')} to boost AI citability from ${r.score}/100`,
+      };
+
+      upsertStmt.run(project, fp, ts, ts, JSON.stringify(data));
+    }
+    db.exec('COMMIT');
+  } catch (e) {
+    db.exec('ROLLBACK');
+    console.error('[aeo] insight upsert failed:', e.message);
+  }
+}
+
+// ── Helpers ────────────────────────────────────────────────────────────────
+
+function getWeakestSignals(targetResults) {
+  if (!targetResults.length) return [];
+
+  const signalTotals = {
+    entity_authority: 0, structured_claims: 0, answer_density: 0,
+    qa_proximity: 0, freshness: 0, schema_coverage: 0,
+  };
+
+  for (const r of targetResults) {
+    for (const [k, v] of Object.entries(r.breakdown)) {
+      signalTotals[k] += v;
+    }
+  }
+
+  return Object.entries(signalTotals)
+    .map(([signal, total]) => ({
+      signal: signal.replace(/_/g, ' '),
+      avg: Math.round(total / targetResults.length),
+    }))
+    .sort((a, b) => a.avg - b.avg);
+}
+
+/**
+ * Read stored citability scores for dashboard
+ */
+export function getCitabilityScores(db, project) {
+  return db.prepare(`
+    SELECT
+      cs.*, p.url, p.title, p.word_count,
+      d.domain, d.role
+    FROM citability_scores cs
+    JOIN pages p ON p.id = cs.page_id
+    JOIN domains d ON d.id = p.domain_id
+    WHERE d.project = ?
+    ORDER BY d.role ASC, cs.score ASC
+  `).all(project);
+}

package/analyses/aeo/scorer.js

@@ -0,0 +1,254 @@
+/**
+ * AEO Citability Scorer — pure function, zero I/O
+ *
+ * Scores a page for how well an AI assistant could cite it as a source.
+ * All inputs are plain objects from the DB; output is a score breakdown.
+ */
+
+// ── Question patterns in headings ──────────────────────────────────────────
+const QUESTION_RE = /^(what|how|why|when|where|which|who|can|does|is|are|should|do)\b/i;
+const COMPARISON_RE = /\bvs\.?\b|\bversus\b|\bcompare[d]?\b|\bcomparison\b|\balternative/i;
+const IMPL_RE = /\bhow to\b|\bstep[- ]by[- ]step\b|\btutorial\b|\bguide\b|\bsetup\b|\binstall/i;
+
+// ── Freshness scoring ──────────────────────────────────────────────────────
+function freshnessScore(page, schemas) {
+  // Best signal: dateModified in schema
+  const schemaDate = schemas.find(s => s.date_modified)?.date_modified
+    || schemas.find(s => s.date_published)?.date_published;
+  const pageDate = page.modified_date || page.published_date;
+  const dateStr = schemaDate || pageDate;
+
+  if (!dateStr) return 0;
+
+  const d = new Date(dateStr);
+  if (isNaN(d.getTime())) return 0;
+
+  const ageMs = Date.now() - d.getTime();
+  const ageDays = ageMs / (1000 * 60 * 60 * 24);
+
+  if (ageDays < 90) return 100;   // < 3 months
+  if (ageDays < 180) return 80;   // < 6 months
+  if (ageDays < 365) return 60;   // < 1 year
+  if (ageDays < 730) return 30;   // < 2 years
+  return 10;                       // 2+ years
+}
+
+// ── Entity authority ───────────────────────────────────────────────────────
+function entityAuthorityScore(entities, headings, wordCount) {
+  if (!entities.length) return 0;
+
+  let score = 0;
+
+  // More entities = deeper coverage
+  if (entities.length >= 5) score += 30;
+  else if (entities.length >= 3) score += 20;
+  else score += 10;
+
+  // Entities appearing in headings = stronger authority signal
+  const headingTexts = headings.map(h => h.text.toLowerCase());
+  const entityInHeading = entities.filter(e =>
+    headingTexts.some(ht => ht.includes(e.toLowerCase()))
+  ).length;
+
+  score += Math.min(entityInHeading * 15, 40);
+
+  // Word count indicates depth of coverage
+  if (wordCount >= 2000) score += 30;
+  else if (wordCount >= 1000) score += 20;
+  else if (wordCount >= 500) score += 10;
+
+  return Math.min(score, 100);
+}
+
+// ── Structured claims ──────────────────────────────────────────────────────
+function structuredClaimsScore(bodyText, headings) {
+  if (!bodyText) return 0;
+
+  let score = 0;
+  const sentences = bodyText.split(/[.!?]+/).filter(s => s.trim().length > 20);
+  if (!sentences.length) return 0;
+
+  // "X is Y" definitional patterns — highly citable
+  const definitional = sentences.filter(s =>
+    /\b(?:is|are|means|refers to|defined as|consists of)\b/i.test(s)
+  ).length;
+  score += Math.min((definitional / sentences.length) * 200, 40);
+
+  // Numbered/bulleted patterns in body (listicle structure)
+  const listPatterns = (bodyText.match(/(?:^|\n)\s*(?:\d+[.)]\s|[-•]\s)/gm) || []).length;
+  if (listPatterns >= 5) score += 25;
+  else if (listPatterns >= 3) score += 15;
+
+  // Comparison patterns
+  if (COMPARISON_RE.test(bodyText)) score += 15;
+
+  // Step-by-step / how-to patterns
+  if (IMPL_RE.test(bodyText)) score += 20;
+
+  return Math.min(score, 100);
+}
+
+// ── Answer density ─────────────────────────────────────────────────────────
+function answerDensityScore(bodyText, wordCount) {
+  if (!bodyText || wordCount < 100) return 0;
+
+  let score = 0;
+
+  // Short paragraphs = more scannable = better for AI extraction
+  const paragraphs = bodyText.split(/\n\s*\n/).filter(p => p.trim().length > 0);
+  if (!paragraphs.length) return 10;
+
+  const avgParaLength = wordCount / paragraphs.length;
+  if (avgParaLength <= 80) score += 30;      // concise
+  else if (avgParaLength <= 150) score += 20; // moderate
+  else score += 5;                            // wall of text
+
+  // First 200 words contain a direct answer? (inverted pyramid)
+  const first200 = bodyText.split(/\s+/).slice(0, 200).join(' ');
+  if (/\b(?:is|are|means|provides?|offers?|enables?|allows?)\b/i.test(first200)) {
+    score += 25;
+  }
+
+  // Ratio of informational content (not just navigation/boilerplate)
+  if (wordCount >= 300 && wordCount <= 3000) score += 25;
+  else if (wordCount > 3000) score += 15; // very long can dilute
+  else score += 10; // too short to cite well
+
+  // Code blocks are highly citable for technical content
+  const codeBlocks = (bodyText.match(/```[\s\S]*?```|`[^`]+`/g) || []).length;
+  if (codeBlocks >= 3) score += 20;
+  else if (codeBlocks >= 1) score += 10;
+
+  return Math.min(score, 100);
+}
+
+// ── Q&A proximity ──────────────────────────────────────────────────────────
+function qaProximityScore(headings, bodyText) {
+  if (!headings.length || !bodyText) return 0;
+
+  const questionHeadings = headings.filter(h =>
+    h.level >= 2 && h.level <= 3 && QUESTION_RE.test(h.text)
+  );
+
+  if (!questionHeadings.length) return 10; // no Q&A structure at all
+
+  let score = 0;
+
+  // More question headings = better Q&A structure
+  const qRatio = questionHeadings.length / headings.filter(h => h.level >= 2).length;
+  score += Math.min(qRatio * 60, 40);
+
+  // FAQ schema present? Huge bonus
+  score += 30;
+
+  // Heading density (one H2/H3 per ~300 words is ideal)
+  const h2h3Count = headings.filter(h => h.level >= 2 && h.level <= 3).length;
+  const words = bodyText.split(/\s+/).length;
+  const idealHeadings = Math.floor(words / 300);
+  const headingRatio = idealHeadings > 0 ? Math.min(h2h3Count / idealHeadings, 2) : 0;
+  if (headingRatio >= 0.7 && headingRatio <= 1.5) score += 30;
+  else if (headingRatio >= 0.4) score += 15;
+
+  return Math.min(score, 100);
+}
+
+// ── Schema coverage ────────────────────────────────────────────────────────
+function schemaCoverageScore(schemaTypes) {
+  if (!schemaTypes.length) return 0;
+
+  let score = 0;
+
+  // High-value schema types for AI citation
+  const highValue = ['FAQPage', 'HowTo', 'Article', 'TechArticle', 'BlogPosting'];
+  const medValue = ['Product', 'Review', 'SoftwareApplication', 'WebApplication'];
+  const baseValue = ['Organization', 'WebSite', 'WebPage', 'BreadcrumbList'];
+
+  for (const t of schemaTypes) {
+    if (highValue.includes(t)) score += 30;
+    else if (medValue.includes(t)) score += 15;
+    else if (baseValue.includes(t)) score += 5;
+  }
+
+  // Multiple schema types = richer structured data
+  if (schemaTypes.length >= 3) score += 20;
+
+  return Math.min(score, 100);
+}
+
+// ── AI Query Intent Classification ─────────────────────────────────────────
+function classifyAiIntent(headings, bodyText, searchIntent) {
+  const allText = [
+    ...headings.map(h => h.text),
+    (bodyText || '').slice(0, 2000)
+  ].join(' ').toLowerCase();
+
+  const intents = [];
+
+  if (COMPARISON_RE.test(allText)) intents.push('synthesis');
+  if (/\bshould\b|\brecommend|\bbest\b.*\bfor\b|\bchoose\b/i.test(allText)) intents.push('decision_support');
+  if (IMPL_RE.test(allText)) intents.push('implementation');
+  if (/\bwhat (is|are)\b|\boverview\b|\bintroduc/i.test(allText)) intents.push('exploration');
+  if (/\bbest practice|\bshould you\b|\bis it worth/i.test(allText)) intents.push('validation');
+
+  // Fallback from extraction intent
+  if (!intents.length) {
+    if (searchIntent === 'Informational') intents.push('exploration');
+    else if (searchIntent === 'Commercial') intents.push('decision_support');
+    else if (searchIntent === 'Transactional') intents.push('implementation');
+    else intents.push('exploration');
+  }
+
+  return intents;
+}
+
+// ── Main scorer ────────────────────────────────────────────────────────────
+
+/**
+ * Score a single page for AI citability.
+ *
+ * @param {object} page - { url, title, body_text, word_count, published_date, modified_date }
+ * @param {object[]} headings - [{ level, text }]
+ * @param {string[]} entities - primary_entities array
+ * @param {string[]} schemaTypes - schema type strings present on page
+ * @param {object[]} schemas - full page_schemas rows
+ * @param {string} searchIntent - from extraction
+ * @returns {object} { score, breakdown, aiIntents, tier }
+ */
+export function scorePage(page, headings, entities, schemaTypes, schemas, searchIntent) {
+  const bodyText = page.body_text || '';
+  const wordCount = page.word_count || bodyText.split(/\s+/).length;
+
+  const breakdown = {
+    entity_authority:   entityAuthorityScore(entities, headings, wordCount),
+    structured_claims:  structuredClaimsScore(bodyText, headings),
+    answer_density:     answerDensityScore(bodyText, wordCount),
+    qa_proximity:       qaProximityScore(headings, bodyText),
+    freshness:          freshnessScore(page, schemas),
+    schema_coverage:    schemaCoverageScore(schemaTypes),
+  };
+
+  // Weighted composite — entity authority and structured claims matter most for AI
+  const weights = {
+    entity_authority:   0.25,
+    structured_claims:  0.20,
+    answer_density:     0.20,
+    qa_proximity:       0.15,
+    freshness:          0.10,
+    schema_coverage:    0.10,
+  };
+
+  const score = Math.round(
+    Object.entries(weights).reduce((sum, [k, w]) => sum + breakdown[k] * w, 0)
+  );
+
+  const aiIntents = classifyAiIntent(headings, bodyText, searchIntent);
+
+  // Tier classification
+  let tier;
+  if (score >= 75) tier = 'excellent';
+  else if (score >= 55) tier = 'good';
+  else if (score >= 35) tier = 'needs_work';
+  else tier = 'poor';
+
+  return { score, breakdown, aiIntents, tier };
+}