seo-intel 1.5.46 → 1.5.52

package/analyses/aeo/ai-access.js

@@ -0,0 +1,210 @@
+/**
+ * AI Crawler Access — robots.txt analysis for AEO citability.
+ *
+ * The single biggest AEO failure mode is invisible to on-page scoring: a page
+ * can be perfectly structured and still be uncitable because robots.txt blocks
+ * the AI assistants' crawlers. This module parses robots.txt and reports which
+ * answer-engine / AI crawlers are allowed to read the site.
+ *
+ * `analyzeAiAccess` is a pure function (robots text → verdict). `fetchAiAccess`
+ * is the only thing that touches the network — callers that want to keep the
+ * AEO core network-free can fetch robots.txt themselves and pass the text in.
+ */
+
+// Known AI / answer-engine crawlers. `tier: citation` = used for live grounding
+// and citation in assistant answers (blocking these directly kills citability);
+// `tier: training` = primarily corpus/training crawlers (blocking hurts long-term
+// model familiarity but not live citation as hard).
+export const AI_BOTS = [
+  { ua: 'ClaudeBot',          vendor: 'Anthropic — Claude',             tier: 'citation' },
+  { ua: 'Claude-Web',         vendor: 'Anthropic — Claude',             tier: 'citation' },
+  { ua: 'anthropic-ai',       vendor: 'Anthropic — Claude',             tier: 'citation' },
+  { ua: 'GPTBot',             vendor: 'OpenAI — ChatGPT',               tier: 'citation' },
+  { ua: 'OAI-SearchBot',      vendor: 'OpenAI — ChatGPT Search',        tier: 'citation' },
+  { ua: 'ChatGPT-User',       vendor: 'OpenAI — ChatGPT (browse)',      tier: 'citation' },
+  { ua: 'PerplexityBot',      vendor: 'Perplexity',                     tier: 'citation' },
+  { ua: 'Perplexity-User',    vendor: 'Perplexity (browse)',            tier: 'citation' },
+  { ua: 'Google-Extended',    vendor: 'Google — Gemini / AI Overviews', tier: 'citation' },
+  { ua: 'Amazonbot',          vendor: 'Amazon — Alexa / Rufus',         tier: 'citation' },
+  { ua: 'DuckAssistBot',      vendor: 'DuckDuckGo — DuckAssist',        tier: 'citation' },
+  { ua: 'Applebot-Extended',  vendor: 'Apple Intelligence',             tier: 'training'  },
+  { ua: 'CCBot',              vendor: 'Common Crawl (feeds many LLMs)', tier: 'training'  },
+  { ua: 'Bytespider',         vendor: 'ByteDance — Doubao',             tier: 'training'  },
+  { ua: 'Meta-ExternalAgent', vendor: 'Meta AI',                        tier: 'training'  },
+  { ua: 'cohere-ai',          vendor: 'Cohere',                         tier: 'training'  },
+];
+
+// ── robots.txt parsing ──────────────────────────────────────────────────────
+
+/**
+ * Parse robots.txt into user-agent groups. A group is one-or-more consecutive
+ * `User-agent` lines followed by the rules that apply to them (per RFC 9309: a
+ * new User-agent after a rule line starts a fresh group).
+ */
+function parseRobots(txt) {
+  const groups = [];
+  let current = null;
+  let lastWasRule = false;
+
+  for (const raw of txt.split(/\r?\n/)) {
+    const line = raw.replace(/#.*$/, '').trim();
+    if (!line) continue;
+    const idx = line.indexOf(':');
+    if (idx === -1) continue;
+    const field = line.slice(0, idx).trim().toLowerCase();
+    const value = line.slice(idx + 1).trim();
+
+    if (field === 'user-agent') {
+      if (!current || lastWasRule) {
+        current = { agents: [], rules: [] };
+        groups.push(current);
+        lastWasRule = false;
+      }
+      current.agents.push(value.toLowerCase());
+    } else if (field === 'allow' || field === 'disallow') {
+      if (!current) { current = { agents: ['*'], rules: [] }; groups.push(current); }
+      current.rules.push({ type: field, path: value });
+      lastWasRule = true;
+    }
+  }
+  return groups;
+}
+
+/** Pick the group that governs `ua`: exact match wins, else the `*` group. */
+function groupFor(groups, ua) {
+  const lc = ua.toLowerCase();
+  return groups.find(g => g.agents.includes(lc))
+    || groups.find(g => g.agents.includes('*'))
+    || null;
+}
+
+/** Does this group block the site root (`/`)? `Disallow: /` blocks all; an
+ *  explicit `Allow: /` overrides; empty `Disallow:` means allow-all. */
+function blocksRoot(group) {
+  if (!group) return false;
+  let blocked = false;
+  for (const r of group.rules) {
+    if (r.type === 'disallow' && (r.path === '/' || r.path === '/*')) blocked = true;
+    else if (r.type === 'allow' && r.path === '/') blocked = false;
+  }
+  return blocked;
+}
+
+// ── Verdict ─────────────────────────────────────────────────────────────────
+
+/**
+ * Analyze robots.txt for AI-crawler access. Pure function.
+ *
+ * @param {string} robotsTxt - raw robots.txt body ('' if none/unavailable)
+ * @param {object} [opts] - { fetched: boolean } — false when robots couldn't be read
+ * @returns {object} {
+ *   score, blocked, verdict, blockedBots[], allowedCount, citationBlocked[],
+ *   aiTrainSignal, fetched, detail
+ * }
+ */
+export function analyzeAiAccess(robotsTxt, opts = {}) {
+  const fetched = opts.fetched ?? true;
+
+  // No robots.txt (or unreadable) → crawlers default-allow. Open, but flagged.
+  if (!fetched || !robotsTxt || !robotsTxt.trim()) {
+    return {
+      score: 100, blocked: false, verdict: 'open',
+      blockedBots: [], citationBlocked: [], allowedCount: AI_BOTS.length,
+      aiTrainSignal: null, fetched,
+      detail: fetched
+        ? 'No robots.txt rules — all AI crawlers default-allowed.'
+        : 'robots.txt unavailable — assuming open (crawlers default-allow when absent).',
+    };
+  }
+
+  const groups = parseRobots(robotsTxt);
+  const aiTrainSignal = /content-signal\s*:[^\n]*\bai-train\s*=\s*no\b/i.test(robotsTxt)
+    ? 'ai-train=no' : null;
+
+  const blockedBots = [];
+  for (const bot of AI_BOTS) {
+    if (blocksRoot(groupFor(groups, bot.ua))) blockedBots.push(bot);
+  }
+  const citationBlocked = blockedBots.filter(b => b.tier === 'citation');
+
+  let penalty = 0;
+  for (const b of blockedBots) penalty += b.tier === 'citation' ? 18 : 7;
+  if (aiTrainSignal) penalty += 8;
+  const score = Math.max(0, 100 - penalty);
+
+  // `blocked` = the hard-reality gate: any live citation crawler is locked out.
+  const blocked = citationBlocked.length > 0;
+  let verdict;
+  if (citationBlocked.length >= 3 || score < 40) verdict = 'blocked';
+  else if (blockedBots.length > 0 || aiTrainSignal) verdict = 'partial';
+  else verdict = 'open';
+
+  const names = citationBlocked.map(b => b.ua);
+  const detail = verdict === 'blocked'
+    ? `robots.txt blocks ${citationBlocked.length} answer-engine crawler(s) (${names.slice(0, 6).join(', ')}${names.length > 6 ? '…' : ''}) — these pages cannot be cited by the assistants developers actually use.`
+    : verdict === 'partial'
+      ? `Some AI crawlers blocked${aiTrainSignal ? ' and Content-Signal ai-train=no set' : ''}; live citation still possible but reduced.`
+      : 'All major AI crawlers allowed.';
+
+  return {
+    score, blocked, verdict,
+    blockedBots: blockedBots.map(b => ({ ua: b.ua, vendor: b.vendor, tier: b.tier })),
+    citationBlocked: citationBlocked.map(b => b.ua),
+    allowedCount: AI_BOTS.length - blockedBots.length,
+    aiTrainSignal, fetched, detail,
+  };
+}
+
+// ── Network fetch (the only I/O in this module) ─────────────────────────────
+
+/**
+ * Fetch + analyze robots.txt for a site. Best-effort: any failure degrades to
+ * an "assume open" verdict rather than throwing.
+ *
+ * @param {string} siteUrl - origin or any URL on the site
+ * @param {object} [opts] - { timeoutMs }
+ */
+export async function fetchAiAccess(siteUrl, opts = {}) {
+  const timeoutMs = opts.timeoutMs ?? 5000;
+  let origin;
+  try {
+    origin = new URL(/^https?:\/\//.test(siteUrl) ? siteUrl : `https://${siteUrl}`).origin;
+  } catch {
+    return { ...analyzeAiAccess('', { fetched: false }), origin: null };
+  }
+
+  try {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    const res = await fetch(`${origin}/robots.txt`, {
+      signal: controller.signal,
+      redirect: 'follow',
+      headers: { 'user-agent': 'seo-intel-aeo (+https://ukkometa.fi/seo-intel)' },
+    });
+    clearTimeout(timer);
+    if (!res.ok) {
+      return { ...analyzeAiAccess('', { fetched: true }), origin, httpStatus: res.status };
+    }
+    const txt = await res.text();
+    return { ...analyzeAiAccess(txt), origin, httpStatus: res.status };
+  } catch (e) {
+    return { ...analyzeAiAccess('', { fetched: false }), origin, error: e.message };
+  }
+}
+
+/**
+ * Fetch AI access for many domains in parallel. Returns Map<domain, verdict>.
+ * Domains can be bare ("docs.carbium.sh") or full URLs.
+ */
+export async function fetchAiAccessForDomains(domains, opts = {}) {
+  const map = new Map();
+  await Promise.all([...new Set(domains)].map(async (d) => {
+    const verdict = await fetchAiAccess(d, opts);
+    // key by bare host so it matches the `domains.domain` column
+    let host = d;
+    try { host = new URL(/^https?:\/\//.test(d) ? d : `https://${d}`).hostname; } catch { /* keep */ }
+    map.set(host, verdict);
+    map.set(host.replace(/^www\./, ''), verdict);
+  }));
+  return map;
+}

package/analyses/aeo/index.js

@@ -12,12 +12,16 @@ import { scorePage } from './scorer.js';
  *
  * @param {import('node:sqlite').DatabaseSync} db
  * @param {string} project
- * @param {object} opts - { includeCompetitors: boolean, log: function }
+ * @param {object} opts - { includeCompetitors, log, aiAccessByDomain }
+ *   aiAccessByDomain: optional Map<domain, verdict> from ai-access.js. Pure —
+ *   this function never touches the network; callers fetch robots.txt and pass
+ *   the verdicts in (preserves the "AEO runs on existing crawl data" contract).
  * @returns {object} { target: PageScore[], competitors: Map<domain, PageScore[]>, summary }
  */
 export function runAeoAnalysis(db, project, opts = {}) {
   const log = opts.log || console.log;
   const includeCompetitors = opts.includeCompetitors ?? true;
+  const aiAccessByDomain = opts.aiAccessByDomain || null;
 
   // ── Gather pages with body_text ─────────────────────────────────────────
   const roleFilter = includeCompetitors
@@ -75,8 +79,12 @@ export function runAeoAnalysis(db, project, opts = {}) {
       entities = JSON.parse(page.primary_entities || '[]');
     } catch { /* ignore */ }
 
+    const aiAccess = aiAccessByDomain
+      ? (aiAccessByDomain.get(page.domain) || aiAccessByDomain.get(page.domain.replace(/^www\./, '')) || null)
+      : null;
+
     const result = scorePage(
-      page, headings, entities, schemaTypes, pageSchemas, page.search_intent
+      page, headings, entities, schemaTypes, pageSchemas, page.search_intent, aiAccess
     );
 
     const pageScore = {
@@ -117,6 +125,20 @@ export function runAeoAnalysis(db, project, opts = {}) {
   const tierCounts = { excellent: 0, good: 0, needs_work: 0, poor: 0 };
   for (const r of targetResults) tierCounts[r.tier]++;
 
+  // Domain-level AI-access rollup (one verdict per target/owned domain).
+  const aiAccess = [];
+  if (aiAccessByDomain) {
+    const seen = new Set();
+    for (const r of targetResults) {
+      const key = r.domain.replace(/^www\./, '');
+      if (seen.has(key)) continue;
+      seen.add(key);
+      const v = aiAccessByDomain.get(r.domain) || aiAccessByDomain.get(key);
+      if (v) aiAccess.push({ domain: key, verdict: v.verdict, score: v.score, blocked: !!v.blocked, blockedBots: v.citationBlocked || [], detail: v.detail });
+    }
+  }
+  const gatedPages = targetResults.filter(r => r.aiAccessGated).length;
+
   const summary = {
     totalScored: scored,
     targetPages: targetResults.length,
@@ -126,6 +148,8 @@ export function runAeoAnalysis(db, project, opts = {}) {
     scoreDelta: avgTarget - avgComp,
     tierCounts,
     weakestSignals: getWeakestSignals(targetResults),
+    aiAccess,
+    gatedPages,
   };
 
   log(`  Scored ${scored} pages (${targetResults.length} target, ${compScores.length} competitor)`);
@@ -171,7 +195,7 @@ export function persistAeoScores(db, results) {
 /**
  * Feed low-scoring pages into Intelligence Ledger as citability_gap insights
  */
-export function upsertCitabilityInsights(db, project, targetResults) {
+export function upsertCitabilityInsights(db, project, targetResults, aiAccess = null) {
   const upsertStmt = db.prepare(`
     INSERT INTO insights (project, type, status, fingerprint, first_seen, last_seen, source_analysis_id, data)
     VALUES (?, 'citability_gap', 'active', ?, ?, ?, NULL, ?)
@@ -183,6 +207,27 @@ export function upsertCitabilityInsights(db, project, targetResults) {
   const ts = Date.now();
   db.exec('BEGIN');
   try {
+    // Domain-level AI-access blocks — the highest-severity citability gap: the
+    // page can't be cited at all because robots.txt locks out the crawlers.
+    if (Array.isArray(aiAccess)) {
+      for (const a of aiAccess) {
+        if (a.verdict === 'open') continue;
+        const fp = `ai-access::${a.domain}`;
+        const data = {
+          domain: a.domain,
+          score: a.score,
+          tier: a.blocked ? 'poor' : 'needs_work',
+          verdict: a.verdict,
+          blocked_crawlers: a.blockedBots,
+          weakest_signals: ['ai access'],
+          recommendation: a.blocked
+            ? `robots.txt blocks AI answer-engine crawlers (${(a.blockedBots || []).slice(0, 5).join(', ')}). Allow ClaudeBot / GPTBot / PerplexityBot / Google-Extended so the assistants developers use can read and cite ${a.domain}.`
+            : `${a.detail} Review robots.txt AI-crawler rules on ${a.domain}.`,
+        };
+        upsertStmt.run(project, fp, ts, ts, JSON.stringify(data));
+      }
+    }
+
     for (const r of targetResults) {
       if (r.score >= 60) continue; // only flag pages that need work
 
@@ -216,14 +261,12 @@ export function upsertCitabilityInsights(db, project, targetResults) {
 function getWeakestSignals(targetResults) {
   if (!targetResults.length) return [];
 
-  const signalTotals = {
-    entity_authority: 0, structured_claims: 0, answer_density: 0,
-    qa_proximity: 0, freshness: 0, schema_coverage: 0,
-  };
-
+  // Key-agnostic: ai_access only appears in the breakdown when robots data was
+  // supplied, so build the accumulator from whatever signals are present.
+  const signalTotals = {};
   for (const r of targetResults) {
     for (const [k, v] of Object.entries(r.breakdown)) {
-      signalTotals[k] += v;
+      signalTotals[k] = (signalTotals[k] || 0) + v;
     }
   }
 

package/analyses/aeo/rescore.js

@@ -0,0 +1,59 @@
+// rescore.js — re-check a single URL's AI citability after a fix.
+//
+// Read-only re-measurement (SEO Intel mutates nothing). Uses the lightweight
+// HTTP crawl — the RAW-HTML / "what bots see" lens: if an agent's fix is
+// server-rendered, the score moves; if it's JS-only, it correctly does not.
+// This closes the agent loop: act → re-score → see the delta.
+import { lightCrawl } from '../../crawler/light.js';
+
+const bucket = (s) => (s == null ? null : s >= 75 ? 'good' : s >= 55 ? 'needs-work' : s >= 35 ? 'weak' : 'poor');
+
+/**
+ * @param {object} db        open SQLite handle (for the stored baseline)
+ * @param {string} project   project name (scopes the baseline lookup)
+ * @param {string} url        the URL to re-score
+ * @param {object} [opts]     { log?: (msg)=>void }
+ * @returns {Promise<{url, status_code, lens, before, after, delta, improved, status_before, status_after, signals, note}>}
+ */
+export async function rescorePage(db, project, url, opts = {}) {
+  // Baseline = most recent stored citability score (from the last full crawl).
+  let before = null;
+  try {
+    const row = db.prepare(`
+      SELECT c.score
+      FROM citability_scores c
+      JOIN pages p ON p.id = c.page_id
+      JOIN domains d ON d.id = p.domain_id
+      WHERE d.project = ? AND p.url = ?
+      ORDER BY c.scored_at DESC
+      LIMIT 1
+    `).get(project, url);
+    before = row ? row.score : null;
+  } catch { /* citability_scores table may not exist yet */ }
+
+  const r = await lightCrawl(url, {
+    maxPages: 1,
+    includeCitability: true,
+    sameOrigin: true,
+    onProgress: opts.log,
+  });
+
+  const page = (r.pages || [])[0] || null;
+  const cite = page && page.citability && !page.citability.error ? page.citability : null;
+  const after = cite ? cite.score : null;
+  const delta = before != null && after != null ? after - before : null;
+
+  return {
+    url,
+    status_code: page ? page.status_code : null,
+    lens: 'raw-html', // bot's-eye view — what crawlers / LLMs actually see
+    before, // last full-crawl score (baseline)
+    after, // current raw / bot-visible score
+    delta,
+    improved: delta != null ? delta > 0 : null,
+    status_before: bucket(before),
+    status_after: bucket(after),
+    signals: cite ? cite.breakdown : null,
+    note: cite ? null : ((page && page.citability && page.citability.error) || 'no citability computed for this URL'),
+  };
+}

package/analyses/aeo/scorer.js

@@ -265,9 +265,12 @@ export function richResultProbability(headings, bodyText, schemaTypes, wordCount
  * @param {string[]} schemaTypes - schema type strings present on page
  * @param {object[]} schemas - full page_schemas rows
  * @param {string} searchIntent - from extraction
- * @returns {object} { score, breakdown, aiIntents, tier, richResult }
+ * @param {object} [aiAccess] - domain-level robots.txt verdict from ai-access.js
+ *   ({ score, blocked, verdict }). When provided, adds a 7th signal and applies
+ *   a hard-reality gate: pages whose AI crawlers are blocked can't be cited.
+ * @returns {object} { score, breakdown, aiIntents, tier, richResult, aiAccess, aiAccessGated }
  */
-export function scorePage(page, headings, entities, schemaTypes, schemas, searchIntent) {
+export function scorePage(page, headings, entities, schemaTypes, schemas, searchIntent, aiAccess = null) {
   const bodyText = page.body_text || '';
   const wordCount = page.word_count || bodyText.split(/\s+/).length;
 
@@ -280,20 +283,37 @@ export function scorePage(page, headings, entities, schemaTypes, schemas, search
     schema_coverage:    schemaCoverageScore(schemaTypes),
   };
 
-  // Weighted composite — entity authority and structured claims matter most for AI
-  const weights = {
-    entity_authority:   0.25,
-    structured_claims:  0.20,
-    answer_density:     0.20,
-    qa_proximity:       0.15,
-    freshness:          0.10,
-    schema_coverage:    0.10,
-  };
+  // Weighted composite — entity authority and structured claims matter most for AI.
+  // When a domain-level AI-access verdict is supplied we fold in a 7th signal and
+  // rebalance to sum to 1.0; otherwise the original 6-signal model is preserved
+  // exactly (callers without robots data are unaffected).
+  const hasAiAccess = aiAccess && typeof aiAccess.score === 'number';
+  let weights;
+  if (hasAiAccess) {
+    breakdown.ai_access = aiAccess.score;
+    weights = {
+      entity_authority: 0.22, structured_claims: 0.18, answer_density: 0.18,
+      qa_proximity: 0.14, freshness: 0.08, schema_coverage: 0.10, ai_access: 0.10,
+    };
+  } else {
+    weights = {
+      entity_authority: 0.25, structured_claims: 0.20, answer_density: 0.20,
+      qa_proximity: 0.15, freshness: 0.10, schema_coverage: 0.10,
+    };
+  }
 
-  const score = Math.round(
+  let score = Math.round(
     Object.entries(weights).reduce((sum, [k, w]) => sum + breakdown[k] * w, 0)
   );
 
+  // Hard-reality gate: if AI assistants are blocked from fetching the page,
+  // on-page quality is moot — they cannot cite what they cannot read.
+  let aiAccessGated = false;
+  if (aiAccess && aiAccess.blocked) {
+    score = Math.min(score, 30);
+    aiAccessGated = true;
+  }
+
   const aiIntents = classifyAiIntent(headings, bodyText, searchIntent);
   const richResult = richResultProbability(headings, bodyText, schemaTypes, wordCount);
 
@@ -304,5 +324,8 @@ export function scorePage(page, headings, entities, schemaTypes, schemas, search
   else if (score >= 35) tier = 'needs_work';
   else tier = 'poor';
 
-  return { score, breakdown, aiIntents, tier, richResult };
+  return {
+    score, breakdown, aiIntents, tier, richResult,
+    ...(hasAiAccess ? { aiAccess: { score: aiAccess.score, verdict: aiAccess.verdict, blocked: !!aiAccess.blocked }, aiAccessGated } : {}),
+  };
 }