seo-intel 1.5.21 → 1.5.23

package/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## 1.5.23 (2026-04-23)
+
+### Technical Audit — extended-data checks
+- New `seo-intel tech-audit <project>` command — runs technical SEO validation off the crawl DB
+- Findings: title length, meta description length, noindex detection (meta + `X-Robots-Tag`), redirect chains, indexable-but-not-in-sitemap, redirect-target cross-reference
+- `--head` pass runs bounded-concurrency HEAD checks against sitemap URLs (flags 3XX / 4XX)
+- Gated under the `extended-data` banner — same tier surface as other audit extensions
+
+### Crawler — new signal capture
+- Captures final URL after redirects (`page.url()`)
+- Walks the Playwright redirect chain and persists it as JSON
+- Reads `X-Robots-Tag` response header (no-index detection now covers meta **and** header)
+- Sitemap URLs discovered during crawl are persisted to a new `sitemap_urls` table
+
+### Schema
+- `pages` table gains `final_url`, `redirect_chain`, `x_robots_tag` (additive `ALTER TABLE`, safe on existing DBs)
+- New `sitemap_urls` table for the HEAD-check inventory pass
+
+### Accumulated since last changelog (1.5.3–1.5.22)
+- LM Studio extraction backend + auto-discovery
+- Scan command auto-resolves `www` when bare domain is unreachable
+- Intelligence modules: intent scores, schema impact, rich-result probability
+- Nav-link detection for external sites + missing-www redirect warning
+- Solo audit prompt rewrite — no more hallucinated competitors
+- Scan/serve/dashboard resilience fixes
+
 ## 1.5.2 (2026-04-11)
 
 ### Unified Export

package/analyses/aeo/scorer.js

@@ -123,7 +123,7 @@ function answerDensityScore(bodyText, wordCount) {
 }
 
 // ── Q&A proximity ──────────────────────────────────────────────────────────
-function qaProximityScore(headings, bodyText) {
+function qaProximityScore(headings, bodyText, schemaTypes) {
   if (!headings.length || !bodyText) return 0;
 
   const questionHeadings = headings.filter(h =>
@@ -138,8 +138,8 @@ function qaProximityScore(headings, bodyText) {
   const qRatio = questionHeadings.length / headings.filter(h => h.level >= 2).length;
   score += Math.min(qRatio * 60, 40);
 
-  // FAQ schema present? Huge bonus
-  score += 30;
+  // FAQ schema present? Huge bonus — only award if schema actually exists
+  if (Array.isArray(schemaTypes) && schemaTypes.includes('FAQPage')) score += 30;
 
   // Heading density (one H2/H3 per ~300 words is ideal)
   const h2h3Count = headings.filter(h => h.level >= 2 && h.level <= 3).length;
@@ -201,6 +201,59 @@ function classifyAiIntent(headings, bodyText, searchIntent) {
   return intents;
 }
 
+// ── Rich Result Probability Predictor ──────────────────────────────────────
+
+/**
+ * Estimate probability of achieving rich results based on page signals.
+ * Returns per-type probability (FAQ, HowTo, Article) and overall best chance.
+ *
+ * @param {object[]} headings
+ * @param {string} bodyText
+ * @param {string[]} schemaTypes - current schema on page
+ * @param {number} wordCount
+ * @returns {object} { faq, howto, article, best: { type, probability } }
+ */
+export function richResultProbability(headings, bodyText, schemaTypes, wordCount) {
+  const text = (bodyText || '').toLowerCase();
+
+  // ── FAQ Rich Result ──
+  const questionHeadings = headings.filter(h => h.level >= 2 && h.level <= 3 && QUESTION_RE.test(h.text));
+  let faq = 0;
+  if (schemaTypes.includes('FAQPage')) faq += 45;
+  if (questionHeadings.length >= 3) faq += 25;
+  else if (questionHeadings.length >= 1) faq += 10;
+  if (wordCount >= 500) faq += 10;
+  if (wordCount >= 1500) faq += 5;
+  const paras = text.split(/\n\s*\n/).filter(p => p.trim()).length;
+  if (paras >= 3 && wordCount / paras < 150) faq += 15;
+  faq = Math.min(faq, 95);
+
+  // ── HowTo Rich Result ──
+  let howto = 0;
+  if (schemaTypes.includes('HowTo')) howto += 45;
+  if (IMPL_RE.test(text)) howto += 20;
+  const steps = (text.match(/(?:^|\n)\s*(?:\d+[.)]\s|step\s+\d)/gm) || []).length;
+  if (steps >= 3) howto += 20;
+  else if (steps >= 1) howto += 8;
+  if (wordCount >= 300) howto += 10;
+  howto = Math.min(howto, 95);
+
+  // ── Article Rich Result ──
+  const articleSchemas = ['Article', 'TechArticle', 'BlogPosting', 'NewsArticle'];
+  let article = 0;
+  if (schemaTypes.some(t => articleSchemas.includes(t))) article += 35;
+  if (wordCount >= 800) article += 20;
+  else if (wordCount >= 400) article += 10;
+  if (headings.filter(h => h.level === 2).length >= 2) article += 15;
+  if (schemaTypes.includes('BreadcrumbList')) article += 10;
+  article = Math.min(article, 95);
+
+  const results = { faq, howto, article };
+  const best = Object.entries(results).sort((a, b) => b[1] - a[1])[0];
+
+  return { ...results, best: { type: best[0], probability: best[1] } };
+}
+
 // ── Main scorer ────────────────────────────────────────────────────────────
 
 /**
@@ -212,7 +265,7 @@ function classifyAiIntent(headings, bodyText, searchIntent) {
  * @param {string[]} schemaTypes - schema type strings present on page
  * @param {object[]} schemas - full page_schemas rows
  * @param {string} searchIntent - from extraction
- * @returns {object} { score, breakdown, aiIntents, tier }
+ * @returns {object} { score, breakdown, aiIntents, tier, richResult }
  */
 export function scorePage(page, headings, entities, schemaTypes, schemas, searchIntent) {
   const bodyText = page.body_text || '';
@@ -222,7 +275,7 @@ export function scorePage(page, headings, entities, schemaTypes, schemas, search
     entity_authority:   entityAuthorityScore(entities, headings, wordCount),
     structured_claims:  structuredClaimsScore(bodyText, headings),
     answer_density:     answerDensityScore(bodyText, wordCount),
-    qa_proximity:       qaProximityScore(headings, bodyText),
+    qa_proximity:       qaProximityScore(headings, bodyText, schemaTypes),
     freshness:          freshnessScore(page, schemas),
     schema_coverage:    schemaCoverageScore(schemaTypes),
   };
@@ -242,6 +295,7 @@ export function scorePage(page, headings, entities, schemaTypes, schemas, search
   );
 
   const aiIntents = classifyAiIntent(headings, bodyText, searchIntent);
+  const richResult = richResultProbability(headings, bodyText, schemaTypes, wordCount);
 
   // Tier classification
   let tier;
@@ -250,5 +304,5 @@ export function scorePage(page, headings, entities, schemaTypes, schemas, search
   else if (score >= 35) tier = 'needs_work';
   else tier = 'poor';
 
-  return { score, breakdown, aiIntents, tier };
+  return { score, breakdown, aiIntents, tier, richResult };
 }

package/analyses/templates/index.js

@@ -44,7 +44,7 @@ export async function runTemplatesAnalysis(project, opts = {}) {
   if (!config) throw new Error(`Project "${project}" not found. Run: seo-intel setup`);
 
   const targetDomain = config.target.domain;
-  const targetUrl = config.target.url || `https://${targetDomain}`;
+  const targetUrl = (config.target.url || `https://${targetDomain}`).replace(/\/+$/, '');
 
   log(`\n  Target: ${targetDomain}`);
 

package/analysis/prompt-builder.js

@@ -25,6 +25,171 @@
  * @param {object} params.context     - project context (industry, audience, goals)
  */
 export function buildAnalysisPrompt({ project, target, competitors, keywordMatrix, headingStructure, context }) {
+  const isSolo = !competitors || competitors.length === 0;
+  return isSolo
+    ? buildSoloPrompt({ target, keywordMatrix, headingStructure, context })
+    : buildCompetitivePrompt({ target, competitors, keywordMatrix, headingStructure, context });
+}
+
+// ── Solo audit prompt (no competitors) ─────────────────────────────────────
+
+function buildSoloPrompt({ target, keywordMatrix, headingStructure, context }) {
+  return `
+# SEO Site Audit — ${context.siteName}
+
+You are an expert SEO strategist performing a solo site audit. You have ONLY the crawled site data below — no competitor data.
+
+**CRITICAL RULES:**
+- You have ZERO competitor data. Do NOT invent, hallucinate, or reference any competitor domains.
+- Never fill "covered_by" with domain names you were not given.
+- Base keyword and content recommendations on: (1) the crawled site data, (2) your knowledge of the "${context.industry}" industry and what audiences search for.
+- Label all industry-knowledge suggestions as "industry research" — not "data-driven".
+- Every URL slug you suggest must be a real path (e.g. "/blog/how-to-x"), never "/undefined".
+
+---
+
+## CONTEXT
+
+**Site:** ${context.siteName} (${context.url})
+**Industry:** ${context.industry}
+**Target audience:** ${context.audience}
+**Business goal:** ${context.goal}
+**Current SEO maturity:** ${context.maturity || 'early stage'}
+
+### Site Architecture
+${context.site_architecture ? `
+${context.site_architecture.note}
+
+Available publishing properties:
+${context.site_architecture.properties.map(p =>
+  `- **${p.id}** (${p.url}, platform: ${p.platform})\n  Best for: ${p.best_for}\n  Difficulty: ${p.difficulty}${p.seo_note ? `\n  SEO note: ${p.seo_note}` : ''}`
+).join('\n')}
+` : 'No site architecture configured — recommend generic URL slugs.'}
+
+---
+
+## SITE DATA
+
+${formatSiteSummary(target)}
+
+### Pages crawled: ${target.page_count || target.pageCount || 0}
+### Keyword coverage:
+${formatKeywordTable(keywordMatrix, target.domain)}
+
+### Heading structure:
+${formatHeadings(headingStructure, target.domain)}
+
+---
+
+## ANALYSIS TASKS
+
+### 1. KEYWORD OPPORTUNITIES
+- Based on the site's existing content and the "${context.industry}" industry, identify 5-10 keyword phrases the site should target
+- For each: search intent, estimated search demand (low/medium/high), difficulty, and whether to add to an existing page or create a new one
+- Focus on keywords that match the site's actual product/service — no speculative gaps
+
+### 2. LONG-TAIL OPPORTUNITIES
+- Generate 10-20 specific long-tail phrases (3-6 words) from the site's content themes
+- Focus on: question queries, feature queries, use-case queries
+- For each: intent, page type, priority
+- Weight toward commercial intent
+
+### 3. CONTENT EXPANSION
+- Topic areas the site should cover based on industry norms and audience needs
+- Do NOT reference competitor domains — use "industry standard" or "common in ${context.industry}" instead
+- For each: why it matters for this audience, suggested format, suggested title
+
+### 4. QUICK WINS (existing pages to improve)
+- Pages with thin content, missing structure, or weak metadata
+- Only reference pages that appear in the crawled data above
+- For each: specific fix, estimated impact
+
+### 5. NEW PAGE SUGGESTIONS
+- Specific new pages to create based on keyword opportunities
+- For each: URL slug (real path like /blog/topic), title, target keyword, content angle
+
+### 6. TECHNICAL SEO AUDIT
+- Schema markup opportunities (FAQ, HowTo, Product, etc.)
+- Meta description quality assessment
+- H1/heading structure recommendations
+- Do NOT compare to competitors — assess against SEO best practices
+
+### 7. MARKET POSITIONING
+- Based on the site's content and industry, what positioning should this site own?
+- What audience need is underserved in this space?
+- What is the site's clearest differentiator from its current content?
+
+---
+
+## OUTPUT SCHEMA
+
+Respond ONLY with valid JSON in this exact structure:
+
+{
+  "keyword_gaps": [
+    {
+      "keyword": "string — 2-4 word SEO phrase",
+      "intent": "informational|commercial|navigational|transactional",
+      "search_demand": "low|medium|high",
+      "difficulty": "low|medium|high",
+      "suggested_action": "add_to_existing|new_page",
+      "suggested_page": "string — URL path like /blog/topic or existing page URL",
+      "priority": "high|medium|low",
+      "source": "site_content|industry_research"
+    }
+  ],
+  "long_tails": [
+    {
+      "phrase": "string",
+      "intent": "string",
+      "page_type": "blog|landing|doc|faq|comparison|glossary",
+      "priority": "high|medium|low",
+      "notes": "string"
+    }
+  ],
+  "content_gaps": [
+    {
+      "topic": "string",
+      "why_it_matters": "string",
+      "format": "blog|comparison|use_case|glossary|how_to|landing",
+      "suggested_title": "string"
+    }
+  ],
+  "quick_wins": [
+    {
+      "page": "string — URL from crawled data",
+      "issue": "string",
+      "fix": "string",
+      "impact": "high|medium|low"
+    }
+  ],
+  "new_pages": [
+    {
+      "title": "string",
+      "target_keyword": "string",
+      "content_angle": "string",
+      "why": "string",
+      "priority": "high|medium|low"
+    }
+  ],
+  "technical_gaps": [
+    {
+      "gap": "string",
+      "fix": "string"
+    }
+  ],
+  "positioning": {
+    "market_context": "string — 2-3 sentences on the industry landscape",
+    "open_angle": "string — what positioning this site should own",
+    "target_differentiator": "string — clearest differentiator from current content"
+  }
+}
+`.trim();
+}
+
+// ── Competitive prompt (with competitors) ──────────────────────────────────
+
+function buildCompetitivePrompt({ target, competitors, keywordMatrix, headingStructure, context }) {
   return `
 # SEO Competitive Intelligence Analysis — ${context.siteName}
 
@@ -59,7 +224,7 @@ For each content recommendation, rank ALL available properties as placement opti
 
 ${formatSiteSummary(target)}
 
-### Pages crawled: ${target.pageCount}
+### Pages crawled: ${target.page_count || target.pageCount || 0}
 ### Keyword coverage:
 ${formatKeywordTable(keywordMatrix, target.domain)}
 
@@ -218,7 +383,7 @@ Respond ONLY with valid JSON in this exact structure:
 function formatSiteSummary(site) {
   return `
 - Domain: ${site.domain}
-- Pages crawled: ${site.pageCount || 0}
+- Pages crawled: ${site.page_count || site.pageCount || 0}
 - Avg word count: ${Math.round(site.avg_word_count || 0)}
 - Product types detected: ${site.product_types || 'unknown'}
 - Pricing model: ${site.pricing_tiers || 'unknown'}

package/analysis/technical-audit.js

@@ -0,0 +1,177 @@
+/**
+ * Technical SEO Audit — reads crawl data from the DB and produces findings.
+ *
+ * Extended-data checks (gated via lib/gate.js `extended-data`):
+ *   1. Title length (>60 warn, missing err)
+ *   2. Meta description length (>160 warn, >320 err, missing err)
+ *   3. Noindex detection (meta robots OR X-Robots-Tag header)
+ *   4. Indexable pages missing from sitemap (set diff)
+ *   5. Redirect chain surfacing (uses final_url + redirect_chain columns)
+ *   6. Canonical points to a redirect target (uses redirect_chain + technical)
+ *
+ * Additional optional pass (network-heavy, must be explicitly enabled):
+ *   - Sitemap HEAD check: flags 3XX / 4XX URLs in the sitemap itself.
+ */
+
+import { gateSection } from '../lib/gate.js';
+import { headCheckAll } from '../crawler/sitemap.js';
+import {
+  getSitemapUrlsForDomain,
+  updateSitemapHeadResult,
+} from '../db/db.js';
+
+const TITLE_WARN = 60;
+const DESC_WARN = 160;
+const DESC_ERR = 320;
+
+/**
+ * Run the audit for a single domain. Returns { findings: [], stats: {} }.
+ * Pass { runSitemapHead: true } to run the HEAD pass over the sitemap inventory.
+ * Gated: the actual checks only run when the `extended-data` gate is open.
+ */
+export async function runTechnicalAudit(db, { project, domain, runSitemapHead = false, sitemapConcurrency = 6 } = {}) {
+  if (!gateSection('extended-data')) {
+    return { gated: true, findings: [], stats: {} };
+  }
+
+  const domainRow = db.prepare(
+    'SELECT id, domain FROM domains WHERE domain = ? AND project = ?'
+  ).get(domain, project);
+  if (!domainRow) {
+    return { gated: false, findings: [], stats: {}, error: `domain not found: ${domain}` };
+  }
+  const domainId = domainRow.id;
+
+  const findings = [];
+
+  // ── Page-level checks (read from pages + technical) ──
+  const pages = db.prepare(`
+    SELECT
+      p.id, p.url, p.final_url, p.redirect_chain, p.x_robots_tag,
+      p.is_indexable, p.status_code, p.title, p.meta_desc,
+      t.has_canonical
+    FROM pages p
+    LEFT JOIN technical t ON t.page_id = p.id
+    WHERE p.domain_id = ?
+  `).all(domainId);
+
+  const redirectTargets = new Set();
+
+  for (const p of pages) {
+    // 1. Title length
+    if (!p.title) {
+      findings.push({ type: 'title_missing', severity: 'error', url: p.url, details: 'No <title>' });
+    } else if (p.title.length > TITLE_WARN) {
+      findings.push({ type: 'title_too_long', severity: 'warn', url: p.url, details: `${p.title.length}/${TITLE_WARN}` });
+    }
+
+    // 2. Meta description length
+    if (!p.meta_desc) {
+      findings.push({ type: 'meta_desc_missing', severity: 'error', url: p.url, details: 'No meta description' });
+    } else if (p.meta_desc.length > DESC_ERR) {
+      findings.push({ type: 'meta_desc_too_long', severity: 'error', url: p.url, details: `${p.meta_desc.length}/${DESC_ERR}` });
+    } else if (p.meta_desc.length > DESC_WARN) {
+      findings.push({ type: 'meta_desc_too_long', severity: 'warn', url: p.url, details: `${p.meta_desc.length}/${DESC_WARN}` });
+    }
+
+    // 3. Noindex (meta OR X-Robots-Tag) — informational only (valid decision, not error)
+    const xrt = (p.x_robots_tag || '').toLowerCase();
+    if (xrt.includes('noindex') && p.is_indexable === 0) {
+      findings.push({ type: 'noindex_header', severity: 'info', url: p.url, details: `X-Robots-Tag: ${p.x_robots_tag}` });
+    }
+
+    // 5. Redirect chain
+    let chain = [];
+    try { chain = p.redirect_chain ? JSON.parse(p.redirect_chain) : []; } catch { chain = []; }
+    if (chain.length > 0) {
+      const finalUrl = p.final_url || p.url;
+      findings.push({
+        type: 'redirect_chain',
+        severity: chain.length >= 2 ? 'warn' : 'info',
+        url: p.url,
+        details: `${chain.length} hop(s) → ${finalUrl}`,
+        hops: chain,
+        finalUrl,
+      });
+      redirectTargets.add(finalUrl);
+    }
+  }
+
+  // 6. Canonical-points-to-redirect — requires a second pass with canonical URLs.
+  // `technical.has_canonical` is a boolean; the canonical URL itself isn't stored.
+  // For now we surface the set of redirect *targets* so reviewers can cross-reference.
+  if (redirectTargets.size > 0) {
+    findings.push({
+      type: 'redirect_targets_summary',
+      severity: 'info',
+      details: `${redirectTargets.size} redirect target URL(s) — review canonical tags pointing to any of these`,
+      urls: [...redirectTargets],
+    });
+  }
+
+  // 4. Indexable-but-not-in-sitemap (set diff)
+  const sitemapRows = getSitemapUrlsForDomain(db, domainId);
+  const sitemapSet = new Set(sitemapRows.map(r => r.url));
+  const missing = pages.filter(p =>
+    p.is_indexable === 1 &&
+    p.status_code === 200 &&
+    !sitemapSet.has(p.url) &&
+    !sitemapSet.has(p.final_url || '')
+  );
+  for (const m of missing) {
+    findings.push({
+      type: 'indexable_missing_from_sitemap',
+      severity: 'warn',
+      url: m.url,
+      details: 'Page is indexable (200) but not declared in sitemap',
+    });
+  }
+
+  // Optional: run HEAD pass over sitemap inventory
+  let sitemapHeadStats = null;
+  if (runSitemapHead && sitemapRows.length > 0) {
+    const uncheckedRows = sitemapRows.filter(r => r.head_checked_at === null);
+    const rowsToCheck = uncheckedRows.length ? uncheckedRows : sitemapRows;
+    let ok = 0, redirected = 0, broken = 0, errored = 0;
+    await headCheckAll(rowsToCheck, {
+      concurrency: sitemapConcurrency,
+      onResult: (row, res) => {
+        updateSitemapHeadResult(db, row.id, res);
+        if (!res.status) errored++;
+        else if (res.status >= 200 && res.status < 300) ok++;
+        else if (res.status >= 300 && res.status < 400) {
+          redirected++;
+          findings.push({
+            type: 'sitemap_redirect',
+            severity: 'warn',
+            url: row.url,
+            details: `Sitemap URL returns ${res.status}${res.location ? ` → ${res.location}` : ''}`,
+          });
+        }
+        else if (res.status >= 400) {
+          broken++;
+          findings.push({
+            type: 'sitemap_broken',
+            severity: 'error',
+            url: row.url,
+            details: `Sitemap URL returns ${res.status}`,
+          });
+        }
+      },
+    });
+    sitemapHeadStats = { checked: rowsToCheck.length, ok, redirected, broken, errored };
+  }
+
+  const stats = {
+    pages: pages.length,
+    sitemap_urls: sitemapRows.length,
+    findings_total: findings.length,
+    findings_by_severity: findings.reduce((acc, f) => {
+      acc[f.severity] = (acc[f.severity] || 0) + 1;
+      return acc;
+    }, {}),
+    sitemap_head: sitemapHeadStats,
+  };
+
+  return { gated: false, findings, stats };
+}