npm - seo-intel - Versions diffs - 1.0.0 - Mend

seo-intel 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (46) hide show

package/.env.example +41 -0
package/LICENSE +75 -0
package/README.md +243 -0
package/Start SEO Intel.bat +9 -0
package/Start SEO Intel.command +8 -0
package/cli.js +3727 -0
package/config/example.json +29 -0
package/config/setup-wizard.js +522 -0
package/crawler/index.js +566 -0
package/crawler/robots.js +103 -0
package/crawler/sanitize.js +124 -0
package/crawler/schema-parser.js +168 -0
package/crawler/sitemap.js +103 -0
package/crawler/stealth.js +393 -0
package/crawler/subdomain-discovery.js +341 -0
package/db/db.js +213 -0
package/db/schema.sql +120 -0
package/exports/competitive.js +186 -0
package/exports/heuristics.js +67 -0
package/exports/queries.js +197 -0
package/exports/suggestive.js +230 -0
package/exports/technical.js +180 -0
package/exports/templates.js +77 -0
package/lib/gate.js +204 -0
package/lib/license.js +369 -0
package/lib/oauth.js +432 -0
package/lib/updater.js +324 -0
package/package.json +68 -0
package/reports/generate-html.js +6194 -0
package/reports/generate-site-graph.js +949 -0
package/reports/gsc-loader.js +190 -0
package/scheduler.js +142 -0
package/seo-audit.js +619 -0
package/seo-intel.png +0 -0
package/server.js +602 -0
package/setup/ROADMAP.md +109 -0
package/setup/checks.js +483 -0
package/setup/config-builder.js +227 -0
package/setup/engine.js +65 -0
package/setup/installers.js +197 -0
package/setup/models.js +328 -0
package/setup/openclaw-bridge.js +329 -0
package/setup/validator.js +395 -0
package/setup/web-routes.js +688 -0
package/setup/wizard.html +2920 -0
package/start-seo-intel.sh +8 -0

package/crawler/sanitize.js ADDED Viewed

@@ -0,0 +1,124 @@
+/**
+ * Sanitize scraped text before sending to any AI model.
+ * Defense against prompt injection from malicious web content.
+ */
+import TurndownService from 'turndown';
+// Patterns that look like prompt injection attempts
+const INJECTION_PATTERNS = [
+  /ignore\s+(previous|above|all|prior)\s+instructions?/gi,
+  /forget\s+(everything|all|prior|previous)/gi,
+  /you\s+are\s+now\s+a/gi,
+  /new\s+instructions?:/gi,
+  /system\s*:/gi,
+  /assistant\s*:/gi,
+  /\[INST\]/gi,
+  /\[\/INST\]/gi,
+  /<\|im_start\|>/gi,
+  /<\|im_end\|>/gi,
+  /###\s*(instruction|system|human|assistant)/gi,
+];
+/**
+ * Strip HTML tags and extract clean visible text.
+ */
+export function stripHtml(html) {
+  return html
+    // Remove script + style blocks entirely
+    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+    .replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ')
+    // Remove HTML comments
+    .replace(/<!--[\s\S]*?-->/g, ' ')
+    // Remove remaining tags
+    .replace(/<[^>]+>/g, ' ')
+    // Decode common entities
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    // Collapse whitespace
+    .replace(/\s{3,}/g, '\n\n')
+    .trim();
+}
+/**
+ * Remove prompt injection patterns from text.
+ * Replaces suspicious phrases with [REMOVED].
+ */
+export function removeInjections(text) {
+  let cleaned = text;
+  for (const pattern of INJECTION_PATTERNS) {
+    cleaned = cleaned.replace(pattern, '[REMOVED]');
+  }
+  return cleaned;
+}
+/**
+ * Truncate text to a safe token limit (rough estimate: 4 chars/token).
+ */
+export function truncate(text, maxTokens = 2000) {
+  const maxChars = maxTokens * 4;
+  if (text.length <= maxChars) return text;
+  return text.slice(0, maxChars) + '\n[TRUNCATED]';
+}
+/**
+ * Full sanitization pipeline for scraped content.
+ */
+export function sanitize(rawHtml, maxTokens = 2000) {
+  const text = stripHtml(rawHtml);
+  const cleaned = removeInjections(text);
+  return truncate(cleaned, maxTokens);
+}
+/**
+ * Extract only text from specific CSS selectors (safer than full page).
+ * Pass the Playwright page object and a list of selectors.
+ */
+export async function extractSelective(page, selectors = ['h1', 'h2', 'h3', 'p', 'li', 'title']) {
+  const parts = [];
+  for (const sel of selectors) {
+    try {
+      const texts = await page.$$eval(sel, els => els.map(e => e.innerText?.trim()).filter(Boolean));
+      parts.push(...texts);
+    } catch {}
+  }
+  return removeInjections(parts.join('\n').replace(/\s{3,}/g, '\n\n').trim());
+}
+/**
+ * Extract page content as clean Markdown via Turndown.
+ * Tries <main> or <article> first for focused content, falls back to <body>.
+ * Strips nav/footer/header/aside/script/style before conversion.
+ */
+const turndown = new TurndownService({
+  headingStyle: 'atx',
+  codeBlockStyle: 'fenced',
+  bulletListMarker: '-',
+});
+// Skip images in markdown output (no value for SEO text extraction)
+turndown.addRule('removeImages', { filter: 'img', replacement: () => '' });
+export async function extractAsMarkdown(page) {
+  // Get focused content HTML — prefer <main> or <article>, fall back to <body>
+  const html = await page.evaluate(() => {
+    const el = document.querySelector('main') || document.querySelector('article') || document.body;
+    if (!el) return '';
+    // Clone to avoid mutating the live DOM
+    const clone = el.cloneNode(true);
+    // Strip non-content elements
+    for (const tag of ['nav', 'footer', 'header', 'aside', 'script', 'style', 'noscript', 'iframe']) {
+      clone.querySelectorAll(tag).forEach(n => n.remove());
+    }
+    return clone.innerHTML;
+  }).catch(() => '');
+  if (!html) return '';
+  const md = turndown.turndown(html);
+  const cleaned = removeInjections(md);
+  return truncate(cleaned, 2000);
+}

package/crawler/schema-parser.js ADDED Viewed

@@ -0,0 +1,168 @@
+/**
+ * JSON-LD Schema Parser
+ *
+ * Extracts structured data from <script type="application/ld+json"> blocks.
+ * Parses @type, name, description, aggregateRating, offers, author, dates,
+ * images — everything Google actually uses for rich results.
+ *
+ * Returns normalized objects ready for page_schemas table insertion.
+ */
+/**
+ * Parse all JSON-LD blocks from raw HTML string.
+ * Works on raw HTML (no DOM needed) — runs during crawl before Qwen extraction.
+ *
+ * @param {string} html - Raw HTML string
+ * @returns {Array<Object>} Parsed schema objects
+ */
+export function parseJsonLd(html) {
+  if (!html) return [];
+  const blocks = extractJsonLdBlocks(html);
+  const schemas = [];
+  for (const block of blocks) {
+    try {
+      const parsed = JSON.parse(block);
+      // Handle @graph arrays (common in Yoast, Schema.org generators)
+      if (parsed['@graph'] && Array.isArray(parsed['@graph'])) {
+        for (const item of parsed['@graph']) {
+          const normalized = normalizeSchema(item);
+          if (normalized) schemas.push(normalized);
+        }
+      } else if (Array.isArray(parsed)) {
+        // Some sites output an array of schemas
+        for (const item of parsed) {
+          const normalized = normalizeSchema(item);
+          if (normalized) schemas.push(normalized);
+        }
+      } else {
+        const normalized = normalizeSchema(parsed);
+        if (normalized) schemas.push(normalized);
+      }
+    } catch {
+      // Malformed JSON-LD — skip silently
+    }
+  }
+  return schemas;
+}
+/**
+ * Extract raw JSON strings from <script type="application/ld+json"> tags.
+ * Uses regex — no DOM parser needed.
+ */
+function extractJsonLdBlocks(html) {
+  const regex = /<script[^>]*type\s*=\s*["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
+  const blocks = [];
+  let match;
+  while ((match = regex.exec(html)) !== null) {
+    const content = match[1].trim();
+    if (content) blocks.push(content);
+  }
+  return blocks;
+}
+/**
+ * Normalize a single JSON-LD object into a flat structure for DB storage.
+ */
+function normalizeSchema(obj) {
+  if (!obj || typeof obj !== 'object') return null;
+  const type = resolveType(obj['@type']);
+  if (!type) return null;
+  const rating = extractRating(obj);
+  const offers = extractOffers(obj);
+  const author = extractAuthor(obj);
+  const image = extractImage(obj);
+  return {
+    type,
+    name: str(obj.name) || str(obj.headline) || null,
+    description: str(obj.description) || null,
+    rating: rating.value,
+    ratingCount: rating.count,
+    price: offers.price,
+    currency: offers.currency,
+    author,
+    datePublished: str(obj.datePublished) || null,
+    dateModified: str(obj.dateModified) || null,
+    imageUrl: image,
+    raw: obj,
+  };
+}
+// ── Extractors ───────────────────────────────────────────────────────────────
+function resolveType(t) {
+  if (!t) return null;
+  if (Array.isArray(t)) return t[0]; // take first type
+  return String(t);
+}
+function extractRating(obj) {
+  const ar = obj.aggregateRating;
+  if (!ar) return { value: null, count: null };
+  return {
+    value: parseFloat(ar.ratingValue) || null,
+    count: parseInt(ar.reviewCount || ar.ratingCount) || null,
+  };
+}
+function extractOffers(obj) {
+  const offers = obj.offers;
+  if (!offers) return { price: null, currency: null };
+  // Single offer
+  if (offers.price !== undefined) {
+    return {
+      price: String(offers.price),
+      currency: str(offers.priceCurrency) || null,
+    };
+  }
+  // Offer with priceRange
+  if (offers.priceRange) {
+    return { price: str(offers.priceRange), currency: str(offers.priceCurrency) || null };
+  }
+  // AggregateOffer
+  if (offers.lowPrice !== undefined || offers.highPrice !== undefined) {
+    const low = offers.lowPrice ?? '?';
+    const high = offers.highPrice ?? '?';
+    return {
+      price: `${low}-${high}`,
+      currency: str(offers.priceCurrency) || null,
+    };
+  }
+  // Array of offers — take first
+  if (Array.isArray(offers) && offers.length > 0) {
+    return extractOffers({ offers: offers[0] });
+  }
+  return { price: null, currency: null };
+}
+function extractAuthor(obj) {
+  const author = obj.author;
+  if (!author) return null;
+  if (typeof author === 'string') return author;
+  if (Array.isArray(author)) return author.map(a => str(a.name) || str(a)).filter(Boolean).join(', ');
+  return str(author.name) || null;
+}
+function extractImage(obj) {
+  const img = obj.image;
+  if (!img) return null;
+  if (typeof img === 'string') return img;
+  if (Array.isArray(img)) return typeof img[0] === 'string' ? img[0] : img[0]?.url || null;
+  return str(img.url) || null;
+}
+function str(v) {
+  if (v === null || v === undefined) return null;
+  if (typeof v === 'string') return v.trim() || null;
+  return String(v);
+}

package/crawler/sitemap.js ADDED Viewed

@@ -0,0 +1,103 @@
+/**
+ * Sitemap.xml fetcher + parser
+ * Discovers URLs from sitemap before link-following begins.
+ */
+import fetch from 'node-fetch';
+const SITEMAP_TIMEOUT = 10000;
+/**
+ * Fetch and parse sitemap.xml for a domain.
+ * Handles sitemap index files (multiple sitemaps) and regular sitemaps.
+ * Returns array of { url, lastmod? } objects.
+ */
+export async function fetchSitemap(startUrl) {
+  const base = new URL(startUrl);
+  const sitemapUrls = [
+    `${base.origin}/sitemap.xml`,
+    `${base.origin}/sitemap_index.xml`,
+    `${base.origin}/sitemap-index.xml`,
+  ];
+  const allUrls = [];
+  const seen = new Set();
+  for (const sitemapUrl of sitemapUrls) {
+    try {
+      const urls = await parseSitemapUrl(sitemapUrl, base.hostname, seen);
+      allUrls.push(...urls);
+      if (urls.length > 0) break; // found a working sitemap
+    } catch {
+      continue;
+    }
+  }
+  return allUrls;
+}
+async function parseSitemapUrl(url, hostname, seen, depth = 0) {
+  if (depth > 2 || seen.has(url)) return []; // prevent infinite recursion
+  seen.add(url);
+  let text;
+  try {
+    const res = await fetch(url, {
+      timeout: SITEMAP_TIMEOUT,
+      headers: { 'User-Agent': 'SEOIntelBot/1.0' },
+    });
+    if (!res.ok) return [];
+    text = await res.text();
+  } catch {
+    return [];
+  }
+  // Check if it's a sitemap index (contains <sitemap> tags)
+  if (text.includes('<sitemap>') || text.includes('<sitemapindex')) {
+    const childUrls = extractTagContent(text, 'loc');
+    const results = [];
+    for (const childUrl of childUrls.slice(0, 20)) { // max 20 child sitemaps
+      const childResults = await parseSitemapUrl(childUrl, hostname, seen, depth + 1);
+      results.push(...childResults);
+    }
+    return results;
+  }
+  // Regular sitemap — extract <url> entries
+  const urls = [];
+  const locs = extractTagContent(text, 'loc');
+  const lastmods = extractTagContent(text, 'lastmod');
+  for (let i = 0; i < locs.length; i++) {
+    const loc = locs[i];
+    try {
+      const parsed = new URL(loc);
+      // Only include URLs from the same hostname
+      if (parsed.hostname !== hostname) continue;
+      // Skip non-page resources
+      if (/\.(pdf|png|jpg|jpeg|gif|svg|css|js|woff|ico|xml)$/i.test(parsed.pathname)) continue;
+      urls.push({
+        url: parsed.href,
+        lastmod: lastmods[i] || null,
+      });
+    } catch {
+      continue;
+    }
+  }
+  return urls;
+}
+/**
+ * Simple XML tag content extractor (no full XML parser needed).
+ */
+function extractTagContent(xml, tagName) {
+  const regex = new RegExp(`<${tagName}[^>]*>([^<]+)</${tagName}>`, 'gi');
+  const results = [];
+  let match;
+  while ((match = regex.exec(xml)) !== null) {
+    results.push(match[1].trim());
+  }
+  return results;
+}