seo-gravity-mcp 1.0.0

package/dist/tools/serp.js

@@ -0,0 +1,225 @@
+import { fetchAndParsePage } from '../utils/scraper.js';
+import { computeContentGapTfIdf, calculateReadability } from '../utils/nlp.js';
+import { scrapeGoogleSerp, scrapeForumDiscussions } from '../utils/searchEngines.js';
+export async function analyzeSerp(query, country = 'us', language = 'en', numResults = 10) {
+    return await scrapeGoogleSerp(query, country, language, numResults);
+}
+export async function profileCompetitor(url) {
+    const page = await fetchAndParsePage(url);
+    const readability = calculateReadability(page.cleanText);
+    // Schema extraction summary
+    const schemaTypes = [];
+    page.schemas.forEach(s => {
+        if (s['@type']) {
+            schemaTypes.push(typeof s['@type'] === 'string' ? s['@type'] : JSON.stringify(s['@type']));
+        }
+    });
+    const missingAlt = page.images.filter(img => !img.alt || img.alt.trim() === '').length;
+    const canonical = page.$('link[rel="canonical"]').attr('href') || '';
+    const robots = page.$('meta[name="robots"]').attr('content') || 'all (default)';
+    const openGraph = {};
+    page.$('meta[property^="og:"]').each((_, el) => {
+        const prop = page.$(el).attr('property');
+        const content = page.$(el).attr('content');
+        if (prop && content) {
+            openGraph[prop] = content;
+        }
+    });
+    return {
+        url,
+        title: page.title,
+        description: page.metaDescription,
+        h1: page.headings.h1,
+        h2: page.headings.h2,
+        h3: page.headings.h3,
+        h4: page.headings.h4,
+        wordCount: page.wordCount,
+        readingTimeMinutes: Math.max(1, Math.round(page.wordCount / 200)),
+        readingGradeLevel: `${readability.gradeLevel} (${readability.readingLevelSummary})`,
+        schemasFound: Array.from(new Set(schemaTypes)),
+        canonical,
+        robotsDirectives: robots,
+        openGraph,
+        imageCount: page.images.length,
+        imagesMissingAlt: missingAlt,
+        internalLinksCount: page.links.internal.length,
+        externalLinksCount: page.links.external.length
+    };
+}
+export async function analyzeCompetitorContentGap(targetUrlOrText, targetKeyword, competitorUrls) {
+    let targetText = targetUrlOrText;
+    let targetWordCount = 0;
+    if (targetUrlOrText.startsWith('http') || targetUrlOrText.includes('.html')) {
+        const targetPage = await fetchAndParsePage(targetUrlOrText);
+        targetText = targetPage.cleanText;
+        targetWordCount = targetPage.wordCount;
+    }
+    else {
+        targetWordCount = targetText.split(/\s+/).filter(Boolean).length;
+    }
+    // Determine competitor URLs
+    let urlsToScrape = competitorUrls || [];
+    if (urlsToScrape.length === 0) {
+        const serp = await scrapeGoogleSerp(targetKeyword, 'us', 'en', 5);
+        urlsToScrape = serp.organicResults.slice(0, 3).map(r => r.url);
+    }
+    const competitorTexts = [];
+    const competitorHeadings = [];
+    const scrapedUrls = [];
+    let totalCompWords = 0;
+    for (const url of urlsToScrape) {
+        try {
+            const page = await fetchAndParsePage(url);
+            competitorTexts.push(page.cleanText);
+            competitorHeadings.push({
+                url,
+                h2s: page.headings.h2,
+                h3s: page.headings.h3
+            });
+            scrapedUrls.push(url);
+            totalCompWords += page.wordCount;
+        }
+        catch {
+            // Continue to next competitor if one fails
+        }
+    }
+    const avgCompWords = competitorTexts.length > 0 ? Math.round(totalCompWords / competitorTexts.length) : 1500;
+    const missingEntities = computeContentGapTfIdf(targetText, competitorTexts);
+    // Identify heading topic gaps
+    const headingGaps = [];
+    competitorHeadings.forEach(comp => {
+        comp.h2s.forEach(h2 => {
+            const h2Lower = h2.toLowerCase();
+            // If target text does not contain key terms from this competitor H2
+            const words = h2Lower.split(/\s+/).filter(w => w.length > 4);
+            const isCovered = words.some(w => targetText.toLowerCase().includes(w));
+            if (!isCovered && words.length >= 2) {
+                const existing = headingGaps.find(g => g.subtopic === h2);
+                if (existing) {
+                    existing.coveredByCompetitorUrls.push(comp.url);
+                }
+                else {
+                    headingGaps.push({
+                        subtopic: h2,
+                        coveredByCompetitorUrls: [comp.url]
+                    });
+                }
+            }
+        });
+    });
+    const actionItems = [];
+    if (targetWordCount < avgCompWords * 0.75) {
+        actionItems.push(`Expand content depth: Target is ~${targetWordCount} words vs competitor average of ~${avgCompWords} words (deficit of ${avgCompWords - targetWordCount} words).`);
+    }
+    if (missingEntities.length > 0) {
+        actionItems.push(`Integrate high-importance semantic entities: ${missingEntities.slice(0, 5).map(e => `"${e.term}"`).join(', ')}.`);
+    }
+    if (headingGaps.length > 0) {
+        actionItems.push(`Add missing H2/H3 subtopics covered by top competitors: "${headingGaps[0]?.subtopic}" and "${headingGaps[1]?.subtopic || headingGaps[0]?.subtopic}".`);
+    }
+    return {
+        targetUrlOrKeyword: targetUrlOrText.substring(0, 100),
+        analyzedCompetitors: scrapedUrls,
+        averageCompetitorWordCount: avgCompWords,
+        targetWordCount,
+        wordCountDelta: targetWordCount - avgCompWords,
+        missingEntities,
+        headingCoverageGaps: headingGaps.slice(0, 8),
+        suggestedActionItems: actionItems
+    };
+}
+export async function diffCompetitor(myUrl, competitorUrl, focusKeyword) {
+    const [myPage, compPage] = await Promise.all([
+        profileCompetitor(myUrl),
+        profileCompetitor(competitorUrl)
+    ]);
+    const kw = focusKeyword.toLowerCase();
+    const myHasKwTitle = myPage.title.toLowerCase().includes(kw);
+    const compHasKwTitle = compPage.title.toLowerCase().includes(kw);
+    const myHasKwH1 = myPage.h1.some(h => h.toLowerCase().includes(kw));
+    const compHasKwH1 = compPage.h1.some(h => h.toLowerCase().includes(kw));
+    const scorecard = [
+        {
+            metric: 'Title Tag Focus Keyword',
+            myValue: myHasKwTitle ? 'Yes' : 'No',
+            competitorValue: compHasKwTitle ? 'Yes' : 'No',
+            winner: myHasKwTitle && !compHasKwTitle ? 'my_site' : !myHasKwTitle && compHasKwTitle ? 'competitor' : 'tie',
+            notes: `My: "${myPage.title.substring(0, 45)}..." | Comp: "${compPage.title.substring(0, 45)}..."`
+        },
+        {
+            metric: 'H1 Tag Focus Keyword',
+            myValue: myHasKwH1 ? 'Yes' : 'No',
+            competitorValue: compHasKwH1 ? 'Yes' : 'No',
+            winner: myHasKwH1 && !compHasKwH1 ? 'my_site' : !myHasKwH1 && compHasKwH1 ? 'competitor' : 'tie',
+            notes: `My H1 count: ${myPage.h1.length} | Comp H1 count: ${compPage.h1.length}`
+        },
+        {
+            metric: 'Total Word Count & Depth',
+            myValue: `${myPage.wordCount} words`,
+            competitorValue: `${compPage.wordCount} words`,
+            winner: myPage.wordCount >= compPage.wordCount ? 'my_site' : 'competitor',
+            notes: myPage.wordCount >= compPage.wordCount ? 'Your content is deeper' : 'Competitor content is more extensive'
+        },
+        {
+            metric: 'Structured Schema Markup',
+            myValue: myPage.schemasFound.length > 0 ? myPage.schemasFound.join(', ') : 'None',
+            competitorValue: compPage.schemasFound.length > 0 ? compPage.schemasFound.join(', ') : 'None',
+            winner: myPage.schemasFound.length > compPage.schemasFound.length ? 'my_site' : myPage.schemasFound.length < compPage.schemasFound.length ? 'competitor' : 'tie',
+            notes: `My schemas: ${myPage.schemasFound.length} | Comp schemas: ${compPage.schemasFound.length}`
+        },
+        {
+            metric: 'Image Alt Tag Hygiene',
+            myValue: `${myPage.imageCount - myPage.imagesMissingAlt}/${myPage.imageCount} with alt`,
+            competitorValue: `${compPage.imageCount - compPage.imagesMissingAlt}/${compPage.imageCount} with alt`,
+            winner: myPage.imagesMissingAlt === 0 ? 'my_site' : myPage.imagesMissingAlt <= compPage.imagesMissingAlt ? 'my_site' : 'competitor',
+            notes: `Missing alt: My site (${myPage.imagesMissingAlt}) vs Competitor (${compPage.imagesMissingAlt})`
+        },
+        {
+            metric: 'Heading Subtopic Structure (H2/H3)',
+            myValue: `${myPage.h2.length} H2s, ${myPage.h3.length} H3s`,
+            competitorValue: `${compPage.h2.length} H2s, ${compPage.h3.length} H3s`,
+            winner: (myPage.h2.length + myPage.h3.length) >= (compPage.h2.length + compPage.h3.length) ? 'my_site' : 'competitor',
+            notes: 'Comparison of subtopic granularity'
+        }
+    ];
+    let myScore = 0;
+    let compScore = 0;
+    scorecard.forEach(item => {
+        if (item.winner === 'my_site')
+            myScore += 10;
+        else if (item.winner === 'competitor')
+            compScore += 10;
+        else {
+            myScore += 5;
+            compScore += 5;
+        }
+    });
+    const topPriorityFixes = [];
+    if (!myHasKwTitle && compHasKwTitle) {
+        topPriorityFixes.push(`Include primary keyword '${focusKeyword}' in your title tag.`);
+    }
+    if (!myHasKwH1 && compHasKwH1) {
+        topPriorityFixes.push(`Ensure your H1 explicitly includes '${focusKeyword}'.`);
+    }
+    if (myPage.wordCount < compPage.wordCount * 0.8) {
+        topPriorityFixes.push(`Expand total word count to match or exceed competitor's ${compPage.wordCount} words.`);
+    }
+    if (myPage.schemasFound.length === 0 && compPage.schemasFound.length > 0) {
+        topPriorityFixes.push(`Add Schema.org markup (Competitor is using: ${compPage.schemasFound.join(', ')}).`);
+    }
+    return {
+        myUrl,
+        competitorUrl,
+        focusKeyword,
+        scorecard,
+        summary: {
+            myScore,
+            competitorScore: compScore,
+            winner: myScore > compScore ? 'my_site' : myScore < compScore ? 'competitor' : 'tie',
+            topPriorityFixes
+        }
+    };
+}
+export async function analyzeForumDiscussions(topic) {
+    return await scrapeForumDiscussions(topic);
+}

package/dist/tools/technical.d.ts

@@ -0,0 +1,36 @@
+import { TechnicalAuditReport, JsRenderingDiffReport } from '../types/seo.js';
+export declare function auditTechnical(url: string): Promise<TechnicalAuditReport>;
+export declare function diffJsRendering(url: string): Promise<JsRenderingDiffReport>;
+export declare function validateRobotsTxt(domainOrUrl: string, testPath?: string, userAgent?: string): Promise<{
+    domain: string;
+    testPath: string;
+    userAgent: string;
+    accessStatus: 'ALLOWED' | 'DISALLOWED';
+    matchedDirective: string;
+    sitemapsFound: string[];
+    rawDirectivesSample: string[];
+}>;
+export declare function inspectSitemap(sitemapUrl: string): Promise<{
+    sitemapUrl: string;
+    isIndex: boolean;
+    totalUrls: number;
+    sampleUrls: Array<{
+        loc: string;
+        lastmod?: string;
+        changefreq?: string;
+        priority?: string;
+    }>;
+    errorsDetected: string[];
+}>;
+export declare function analyzeInternalLinks(url: string, maxCrawlDepth?: number): Promise<{
+    targetUrl: string;
+    totalInternalLinksFound: number;
+    uniqueInternalDestinations: number;
+    genericAnchorTextCount: number;
+    topAnchorTexts: Array<{
+        anchor: string;
+        count: number;
+    }>;
+    nofollowInternalLinks: string[];
+    recommendations: string[];
+}>;

package/dist/tools/technical.js

@@ -0,0 +1,277 @@
+import axios from 'axios';
+import { XMLParser } from 'fast-xml-parser';
+import { fetchAndParsePage, getRandomUserAgent } from '../utils/scraper.js';
+import { compareServerVsClientDom } from '../utils/jsdomRenderer.js';
+export async function auditTechnical(url) {
+    const startTime = Date.now();
+    const redirectChain = [];
+    let response;
+    try {
+        response = await axios.get(url, {
+            headers: { 'User-Agent': getRandomUserAgent() },
+            timeout: 15000,
+            maxRedirects: 10,
+            validateStatus: () => true
+        });
+    }
+    catch (err) {
+        throw new Error(`Technical audit failed to connect to ${url}: ${err.message}`);
+    }
+    const responseTimeMs = Date.now() - startTime;
+    const page = await fetchAndParsePage(url);
+    const html = page.html;
+    const $ = page.$;
+    // Canonical check
+    const canonicalVal = $('link[rel="canonical"]').attr('href') || '';
+    const isSelfRef = canonicalVal ? canonicalVal.replace(/\/$/, '') === url.replace(/\/$/, '') : false;
+    // Robots meta
+    const metaRobots = $('meta[name="robots"]').attr('content') || '';
+    const xRobots = response.headers['x-robots-tag'] || '';
+    const isNoIndex = /noindex/i.test(metaRobots) || /noindex/i.test(xRobots);
+    const isNoFollow = /nofollow/i.test(metaRobots) || /nofollow/i.test(xRobots);
+    // Hreflang
+    const hreflangs = [];
+    $('link[rel="alternate"][hreflang]').each((_, el) => {
+        const lang = $(el).attr('hreflang') || '';
+        const href = $(el).attr('href') || '';
+        if (lang && href)
+            hreflangs.push({ lang, href });
+    });
+    // OpenGraph & Twitter
+    const ogTitle = $('meta[property="og:title"]').attr('content');
+    const ogDesc = $('meta[property="og:description"]').attr('content');
+    const ogImage = $('meta[property="og:image"]').attr('content');
+    const ogType = $('meta[property="og:type"]').attr('content');
+    const twCard = $('meta[name="twitter:card"]').attr('content');
+    const twTitle = $('meta[name="twitter:title"]').attr('content');
+    const twImage = $('meta[name="twitter:image"]').attr('content');
+    const issues = [];
+    if (isNoIndex) {
+        issues.push({
+            severity: 'critical',
+            message: 'Page has a NOINDEX directive, preventing Google from ranking it in search results.',
+            fix: 'Remove "noindex" from <meta name="robots"> or X-Robots-Tag header before launching in production.'
+        });
+    }
+    if (!canonicalVal) {
+        issues.push({
+            severity: 'warning',
+            message: 'Missing canonical tag.',
+            fix: `Add <link rel="canonical" href="${url}" /> to clarify the preferred master URL.`
+        });
+    }
+    if (response.status >= 400) {
+        issues.push({
+            severity: 'critical',
+            message: `HTTP Status Code is ${response.status} (Error).`,
+            fix: 'Ensure server returns HTTP 200 OK for valid pages.'
+        });
+    }
+    if (responseTimeMs > 1800) {
+        issues.push({
+            severity: 'warning',
+            message: `Slow Server Response Time: ${responseTimeMs}ms (TTFB is over 1.8s).`,
+            fix: 'Enable edge caching, optimize database queries, or use a CDN to bring TTFB under 800ms.'
+        });
+    }
+    if (!url.startsWith('https://') && !url.includes('localhost') && !url.includes('127.0.0.1')) {
+        issues.push({
+            severity: 'critical',
+            message: 'Page is served over unencrypted HTTP rather than HTTPS.',
+            fix: 'Enforce SSL/TLS certificate and 301 redirect all HTTP traffic to HTTPS.'
+        });
+    }
+    return {
+        url,
+        statusCode: response.status,
+        responseTimeMs,
+        redirectChain,
+        isHttps: url.startsWith('https://'),
+        canonicalTag: {
+            present: Boolean(canonicalVal),
+            value: canonicalVal,
+            isSelfReferencing: isSelfRef
+        },
+        robotsDirectives: {
+            metaRobots,
+            xRobotsTag: xRobots,
+            isNoIndex,
+            isNoFollow
+        },
+        hreflangTags: hreflangs,
+        openGraphTags: {
+            hasTitle: Boolean(ogTitle),
+            hasDescription: Boolean(ogDesc),
+            hasImage: Boolean(ogImage),
+            hasType: Boolean(ogType)
+        },
+        twitterCards: {
+            hasCard: Boolean(twCard),
+            hasTitle: Boolean(twTitle),
+            hasImage: Boolean(twImage)
+        },
+        issuesFound: issues
+    };
+}
+export async function diffJsRendering(url) {
+    return await compareServerVsClientDom(url);
+}
+export async function validateRobotsTxt(domainOrUrl, testPath = '/', userAgent = 'Googlebot') {
+    const domain = domainOrUrl.startsWith('http') ? new URL(domainOrUrl).origin : `https://${domainOrUrl}`;
+    const robotsUrl = `${domain}/robots.txt`;
+    let content = '';
+    try {
+        const res = await axios.get(robotsUrl, {
+            headers: { 'User-Agent': getRandomUserAgent() },
+            timeout: 8000
+        });
+        content = res.data;
+    }
+    catch {
+        return {
+            domain,
+            testPath,
+            userAgent,
+            accessStatus: 'ALLOWED',
+            matchedDirective: 'No robots.txt found (default ALLOWED)',
+            sitemapsFound: [],
+            rawDirectivesSample: []
+        };
+    }
+    const sitemaps = [];
+    const lines = content.split('\n').map(l => l.trim());
+    let currentAgent = '';
+    let matchedDirective = 'None (Default Allow)';
+    let isDisallowed = false;
+    for (const line of lines) {
+        if (line.toLowerCase().startsWith('sitemap:')) {
+            sitemaps.push(line.substring(8).trim());
+        }
+        if (line.toLowerCase().startsWith('user-agent:')) {
+            currentAgent = line.substring(11).trim();
+        }
+        if (currentAgent === '*' || currentAgent.toLowerCase() === userAgent.toLowerCase()) {
+            if (line.toLowerCase().startsWith('disallow:')) {
+                const pathRule = line.substring(9).trim();
+                if (pathRule && testPath.startsWith(pathRule)) {
+                    isDisallowed = true;
+                    matchedDirective = `User-agent: ${currentAgent} -> Disallow: ${pathRule}`;
+                }
+            }
+            if (line.toLowerCase().startsWith('allow:')) {
+                const pathRule = line.substring(6).trim();
+                if (pathRule && testPath.startsWith(pathRule)) {
+                    isDisallowed = false;
+                    matchedDirective = `User-agent: ${currentAgent} -> Allow: ${pathRule}`;
+                }
+            }
+        }
+    }
+    return {
+        domain,
+        testPath,
+        userAgent,
+        accessStatus: isDisallowed ? 'DISALLOWED' : 'ALLOWED',
+        matchedDirective,
+        sitemapsFound: sitemaps,
+        rawDirectivesSample: lines.slice(0, 15)
+    };
+}
+export async function inspectSitemap(sitemapUrl) {
+    let xmlData = '';
+    try {
+        const res = await axios.get(sitemapUrl, {
+            headers: { 'User-Agent': getRandomUserAgent() },
+            timeout: 12000
+        });
+        xmlData = res.data;
+    }
+    catch (err) {
+        throw new Error(`Failed to download sitemap from ${sitemapUrl}: ${err.message}`);
+    }
+    const parser = new XMLParser();
+    const jsonObj = parser.parse(xmlData);
+    const errors = [];
+    let isIndex = false;
+    let totalUrls = 0;
+    const sample = [];
+    if (jsonObj.sitemapindex && jsonObj.sitemapindex.sitemap) {
+        isIndex = true;
+        const subMaps = Array.isArray(jsonObj.sitemapindex.sitemap) ? jsonObj.sitemapindex.sitemap : [jsonObj.sitemapindex.sitemap];
+        totalUrls = subMaps.length;
+        subMaps.slice(0, 10).forEach((s) => {
+            sample.push({ loc: s.loc, lastmod: s.lastmod });
+        });
+    }
+    else if (jsonObj.urlset && jsonObj.urlset.url) {
+        const urls = Array.isArray(jsonObj.urlset.url) ? jsonObj.urlset.url : [jsonObj.urlset.url];
+        totalUrls = urls.length;
+        urls.slice(0, 10).forEach((u) => {
+            sample.push({
+                loc: u.loc,
+                lastmod: u.lastmod,
+                changefreq: u.changefreq,
+                priority: u.priority
+            });
+        });
+        if (totalUrls > 50000) {
+            errors.push(`Sitemap contains ${totalUrls} URLs, exceeding Google's single-sitemap limit of 50,000 URLs. Break into a Sitemap Index.`);
+        }
+    }
+    else {
+        errors.push('Unrecognized XML sitemap format. Missing <urlset> or <sitemapindex> root node.');
+    }
+    return {
+        sitemapUrl,
+        isIndex,
+        totalUrls,
+        sampleUrls: sample,
+        errorsDetected: errors
+    };
+}
+export async function analyzeInternalLinks(url, maxCrawlDepth = 2) {
+    const page = await fetchAndParsePage(url);
+    const $ = page.$;
+    const domain = url.startsWith('http') ? new URL(url).hostname : '';
+    const anchorCounts = new Map();
+    const internalDestinations = new Set();
+    const nofollowLinks = [];
+    let genericCount = 0;
+    $('a[href]').each((_, el) => {
+        const href = $(el).attr('href')?.trim() || '';
+        const anchor = $(el).text().trim();
+        const rel = $(el).attr('rel') || '';
+        if (href.startsWith('/') || (domain && href.includes(domain))) {
+            internalDestinations.add(href);
+            if (['click here', 'read more', 'learn more', 'here', 'link'].includes(anchor.toLowerCase())) {
+                genericCount++;
+            }
+            if (anchor) {
+                anchorCounts.set(anchor, (anchorCounts.get(anchor) || 0) + 1);
+            }
+            if (rel.includes('nofollow')) {
+                nofollowLinks.push(href);
+            }
+        }
+    });
+    const topAnchors = Array.from(anchorCounts.entries())
+        .map(([anchor, count]) => ({ anchor, count }))
+        .sort((a, b) => b.count - a.count)
+        .slice(0, 10);
+    const recommendations = [];
+    if (genericCount > 0) {
+        recommendations.push(`Replace ${genericCount} generic anchor texts ("click here", "read more") with descriptive keyword-rich anchors.`);
+    }
+    if (nofollowLinks.length > 0) {
+        recommendations.push(`Detected ${nofollowLinks.length} internal links marked with "nofollow". Avoid nofollowing internal links to preserve PageRank flow.`);
+    }
+    return {
+        targetUrl: url,
+        totalInternalLinksFound: page.links.internal.length,
+        uniqueInternalDestinations: internalDestinations.size,
+        genericAnchorTextCount: genericCount,
+        topAnchorTexts: topAnchors,
+        nofollowInternalLinks: nofollowLinks.slice(0, 5),
+        recommendations
+    };
+}