rankforge 0.3.0

package/src/rules.mjs

@@ -0,0 +1,350 @@
+const sources = {
+  searchEssentials: "https://developers.google.com/search/docs/essentials",
+  technicalRequirements: "https://developers.google.com/search/docs/essentials/technical",
+  howSearchWorks: "https://developers.google.com/search/docs/fundamentals/how-search-works",
+  crawlableLinks: "https://developers.google.com/search/docs/crawling-indexing/links-crawlable",
+  robotsMeta: "https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag",
+  robotsTxt: "https://developers.google.com/search/docs/crawling-indexing/robots/intro",
+  canonicalization: "https://developers.google.com/search/docs/crawling-indexing/canonicalization",
+  consolidateDuplicates: "https://developers.google.com/search/docs/crawling-indexing/consolidate-duplicate-urls",
+  sitemaps: "https://developers.google.com/search/docs/crawling-indexing/sitemaps/overview",
+  javascriptSeo: "https://developers.google.com/search/docs/crawling-indexing/javascript/javascript-seo-basics",
+  helpfulContent: "https://developers.google.com/search/docs/fundamentals/creating-helpful-content",
+  aiOptimization: "https://developers.google.com/search/docs/fundamentals/ai-optimization-guide",
+  structuredDataIntro: "https://developers.google.com/search/docs/appearance/structured-data/intro-structured-data",
+  structuredDataPolicies: "https://developers.google.com/search/docs/appearance/structured-data/sd-policies",
+  organizationSchema: "https://developers.google.com/search/docs/appearance/structured-data/organization",
+  pageExperience: "https://developers.google.com/search/docs/appearance/page-experience",
+  coreWebVitals: "https://developers.google.com/search/docs/appearance/core-web-vitals",
+  titleLinks: "https://developers.google.com/search/docs/appearance/title-link",
+  snippets: "https://developers.google.com/search/docs/appearance/snippet",
+  googleImages: "https://developers.google.com/search/docs/appearance/google-images",
+  favicon: "https://developers.google.com/search/docs/appearance/favicon-in-search",
+  spamPolicies: "https://developers.google.com/search/docs/essentials/spam-policies",
+};
+
+const rule = (id, dimension, defaultSeverity, title, recommendation, ruleSources) => ({
+  id,
+  dimension,
+  defaultSeverity,
+  title,
+  recommendation,
+  sources: ruleSources,
+});
+
+export const rules = [
+  rule(
+    "technical.http_error",
+    "technical",
+    "P0",
+    "Important page returns an unsuccessful HTTP status",
+    "Fix server, routing, or deployment errors so important pages return a successful status.",
+    [sources.technicalRequirements, sources.howSearchWorks],
+  ),
+  rule(
+    "technical.redirect_chain",
+    "technical",
+    "P2",
+    "Page uses a long or unstable redirect chain",
+    "Reduce redirects so canonical URLs resolve directly and predictably.",
+    [sources.technicalRequirements],
+  ),
+  rule(
+    "technical.https_missing",
+    "technical",
+    "P1",
+    "Page is not available over HTTPS",
+    "Serve important pages over HTTPS and redirect HTTP variants to the HTTPS canonical URL.",
+    [sources.searchEssentials],
+  ),
+  rule(
+    "technical.rendered_content_missing",
+    "technical",
+    "P1",
+    "Rendered page is missing primary content",
+    "Ensure primary content is available in rendered HTML and does not depend on fragile client behavior.",
+    [sources.javascriptSeo, sources.technicalRequirements],
+  ),
+  rule(
+    "technical.raw_rendered_mismatch",
+    "technical",
+    "P2",
+    "Raw and rendered page content differ materially",
+    "Review JavaScript rendering and hydration so crawlers and users receive consistent primary content.",
+    [sources.javascriptSeo],
+  ),
+  rule(
+    "technical.rendered_title_changed",
+    "technical",
+    "P2",
+    "Rendered HTML changes the page title",
+    "Keep rendered title output aligned with raw HTML so crawlers and users receive a stable title signal.",
+    [sources.javascriptSeo, sources.titleLinks],
+  ),
+  rule(
+    "technical.rendered_description_changed",
+    "technical",
+    "P3",
+    "Rendered HTML changes the meta description",
+    "Keep rendered meta description output aligned with raw HTML when the page relies on it for snippet context.",
+    [sources.javascriptSeo, sources.snippets],
+  ),
+  rule(
+    "technical.rendered_canonical_changed",
+    "technical",
+    "P1",
+    "Rendered HTML changes the canonical URL",
+    "Keep canonical URL output stable between raw and rendered HTML so indexing signals remain consistent.",
+    [sources.javascriptSeo, sources.canonicalization],
+  ),
+  rule(
+    "technical.rendered_primary_heading_missing",
+    "technical",
+    "P1",
+    "Rendered HTML removes the primary heading",
+    "Ensure the rendered page preserves the primary heading that communicates page purpose.",
+    [sources.javascriptSeo, sources.helpfulContent],
+  ),
+  rule(
+    "technical.rendered_structured_data_lost",
+    "technical",
+    "P2",
+    "Rendered HTML removes structured data",
+    "Preserve structured data through rendering and hydration so eligible markup remains available.",
+    [sources.javascriptSeo, sources.structuredDataIntro, sources.structuredDataPolicies],
+  ),
+  rule(
+    "crawl.robots_blocked",
+    "crawl_index",
+    "P1",
+    "robots.txt blocks an important page or resource",
+    "Update robots.txt rules so important pages and resources needed for rendering can be crawled.",
+    [sources.robotsTxt, sources.technicalRequirements],
+  ),
+  rule(
+    "crawl.sitemap_missing",
+    "crawl_index",
+    "P2",
+    "No sitemap was discovered",
+    "Publish a sitemap that lists canonical, indexable URLs for important pages.",
+    [sources.sitemaps],
+  ),
+  rule(
+    "crawl.broken_internal_link",
+    "crawl_index",
+    "P2",
+    "Internal link points to an unavailable URL",
+    "Fix or remove broken internal links so crawlers and users can move through the site reliably.",
+    [sources.crawlableLinks, sources.technicalRequirements],
+  ),
+  rule(
+    "indexability.noindex",
+    "crawl_index",
+    "P1",
+    "Important page has a noindex directive",
+    "Remove noindex from pages intended to appear in Google Search.",
+    [sources.robotsMeta],
+  ),
+  rule(
+    "indexability.x_robots_noindex",
+    "crawl_index",
+    "P1",
+    "Important page has an X-Robots-Tag noindex directive",
+    "Remove X-Robots-Tag noindex from pages intended to appear in Google Search.",
+    [sources.robotsMeta],
+  ),
+  rule(
+    "indexability.canonical_missing",
+    "crawl_index",
+    "P2",
+    "Canonical URL is missing",
+    "Add a canonical signal to important indexable pages when duplicate or alternate URLs may exist.",
+    [sources.canonicalization, sources.consolidateDuplicates],
+  ),
+  rule(
+    "indexability.canonical_target_error",
+    "crawl_index",
+    "P1",
+    "Canonical target is unavailable or not indexable",
+    "Point canonical tags to successful, indexable preferred URLs.",
+    [sources.canonicalization],
+  ),
+  rule(
+    "indexability.noindex_canonical_conflict",
+    "crawl_index",
+    "P1",
+    "Page combines noindex with a canonical signal",
+    "Choose a single indexing strategy: remove noindex for indexable pages or align links and canonical signals to the preferred URL.",
+    [sources.robotsMeta, sources.canonicalization],
+  ),
+  rule(
+    "indexability.noncanonical_in_sitemap",
+    "crawl_index",
+    "P2",
+    "Sitemap includes non-canonical URLs",
+    "List only canonical, indexable URLs in sitemaps.",
+    [sources.sitemaps, sources.canonicalization],
+  ),
+  rule(
+    "appearance.title_missing",
+    "search_appearance",
+    "P1",
+    "Page is missing a title element",
+    "Add a unique, descriptive title that reflects the page's main content.",
+    [sources.titleLinks],
+  ),
+  rule(
+    "appearance.title_duplicate",
+    "search_appearance",
+    "P2",
+    "Multiple pages use the same title",
+    "Write unique titles that distinguish page purpose and topic.",
+    [sources.titleLinks],
+  ),
+  rule(
+    "appearance.meta_description_missing",
+    "search_appearance",
+    "P3",
+    "Page is missing a meta description",
+    "Add concise, useful page descriptions that can support better snippets.",
+    [sources.snippets],
+  ),
+  rule(
+    "appearance.meta_description_duplicate",
+    "search_appearance",
+    "P3",
+    "Multiple pages use the same meta description",
+    "Write unique descriptions that summarize each page's specific purpose.",
+    [sources.snippets],
+  ),
+  rule(
+    "appearance.h1_missing",
+    "search_appearance",
+    "P2",
+    "Page is missing a clear H1",
+    "Add one visible primary heading that describes the page topic.",
+    [sources.titleLinks, sources.helpfulContent],
+  ),
+  rule(
+    "appearance.image_alt_missing",
+    "search_appearance",
+    "P3",
+    "Important image is missing alt text",
+    "Add descriptive alt text for meaningful images.",
+    [sources.googleImages],
+  ),
+  rule(
+    "appearance.favicon_missing",
+    "search_appearance",
+    "P3",
+    "Favicon signal was not discovered",
+    "Publish a valid favicon that Google can access.",
+    [sources.favicon],
+  ),
+  rule(
+    "structured_data.invalid_jsonld",
+    "structured_data",
+    "P1",
+    "JSON-LD structured data is invalid",
+    "Fix JSON-LD syntax so structured data can be parsed reliably.",
+    [sources.structuredDataIntro],
+  ),
+  rule(
+    "structured_data.required_property_missing",
+    "structured_data",
+    "P2",
+    "Structured data is missing required properties",
+    "Add the required properties for supported schema types when the page's visible content supports the markup.",
+    [sources.structuredDataIntro, sources.structuredDataPolicies],
+  ),
+  rule(
+    "structured_data.visible_content_mismatch",
+    "structured_data",
+    "P1",
+    "Structured data does not match visible page content",
+    "Keep structured data aligned with user-visible content and Google structured data policies.",
+    [sources.structuredDataPolicies],
+  ),
+  rule(
+    "structured_data.organization_missing",
+    "structured_data",
+    "P2",
+    "Organization structured data is missing or unclear",
+    "Add Organization structured data where visible business identity supports it.",
+    [sources.organizationSchema, sources.structuredDataIntro],
+  ),
+  rule(
+    "performance.lighthouse_poor",
+    "performance",
+    "P2",
+    "Imported Lighthouse performance score is poor",
+    "Investigate page weight, render-blocking work, server response time, and client-side execution before treating performance as search-ready.",
+    [sources.pageExperience, sources.coreWebVitals],
+  ),
+  rule(
+    "performance.lcp_poor",
+    "performance",
+    "P2",
+    "Largest Contentful Paint is poor",
+    "Improve loading of the largest above-the-fold content element by optimizing server response, critical resources, images, and rendering work.",
+    [sources.pageExperience, sources.coreWebVitals],
+  ),
+  rule(
+    "performance.cls_poor",
+    "performance",
+    "P2",
+    "Cumulative Layout Shift is poor",
+    "Reserve stable layout space for images, embeds, ads, and late-loading UI so visible content does not shift unexpectedly.",
+    [sources.pageExperience, sources.coreWebVitals],
+  ),
+  rule(
+    "content.thin_content",
+    "helpful_content",
+    "P2",
+    "Page has limited useful main content",
+    "Expand visible content to satisfy the visitor's task with original, helpful information.",
+    [sources.helpfulContent],
+  ),
+  rule(
+    "content.answerability_gap",
+    "geo_readiness",
+    "P2",
+    "Page lacks directly answerable sections",
+    "Add concise answer sections, definitions, steps, comparisons, FAQs, or evidence where useful.",
+    [sources.aiOptimization, sources.helpfulContent],
+  ),
+  rule(
+    "entity.about_contact_missing",
+    "entity_clarity",
+    "P2",
+    "Organization identity or contact details are missing",
+    "Make organization identity, contact, and trust details easy to find on crawlable pages.",
+    [sources.searchEssentials, sources.organizationSchema],
+  ),
+  rule(
+    "geo.entity_clarity_gap",
+    "geo_readiness",
+    "P2",
+    "Important entities and relationships are unclear",
+    "Name products, services, people, locations, and relationships plainly in visible content.",
+    [sources.aiOptimization, sources.helpfulContent],
+  ),
+  rule(
+    "policy.hidden_text_risk",
+    "policy_risk",
+    "P1",
+    "Hidden text risk detected",
+    "Remove or justify hidden text patterns that could mislead users or search systems.",
+    [sources.spamPolicies],
+  ),
+  rule(
+    "policy.duplicate_content_cluster",
+    "policy_risk",
+    "P2",
+    "Large duplicate content cluster detected",
+    "Consolidate duplicative pages into stronger canonical resources or clarify unique page value.",
+    [sources.consolidateDuplicates, sources.helpfulContent, sources.spamPolicies],
+  ),
+];
+
+export const getRule = (id) => rules.find((item) => item.id === id);

package/src/site-rule-engine.mjs

@@ -0,0 +1,177 @@
+import { implementationTaskFor } from "./finding-task.mjs";
+import { duplicateContentClusterFacts } from "./rule-depth.mjs";
+import { getRule } from "./rules.mjs";
+import { isHttpUrl, normalizeUrl, sameOrigin } from "./url-utils.mjs";
+
+const ownerFor = (dimension) => {
+  if (dimension === "crawl_index" || dimension === "technical" || dimension === "structured_data") return "Engineering";
+  return "SEO";
+};
+
+const effortFor = (severity) => (severity === "P1" ? "M" : "S");
+
+const finding = (ruleId, affectedUrls, evidence, impact) => {
+  const rule = getRule(ruleId);
+  const owner = ownerFor(rule.dimension);
+  const effort = effortFor(rule.defaultSeverity);
+  return {
+    ruleId: rule.id,
+    title: rule.title,
+    severity: rule.defaultSeverity,
+    dimension: rule.dimension,
+    affectedUrls,
+    evidence,
+    impact,
+    recommendation: rule.recommendation,
+    implementationTask: implementationTaskFor(rule, owner, effort),
+    owner,
+    effort,
+    confidence: "high",
+    sources: rule.sources,
+  };
+};
+
+const groupBy = (pages, selector) => {
+  const groups = new Map();
+  for (const [index, page] of pages.entries()) {
+    const key = selector(page);
+    if (!key) continue;
+    const normalized = String(key).trim().toLowerCase();
+    if (!normalized) continue;
+    const group = groups.get(normalized) || [];
+    group.push({ page, index });
+    groups.set(normalized, group);
+  }
+  return [...groups.values()].filter((group) => group.length > 1);
+};
+
+const pageMap = (pages) => {
+  const map = new Map();
+  for (const page of pages) {
+    try {
+      map.set(normalizeUrl(page.finalUrl), page);
+    } catch {
+      map.set(page.finalUrl, page);
+    }
+  }
+  return map;
+};
+
+const safeNormalize = (value) => {
+  try {
+    return normalizeUrl(value);
+  } catch {
+    return value;
+  }
+};
+
+export const evaluateSite = (pages, context = {}) => {
+  const findings = [];
+  const byUrl = pageMap(pages);
+
+  if (context.crawled && (context.sitemaps || []).length === 0) {
+    findings.push(
+      finding(
+        "crawl.sitemap_missing",
+        [pages[0]?.finalUrl || context.origin || "site"],
+        ["$.site.sitemaps"],
+        "No sitemap evidence was discovered or supplied for this crawled audit.",
+      ),
+    );
+  }
+
+  for (const skipped of context.skipped || []) {
+    if (skipped.reason !== "robots_blocked") continue;
+    findings.push(
+      finding(
+        "crawl.robots_blocked",
+        [skipped.url],
+        ["$.site.skipped"],
+        "robots.txt blocks a URL discovered during crawl evidence collection.",
+      ),
+    );
+  }
+
+  for (const group of groupBy(pages, (page) => page.evidence?.title)) {
+    findings.push(
+      finding(
+        "appearance.title_duplicate",
+        group.map(({ page }) => page.finalUrl),
+        group.map(({ index }) => `$.pages[${index}].evidence.title`),
+        "Duplicate titles make it harder to distinguish page purpose in search results.",
+      ),
+    );
+  }
+
+  for (const group of groupBy(pages, (page) => page.evidence?.description)) {
+    findings.push(
+      finding(
+        "appearance.meta_description_duplicate",
+        group.map(({ page }) => page.finalUrl),
+        group.map(({ index }) => `$.pages[${index}].evidence.description`),
+        "Duplicate meta descriptions reduce page-specific snippet clarity.",
+      ),
+    );
+  }
+
+  for (const [index, page] of pages.entries()) {
+    if (!isHttpUrl(page.finalUrl)) continue;
+
+    for (const link of page.evidence?.links || []) {
+      if (!link.href || !isHttpUrl(link.href) || !sameOrigin(page.finalUrl, link.href)) continue;
+      const linked = byUrl.get(safeNormalize(link.href));
+      if (linked && Number.isInteger(linked.status) && linked.status >= 400) {
+        findings.push(
+          finding(
+            "crawl.broken_internal_link",
+            [page.finalUrl, linked.finalUrl],
+            [`$.pages[${index}].evidence.links`, `$.pages[${pages.indexOf(linked)}].status`],
+            "Internal links to unavailable pages waste crawl paths and create a poor user experience.",
+          ),
+        );
+      }
+    }
+
+    const canonical = page.evidence?.canonical;
+    if (canonical) {
+      const canonicalPage = byUrl.get(normalizeUrl(canonical));
+      if (canonicalPage && Number.isInteger(canonicalPage.status) && canonicalPage.status >= 400) {
+        findings.push(
+          finding(
+            "indexability.canonical_target_error",
+            [page.finalUrl, canonicalPage.finalUrl],
+            [`$.pages[${index}].evidence.canonical`, `$.pages[${pages.indexOf(canonicalPage)}].status`],
+            "Canonical signals should point to successful, indexable preferred URLs.",
+          ),
+        );
+      }
+
+      if (
+        context.sitemapUrls?.includes(safeNormalize(page.finalUrl)) &&
+        safeNormalize(canonical) !== safeNormalize(page.finalUrl)
+      ) {
+        findings.push(
+          finding(
+            "indexability.noncanonical_in_sitemap",
+            [page.finalUrl],
+            [`$.pages[${index}].evidence.canonical`],
+            "Sitemaps should list canonical URLs rather than alternates or duplicates.",
+          ),
+        );
+      }
+    }
+  }
+
+  for (const clusterFact of duplicateContentClusterFacts(pages)) {
+    findings.push(
+      finding(
+        clusterFact.ruleId,
+        clusterFact.affectedUrls,
+        clusterFact.evidence,
+        clusterFact.impact,
+      ),
+    );
+  }
+
+  return findings;
+};

package/src/sitemap.mjs

@@ -0,0 +1,30 @@
+const decodeEntities = (value) =>
+  String(value || "")
+    .replace(/&/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'");
+
+const locs = (xml) => {
+  const results = [];
+  let match;
+  const pattern = /<loc\b[^>]*>([\s\S]*?)<\/loc>/gi;
+  while ((match = pattern.exec(xml))) results.push(decodeEntities(match[1].trim()));
+  return results;
+};
+
+export const parseSitemap = (xml) => {
+  const body = String(xml || "");
+  const locations = locs(body);
+
+  if (/<sitemapindex\b/i.test(body)) {
+    return { type: "sitemapindex", urls: [], sitemaps: locations };
+  }
+
+  if (/<urlset\b/i.test(body)) {
+    return { type: "urlset", urls: locations, sitemaps: [] };
+  }
+
+  return { type: "unknown", urls: [], sitemaps: [] };
+};

package/src/snapshot.mjs

@@ -0,0 +1,119 @@
+import crypto from "node:crypto";
+import path from "node:path";
+import { extractHtmlEvidence } from "./html-extract.mjs";
+import { fetchWithGuards, readResponseTextLimited, readTextFileLimited, resolveLimits } from "./io-guards.mjs";
+import { renderHtml } from "./render.mjs";
+import { isHttpUrl } from "./url-utils.mjs";
+
+const userAgent = "RankForge GEO SEO audit snapshot";
+
+const hash = (value) => crypto.createHash("sha256").update(value).digest("hex");
+
+const headersToObject = (headers) => {
+  const result = {};
+  for (const [key, value] of headers.entries()) result[key] = value;
+  return result;
+};
+
+const fetchRaw = async (target, options = {}) => {
+  let current = target;
+  const redirectChain = [];
+  const maxRedirects = options.maxRedirects ?? 5;
+  const limits = resolveLimits(options.limits);
+
+  for (let attempt = 0; attempt <= maxRedirects; attempt++) {
+    const response = await fetchWithGuards(current, {
+      security: options.security,
+      limits,
+      fetchOptions: {
+        headers: { "user-agent": userAgent },
+        redirect: "manual",
+      },
+    });
+
+    const headers = headersToObject(response.headers);
+    const location = response.headers.get("location");
+    if ([301, 302, 303, 307, 308].includes(response.status) && location) {
+      const nextUrl = new URL(location, current).href;
+      redirectChain.push({ url: current, status: response.status, location: nextUrl });
+      current = nextUrl;
+      continue;
+    }
+
+    return {
+      sourceType: "url",
+      finalUrl: response.url || current,
+      status: response.status,
+      ok: response.ok,
+      headers,
+      redirectChain,
+      html: await readResponseTextLimited(response, {
+        limits,
+        maxBytes: limits.maxHtmlBytes,
+        label: current,
+      }),
+    };
+  }
+
+  throw new Error(`Too many redirects while fetching ${target}`);
+};
+
+const readFile = (target, options = {}) => {
+  const fullPath = path.resolve(target);
+  const limits = resolveLimits(options.limits);
+  return {
+    sourceType: "file",
+    finalUrl: fullPath,
+    status: null,
+    ok: true,
+    headers: {},
+    redirectChain: [],
+    html: readTextFileLimited(fullPath, {
+      security: options.security,
+      limits,
+      maxBytes: limits.maxHtmlBytes,
+    }),
+  };
+};
+
+export const collectSnapshot = async (target, options = {}) => {
+  const raw = isHttpUrl(target) ? await fetchRaw(target, options) : readFile(target, options);
+  const baseUrl = raw.sourceType === "url" ? raw.finalUrl : null;
+  const evidence = extractHtmlEvidence(raw.html, baseUrl);
+  const shouldRender = options.render === "always" || options.render === "auto" || options.renderer;
+
+  let render = { status: "not_requested" };
+  if (shouldRender) {
+    const rendered = await renderHtml(raw.finalUrl, {
+      ...options,
+      html: raw.html,
+      finalUrl: raw.finalUrl,
+    });
+    if (rendered.status === "rendered") {
+      const renderedEvidence = extractHtmlEvidence(rendered.html, baseUrl);
+      render = {
+        status: "rendered",
+        renderedHash: hash(rendered.html),
+        evidence: renderedEvidence,
+        textDeltaCharacters: Math.abs(
+          (renderedEvidence.counts?.visibleTextCharacters || 0) - (evidence.counts?.visibleTextCharacters || 0),
+        ),
+      };
+    } else {
+      render = rendered;
+    }
+  }
+
+  return {
+    target,
+    sourceType: raw.sourceType,
+    finalUrl: raw.finalUrl,
+    status: raw.status,
+    ok: raw.ok,
+    headers: raw.headers,
+    redirectChain: raw.redirectChain,
+    rawHash: hash(raw.html),
+    evidence,
+    render,
+  };
+};