rankforge 0.3.0

package/src/robots.mjs

@@ -0,0 +1,60 @@
+const emptyGroup = () => ({ agents: [], rules: [] });
+
+export const parseRobotsTxt = (text) => {
+  const groups = [];
+  let current = emptyGroup();
+
+  const pushCurrent = () => {
+    if (current.agents.length || current.rules.length) groups.push(current);
+    current = emptyGroup();
+  };
+
+  for (const rawLine of String(text || "").split(/\r?\n/)) {
+    const line = rawLine.replace(/#.*/, "").trim();
+    if (!line) continue;
+
+    const [rawKey, ...rawValue] = line.split(":");
+    const key = rawKey.trim().toLowerCase();
+    const value = rawValue.join(":").trim();
+
+    if (key === "user-agent") {
+      if (current.rules.length) pushCurrent();
+      current.agents.push(value.toLowerCase());
+      continue;
+    }
+
+    if ((key === "allow" || key === "disallow") && current.agents.length) {
+      current.rules.push({ type: key, path: value });
+    }
+  }
+
+  pushCurrent();
+  return { groups };
+};
+
+const pathFor = (pathOrUrl) => {
+  try {
+    const parsed = new URL(pathOrUrl);
+    return parsed.pathname || "/";
+  } catch {
+    return String(pathOrUrl || "/");
+  }
+};
+
+const matchingGroups = (parsed, userAgent) => {
+  const agent = String(userAgent || "").toLowerCase();
+  const groups = parsed?.groups || [];
+  const exact = groups.filter((group) => group.agents.some((item) => item !== "*" && agent.includes(item)));
+  return exact.length ? exact : groups.filter((group) => group.agents.includes("*"));
+};
+
+export const isAllowedByRobots = (parsed, pathOrUrl, userAgent = "RankForgeBot") => {
+  const pathname = pathFor(pathOrUrl);
+  const rules = matchingGroups(parsed, userAgent)
+    .flatMap((group) => group.rules)
+    .filter((rule) => rule.path && pathname.startsWith(rule.path))
+    .sort((a, b) => b.path.length - a.path.length || (a.type === "allow" ? -1 : 1));
+
+  if (!rules.length) return true;
+  return rules[0].type === "allow";
+};

package/src/rule-depth.mjs

@@ -0,0 +1,190 @@
+import { cleanText } from "./html-extract.mjs";
+import { structuredDataNodes, structuredDataTypeNames } from "./structured-data.mjs";
+import { normalizeUrl } from "./url-utils.mjs";
+
+const noindexPattern = /(?:^|,|\s)noindex(?:,|\s|$)/i;
+const entitySchemaTypes = new Set(["Organization", "LocalBusiness", "Person", "Product", "Article"]);
+const supportedStructuredDataFields = {
+  Organization: "name",
+  LocalBusiness: "name",
+  Product: "name",
+  Article: "headline",
+};
+const stopwords = new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "by",
+  "for",
+  "from",
+  "in",
+  "of",
+  "on",
+  "or",
+  "the",
+  "to",
+  "with",
+]);
+
+const evidencePath = (pageIndex, path) => `$.pages[${pageIndex}].evidence.${path}`;
+
+const normalizeText = (value) =>
+  cleanText(value)
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s]/gu, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+
+const tokensFor = (value) =>
+  normalizeText(value)
+    .split(/\s+/)
+    .filter((token) => token.length > 2 && !stopwords.has(token));
+
+const isNoindexed = (evidence = {}) => noindexPattern.test(evidence.robots || "");
+
+const isSuccessfulPage = (page) => !Number.isInteger(page.status) || page.status < 400;
+
+const sameCanonical = (page) => {
+  const canonical = page.evidence?.canonical;
+  if (!canonical) return true;
+  try {
+    return normalizeUrl(canonical) === normalizeUrl(page.finalUrl);
+  } catch {
+    return canonical === page.finalUrl;
+  }
+};
+
+const visibleContentSurface = (evidence = {}) =>
+  normalizeText([
+    evidence.title,
+    evidence.description,
+    evidence.siteName,
+    ...(Array.isArray(evidence.h1) ? evidence.h1 : []),
+    ...(Array.isArray(evidence.headings) ? evidence.headings.map((heading) => heading.text) : []),
+    evidence.visibleTextPreview,
+  ].filter(Boolean).join(" "));
+
+const valueAppearsInSurface = (value, surface) => {
+  const normalizedValue = normalizeText(value);
+  if (normalizedValue.length < 4) return true;
+  const paddedSurface = ` ${surface} `;
+  if (paddedSurface.includes(` ${normalizedValue} `)) return true;
+
+  const tokens = tokensFor(normalizedValue);
+  if (tokens.length === 0) return true;
+  const surfaceTokens = new Set(tokensFor(surface));
+  const matched = tokens.filter((token) => surfaceTokens.has(token)).length;
+  if (tokens.length === 1) return matched === 1;
+  return matched >= Math.max(2, Math.ceil(tokens.length * 0.5));
+};
+
+export const contentFingerprint = (value) => {
+  const normalized = normalizeText(value);
+  return normalized.length >= 500 ? normalized : "";
+};
+
+export const structuredDataVisibleContentFacts = (page, pageIndex = 0) => {
+  const evidence = page.evidence || {};
+  const surface = visibleContentSurface(evidence);
+  const facts = [];
+
+  for (const [blockIndex, block] of (evidence.structuredData || []).entries()) {
+    if (!block?.data || block.parseError) continue;
+
+    for (const node of structuredDataNodes(block.data)) {
+      for (const type of structuredDataTypeNames(node["@type"])) {
+        const property = supportedStructuredDataFields[type];
+        if (!property) continue;
+
+        const value = node[property];
+        if (Array.isArray(value) || typeof value === "object") continue;
+        const cleaned = cleanText(value);
+        if (cleaned.length < 4 || valueAppearsInSurface(cleaned, surface)) continue;
+
+        facts.push({
+          ruleId: "structured_data.visible_content_mismatch",
+          evidence: [
+            evidencePath(pageIndex, `structuredData[${blockIndex}]`),
+            evidencePath(pageIndex, "visibleTextPreview"),
+            evidencePath(pageIndex, "title"),
+            evidencePath(pageIndex, "h1"),
+            evidencePath(pageIndex, "headings"),
+          ],
+          impact: `${type} structured data names "${cleaned}", but that value is not visible in page evidence.`,
+        });
+      }
+    }
+  }
+
+  return facts;
+};
+
+const hasAboutOrContactLink = (evidence = {}) =>
+  (evidence.links || []).some((link) => {
+    const haystack = `${link.href || ""} ${link.text || ""}`.toLowerCase();
+    return haystack.includes("about") || haystack.includes("contact");
+  });
+
+const entitySignalCount = (evidence = {}) => {
+  let count = 0;
+  if (cleanText(evidence.siteName)) count += 1;
+  if ((evidence.schemaTypes || []).some((type) => entitySchemaTypes.has(type))) count += 1;
+  if (hasAboutOrContactLink(evidence)) count += 1;
+  if ((evidence.entitySignals?.authors || []).length > 0) count += 1;
+  if ((evidence.entitySignals?.dates || []).length > 0) count += 1;
+  return count;
+};
+
+export const entityClarityFacts = (page, pageIndex = 0) => {
+  const evidence = page.evidence || {};
+  const visibleTextCharacters = evidence.counts?.visibleTextCharacters || 0;
+  const hasPurposeSignal = cleanText(evidence.title) || (evidence.h1 || []).some((item) => cleanText(item));
+
+  if (!isSuccessfulPage(page) || isNoindexed(evidence) || visibleTextCharacters < 800 || !hasPurposeSignal) return [];
+  const signalCount = entitySignalCount(evidence);
+  if (signalCount > 1) return [];
+
+  return [{
+    ruleId: "geo.entity_clarity_gap",
+    evidence: [
+      evidencePath(pageIndex, "siteName"),
+      evidencePath(pageIndex, "schemaTypes"),
+      evidencePath(pageIndex, "links"),
+      evidencePath(pageIndex, "entitySignals"),
+      evidencePath(pageIndex, "counts.visibleTextCharacters"),
+    ],
+    impact: `Substantial page content has only ${signalCount} deterministic entity signal(s).`,
+  }];
+};
+
+export const duplicateContentClusterFacts = (pages = []) => {
+  const groups = new Map();
+
+  for (const [index, page] of pages.entries()) {
+    const evidence = page.evidence || {};
+    if (!isSuccessfulPage(page) || isNoindexed(evidence) || !sameCanonical(page)) continue;
+    if ((evidence.counts?.visibleTextCharacters || 0) < 600) continue;
+
+    const fingerprint = contentFingerprint(evidence.visibleTextPreview || "");
+    if (!fingerprint) continue;
+
+    const group = groups.get(fingerprint) || [];
+    group.push({ page, index });
+    groups.set(fingerprint, group);
+  }
+
+  return [...groups.values()]
+    .filter((group) => group.length >= 3)
+    .map((group) => ({
+      ruleId: "policy.duplicate_content_cluster",
+      affectedUrls: group.map(({ page }) => page.finalUrl),
+      evidence: group.flatMap(({ index }) => [
+        evidencePath(index, "visibleTextPreview"),
+        evidencePath(index, "counts.visibleTextCharacters"),
+      ]),
+      impact: `${group.length} pages share the same substantial normalized visible text preview.`,
+    }));
+};

package/src/rule-engine.mjs

@@ -0,0 +1,360 @@
+import { implementationTaskFor } from "./finding-task.mjs";
+import { renderParityFacts } from "./render-parity.mjs";
+import { entityClarityFacts, structuredDataVisibleContentFacts } from "./rule-depth.mjs";
+import { getRule } from "./rules.mjs";
+import { validateStructuredData } from "./structured-data.mjs";
+
+const severityImpact = {
+  P0: 60,
+  P1: 40,
+  P2: 20,
+  P3: 10,
+};
+
+const ownerFor = (dimension) => {
+  if (
+    dimension === "structured_data" ||
+    dimension === "technical" ||
+    dimension === "crawl_index" ||
+    dimension === "performance"
+  ) {
+    return "Engineering";
+  }
+  if (dimension === "helpful_content" || dimension === "geo_readiness" || dimension === "entity_clarity") {
+    return "Content";
+  }
+  return "SEO";
+};
+
+const effortFor = (severity) => (severity === "P0" || severity === "P1" ? "M" : "S");
+
+const noindexPattern = /(?:^|,|\s)noindex(?:,|\s|$)/i;
+
+const headerValue = (headers, name) => {
+  const wanted = name.toLowerCase();
+  for (const [key, value] of Object.entries(headers || {})) {
+    if (key.toLowerCase() === wanted) return String(value);
+  }
+  return "";
+};
+
+const isHomepageLike = (snapshot) => {
+  try {
+    const parsed = new URL(snapshot.finalUrl);
+    return parsed.pathname === "/" || /\/index\.html?$/i.test(parsed.pathname);
+  } catch {
+    return /(?:^|\/)index\.html?$/i.test(snapshot.finalUrl || "");
+  }
+};
+
+const structuredDataTypes = (value) => {
+  if (!value) return [];
+  if (Array.isArray(value)) return value.flatMap(structuredDataTypes);
+  if (typeof value !== "object") return [];
+
+  const types = [];
+  if (value["@type"]) {
+    if (Array.isArray(value["@type"])) types.push(...value["@type"].map(String));
+    else types.push(String(value["@type"]));
+  }
+  if (value["@graph"]) types.push(...structuredDataTypes(value["@graph"]));
+  return types;
+};
+
+const hasStructuredDataType = (evidence, type) =>
+  (evidence.structuredData || []).some((item) => structuredDataTypes(item.data).includes(type));
+
+const hasAboutOrContactLink = (evidence) =>
+  (evidence.links || []).some((link) => {
+    const haystack = `${link.href || ""} ${link.text || ""}`.toLowerCase();
+    return haystack.includes("about") || haystack.includes("contact");
+  });
+
+const createFinding = (ruleId, snapshot, evidence, pageIndex, impact = null) => {
+  const rule = getRule(ruleId);
+  if (!rule) throw new Error(`Unknown rule: ${ruleId}`);
+  const owner = ownerFor(rule.dimension);
+  const effort = effortFor(rule.defaultSeverity);
+
+  return {
+    ruleId: rule.id,
+    title: rule.title,
+    severity: rule.defaultSeverity,
+    dimension: rule.dimension,
+    affectedUrls: [snapshot.finalUrl],
+    evidence,
+    impact: impact || rule.title,
+    recommendation: rule.recommendation,
+    implementationTask: implementationTaskFor(rule, owner, effort),
+    owner,
+    effort,
+    confidence: "high",
+    sources: rule.sources,
+    pageIndex,
+  };
+};
+
+export const evaluatePage = (snapshot, pageIndex = 0) => {
+  const findings = [];
+  const evidence = snapshot.evidence || {};
+  const visibleTextCharacters = evidence.counts?.visibleTextCharacters || 0;
+
+  if ((snapshot.redirectChain || []).length > 1) {
+    findings.push(
+      createFinding(
+        "technical.redirect_chain",
+        snapshot,
+        [`$.pages[${pageIndex}].redirectChain`],
+        pageIndex,
+        "Long redirect chains slow crawling and make canonical URL resolution less predictable.",
+      ),
+    );
+  }
+
+  if (Number.isInteger(snapshot.status) && snapshot.status >= 400) {
+    findings.push(
+      createFinding(
+        "technical.http_error",
+        snapshot,
+        [`$.pages[${pageIndex}].status`],
+        pageIndex,
+        "Important pages with HTTP errors may be ineligible for indexing or unusable for visitors.",
+      ),
+    );
+  }
+
+  if (snapshot.sourceType === "url" && /^http:\/\//i.test(snapshot.finalUrl)) {
+    findings.push(
+      createFinding(
+        "technical.https_missing",
+        snapshot,
+        [`$.pages[${pageIndex}].finalUrl`],
+        pageIndex,
+        "Important pages should be available over HTTPS.",
+      ),
+    );
+  }
+
+  if (snapshot.sourceType === "url" && !evidence.canonical && !noindexPattern.test(evidence.robots || "")) {
+    findings.push(
+      createFinding(
+        "indexability.canonical_missing",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.canonical`],
+        pageIndex,
+        "Canonical signals help consolidate duplicate or alternate URL signals.",
+      ),
+    );
+  }
+
+  for (const parityFact of renderParityFacts(snapshot, pageIndex)) {
+    findings.push(
+      createFinding(parityFact.ruleId, snapshot, parityFact.evidence, pageIndex, parityFact.impact),
+    );
+  }
+
+  if (noindexPattern.test(evidence.robots || "")) {
+    findings.push(
+      createFinding(
+        "indexability.noindex",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.robots`],
+        pageIndex,
+        "Pages with noindex are not eligible to appear in Google Search results.",
+      ),
+    );
+  }
+
+  if (noindexPattern.test(evidence.robots || "") && evidence.canonical) {
+    findings.push(
+      createFinding(
+        "indexability.noindex_canonical_conflict",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.robots`, `$.pages[${pageIndex}].evidence.canonical`],
+        pageIndex,
+        "Combining noindex with a canonical signal creates an ambiguous indexing strategy.",
+      ),
+    );
+  }
+
+  if (noindexPattern.test(headerValue(snapshot.headers, "x-robots-tag"))) {
+    findings.push(
+      createFinding(
+        "indexability.x_robots_noindex",
+        snapshot,
+        [`$.pages[${pageIndex}].headers.x-robots-tag`],
+        pageIndex,
+        "X-Robots-Tag noindex prevents indexing even when page HTML appears indexable.",
+      ),
+    );
+  }
+
+  if (!evidence.title) {
+    findings.push(
+      createFinding(
+        "appearance.title_missing",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.title`],
+        pageIndex,
+        "Missing titles limit control over search result title links.",
+      ),
+    );
+  }
+
+  if (!evidence.description) {
+    findings.push(
+      createFinding(
+        "appearance.meta_description_missing",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.description`],
+        pageIndex,
+        "Missing descriptions reduce the page's ability to provide useful snippet text.",
+      ),
+    );
+  }
+
+  if (!Array.isArray(evidence.h1) || evidence.h1.length === 0) {
+    findings.push(
+      createFinding(
+        "appearance.h1_missing",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.h1`],
+        pageIndex,
+        "A missing primary heading makes page purpose less explicit for users and crawlers.",
+      ),
+    );
+  }
+
+  if ((evidence.images || []).some((image) => image.alt === null || image.alt === "")) {
+    findings.push(
+      createFinding(
+        "appearance.image_alt_missing",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.images`],
+        pageIndex,
+        "Images without alt text provide weaker context for accessibility and image search.",
+      ),
+    );
+  }
+
+  if (snapshot.sourceType === "url" && isHomepageLike(snapshot) && !evidence.favicon) {
+    findings.push(
+      createFinding(
+        "appearance.favicon_missing",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.favicon`],
+        pageIndex,
+        "A missing favicon weakens brand presentation in search appearance surfaces.",
+      ),
+    );
+  }
+
+  if ((evidence.structuredData || []).some((item) => item.parseError)) {
+    findings.push(
+      createFinding(
+        "structured_data.invalid_jsonld",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.structuredData`],
+        pageIndex,
+        "Invalid JSON-LD cannot be parsed reliably for structured data eligibility.",
+      ),
+    );
+  }
+
+  for (const issue of validateStructuredData(evidence.structuredData || [])) {
+    findings.push(
+      createFinding(
+        "structured_data.required_property_missing",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.structuredData[${issue.blockIndex}]`],
+        pageIndex,
+        `${issue.type} structured data is missing required properties: ${issue.missing.join(", ")}.`,
+      ),
+    );
+  }
+
+  for (const depthFact of [
+    ...structuredDataVisibleContentFacts(snapshot, pageIndex),
+    ...entityClarityFacts(snapshot, pageIndex),
+  ]) {
+    findings.push(
+      createFinding(depthFact.ruleId, snapshot, depthFact.evidence, pageIndex, depthFact.impact),
+    );
+  }
+
+  if (visibleTextCharacters > 0 && visibleTextCharacters < 300) {
+    findings.push(
+      createFinding(
+        "content.thin_content",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.counts.visibleTextCharacters`],
+        pageIndex,
+        "Thin pages are less likely to satisfy visitor tasks or support AI/search answerability.",
+      ),
+    );
+  }
+
+  if (visibleTextCharacters >= 500 && (evidence.headings || []).length < 2) {
+    findings.push(
+      createFinding(
+        "content.answerability_gap",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.headings`, `$.pages[${pageIndex}].evidence.counts.visibleTextCharacters`],
+        pageIndex,
+        "Substantial pages need clear answerable sections to support users and generative search features.",
+      ),
+    );
+  }
+
+  if (
+    isHomepageLike(snapshot) &&
+    !hasStructuredDataType(evidence, "Organization") &&
+    !hasStructuredDataType(evidence, "LocalBusiness")
+  ) {
+    findings.push(
+      createFinding(
+        "structured_data.organization_missing",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.structuredData`],
+        pageIndex,
+        "Homepage-like pages should make organization identity clear where visible content supports it.",
+      ),
+    );
+  }
+
+  if (isHomepageLike(snapshot) && !hasAboutOrContactLink(evidence)) {
+    findings.push(
+      createFinding(
+        "entity.about_contact_missing",
+        snapshot,
+        [`$.pages[${pageIndex}].evidence.links`],
+        pageIndex,
+        "Homepage-like pages should expose crawlable about or contact paths for entity trust signals.",
+      ),
+    );
+  }
+
+  return findings;
+};
+
+export const scoreFindings = (findings) => {
+  const scores = {};
+
+  for (const finding of findings) {
+    const score = scores[finding.dimension] || {
+      score: 100,
+      findings: [],
+      p0: 0,
+      p1: 0,
+      p2: 0,
+      p3: 0,
+    };
+
+    score.findings.push(finding.ruleId);
+    score[finding.severity.toLowerCase()] += 1;
+    score.score = Math.max(0, score.score - severityImpact[finding.severity]);
+    scores[finding.dimension] = score;
+  }
+
+  return scores;
+};