seo 0.16.0 → 0.16.1

package/dist/mcp.js

@@ -1,5 +1,5 @@
 import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
-import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-CV5s7esU.js";
+import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-Cvmg0qhZ.js";
 import { createHash, randomUUID, sign } from "node:crypto";
 import { ZodOptional, z } from "zod";
 import { arch, homedir, platform, tmpdir, totalmem } from "node:os";
@@ -19158,17 +19158,15 @@ function recordHostFetch(input) {
 	const cfg = input.rate.backpressure;
 	health.durations.push(input.durationMs);
 	if (health.durations.length > 20) health.durations.shift();
-	const blocked = [
-		401,
-		403,
-		429
-	].includes(input.status);
-	const errored = input.status >= 500;
+	const rateLimited = input.status === 429 || input.status === 503 && input.retryAfterMs !== void 0;
+	const blocked = [401, 403].includes(input.status);
+	const errored = input.status >= 500 && !rateLimited;
 	const slow = input.durationMs >= cfg.slowMs;
 	health.consecutiveBlocked = blocked ? health.consecutiveBlocked + 1 : 0;
 	health.consecutiveErrors = errored ? health.consecutiveErrors + 1 : 0;
 	health.consecutiveSlow = slow ? health.consecutiveSlow + 1 : 0;
-	if (!blocked && !errored && !slow) health.reason = void 0;
+	if (!blocked && !errored && !slow && !rateLimited) health.reason = void 0;
+	if (rateLimited) health.reason = `origin rate limited the crawler (HTTP ${input.status})`;
 	const retryDelay = input.retryAfterMs === void 0 ? 0 : Math.min(input.retryAfterMs, cfg.retryAfterCapMs);
 	if (retryDelay > 0) {
 		health.reason = `origin sent Retry-After (${Math.round(retryDelay / 1e3)}s)`;
@@ -22662,6 +22660,17 @@ function isAuditableHtmlPage(page) {
 	const contentAuditAllowed = page.contentAuditAllowed ?? (!page.blocked && page.robotsTxt?.allowed !== false);
 	return isSuccessfulResponse(page) && isHtmlPage(page) && !isRedirectedPage(page) && contentAuditAllowed;
 }
+/**
+* Cloudflare serves its own endpoints under `/cdn-cgi/`, such as the email
+* obfuscation link `/cdn-cgi/l/email-protection`. They are not site pages.
+*/
+function isCdnUtilityUrl(value) {
+	try {
+		return new URL(value).pathname.startsWith("/cdn-cgi/");
+	} catch {
+		return false;
+	}
+}
 // packages/core/dist/analyze/crawler/audit.js
 function sameUrl(a, b) {
 	if (!a || !b) return false;
@@ -22931,7 +22940,7 @@ function auditCrawlPages(pages, opts = {}) {
 		if (page.accessBlock) continue;
 		if (page.status >= 500) {
 			issues.push(issue$1("server_error", page, String(page.status), { status: page.status }));
-			if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
+			if ((page.internalInlinkCount ?? 0) > 0 && !isCdnUtilityUrl(page.url)) issues.push(issue$1("broken_internal_link", page, String(page.status), {
 				status: page.status,
 				internalInlinkCount: page.internalInlinkCount
 			}));
@@ -22939,7 +22948,7 @@ function auditCrawlPages(pages, opts = {}) {
 		}
 		if (page.status >= 400) {
 			issues.push(issue$1("client_error", page, String(page.status), { status: page.status }));
-			if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
+			if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0 && !isCdnUtilityUrl(page.url)) issues.push(issue$1("broken_internal_link", page, String(page.status), {
 				status: page.status,
 				internalInlinkCount: page.internalInlinkCount
 			}));
@@ -23162,6 +23171,7 @@ function auditCrawlPages(pages, opts = {}) {
 // packages/core/dist/analyze/crawler/crawl-skip-reasons.js
 const CRAWL_SKIP_REASONS = [
 	"asset-url",
+	"cdn-utility-url",
 	"configured-exclusion",
 	"duplicate-normalization",
 	"invalid-url",
@@ -23176,6 +23186,7 @@ const CRAWL_SKIP_REASONS = [
 ];
 const SKIP_IMPACT = {
 	"asset-url": "non-impacting",
+	"cdn-utility-url": "non-impacting",
 	"configured-exclusion": "non-impacting",
 	"duplicate-normalization": "non-impacting",
 	"invalid-url": "non-impacting",
@@ -23236,161 +23247,85 @@ function crawlSkipReasonCountsFromRecord(input = {}) {
 		count
 	})) }).skipReasons;
 }
-// packages/core/dist/analyze/crawler/top-fixes.js
-const SEVERITY_SCORE = {
-	high: 5e3,
-	medium: 1200,
-	low: 100
-};
-const EFFORT_BY_CATEGORY = {
-	canonical: "medium",
-	content: "high",
-	agents: "low",
-	headings: "low",
-	images: "medium",
-	indexability: "medium",
-	international: "low",
-	metadata: "low",
-	mobile: "low",
-	response: "medium",
-	security: "medium",
-	social: "low",
-	"structured-data": "medium"
-};
-const EFFORT_SCORE = {
-	low: 60,
-	medium: 30,
-	high: 0
-};
-const AFFECTED_URL_WEIGHT = {
-	high: 25,
-	medium: 15,
-	low: 2
-};
-const VALUE_WEIGHT = {
-	high: 1,
-	medium: .75,
-	low: .15
-};
-const CRAWL_REACH_RULES = /* @__PURE__ */ new Set(["crawl_single_page_reach"]);
-function reachRank(ruleId) {
-	return CRAWL_REACH_RULES.has(ruleId) ? 0 : 1;
+// packages/core/dist/analyze/crawler/crawl-url-queue.js
+function before(left, right) {
+	return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
 }
-function matchesPattern(pattern, value) {
-	if (!pattern.includes("*")) return value.includes(pattern);
-	const parts = pattern.split("*");
-	let position = 0;
-	for (const part of parts) {
-		if (!part) continue;
-		const found = value.slice(position).indexOf(part);
-		if (found === -1) return false;
-		position += found + part.length;
+var CrawlUrlQueue = class {
+	#items = [];
+	#positions = /* @__PURE__ */ new Map();
+	get size() {
+		return this.#items.length;
 	}
-	return true;
-}
-function filteredIssues(report, filters) {
-	return report.issues.filter((issue) => {
-		if (filters.category && issue.category !== filters.category) return false;
-		if (filters.severity && issue.severity !== filters.severity) return false;
-		if (filters.url && !matchesPattern(filters.url, issue.url)) return false;
-		if ((filters.recommendation ?? "fix") !== recommendationForRule(issue.ruleId)) return false;
+	has(url) {
+		return this.#positions.has(url);
+	}
+	push(item) {
+		if (this.has(item.url)) throw new Error(`URL is already queued: ${item.url}`);
+		const index = this.#items.length;
+		this.#items.push(item);
+		this.#positions.set(item.url, index);
+		this.#bubbleUp(index);
+	}
+	decreaseDepth(url, depth) {
+		const index = this.#positions.get(url);
+		if (index === void 0) return false;
+		const item = this.#items[index];
+		if (!item || depth >= item.depth) return false;
+		item.depth = depth;
+		this.#bubbleUp(index);
 		return true;
-	});
-}
-function groupIssues(issues) {
-	const groups = /* @__PURE__ */ new Map();
-	for (const issue of issues) {
-		const existing = groups.get(issue.ruleId) ?? {
-			ruleId: issue.ruleId,
-			title: issue.title,
-			category: issue.category,
-			severity: issue.severity,
-			count: 0,
-			sampleUrls: []
-		};
-		existing.count += 1;
-		if (existing.sampleUrls.length < 10) existing.sampleUrls.push(issue.url);
-		groups.set(issue.ruleId, existing);
 	}
-	return [...groups.values()];
-}
-function searchValueForGroup(report, urls) {
-	const wanted = new Set(urls);
-	const pages = report.pages.filter((page) => wanted.has(page.url) || wanted.has(page.finalUrl));
-	const metrics = pages.map((page) => page.searchMetrics).filter((item) => Boolean(item));
-	const clicks = metrics.reduce((sum, item) => sum + item.clicks, 0);
-	const impressions = metrics.reduce((sum, item) => sum + item.impressions, 0);
-	const sessions = pages.reduce((sum, page) => sum + (page.analytics?.sessions ?? 0), 0);
-	const observedMetrics = new Set(report.dataSources?.analytics.observedMetrics ?? [
-		"sessions",
-		"totalUsers",
-		"conversions"
-	]);
-	const totalUsers = observedMetrics.has("totalUsers") ? pages.reduce((sum, page) => sum + (page.analytics?.totalUsers ?? 0), 0) : void 0;
-	const conversions = observedMetrics.has("conversions") ? pages.reduce((sum, page) => sum + (page.analytics?.conversions ?? 0), 0) : void 0;
-	const avgPosition = metrics.length ? metrics.reduce((sum, item) => sum + item.position, 0) / metrics.length : void 0;
-	return {
-		searchVisibleUrls: metrics.length,
-		clicks,
-		impressions,
-		sessions,
-		totalUsers,
-		conversions,
-		avgPosition
-	};
-}
-function whyThisRanks(input, recommendation, searchStatus, analyticsProvider) {
-	const visibility = input.searchVisibleUrls ? `${input.searchVisibleUrls} affected URLs have GSC visibility (${input.clicks} clicks, ${input.impressions} impressions).` : searchStatus && !["joined", "none"].includes(searchStatus) ? `No joined GSC visibility is available for these affected URLs; Search Console evidence is ${searchStatus}.` : "No affected URL has joined GSC visibility yet.";
-	const analyticsDetails = analyticsProvider ? analyticsProviderDetails(analyticsProvider) : analyticsProviderDetails("google");
-	const analyticsLabel = analyticsDetails.label;
-	const sessionLabel = analyticsDetails.landingMetricLabel;
-	return `${visibility}${input.sessions ? ` ${analyticsLabel} adds ${input.sessions} ${sessionLabel}${input.conversions === void 0 ? "" : ` and ${input.conversions} conversions`} from affected pages.` : ""} ${recommendation === "review" ? "This is a review observation. Confirm it before scheduling implementation work." : `This ranks as a fix. Severity is ${input.severity}; ${input.affectedUrls} URL${input.affectedUrls === 1 ? " is" : "s are"} affected; estimated effort is ${input.effort}.`}`;
-}
-function verificationCommand(report, severity) {
-	return `seo crawl ${report.config.url} --severity ${severity} --max-pages ${Math.min(report.config.maxPages, 100)}`;
-}
-function topFixes(report, filters = {}) {
-	return groupIssues(filteredIssues(report, filters)).map((group) => {
-		const search = searchValueForGroup(report, group.sampleUrls);
-		const effort = EFFORT_BY_CATEGORY[group.category] ?? "medium";
-		const scoreFactors = {
-			severity: SEVERITY_SCORE[group.severity],
-			affectedUrls: group.count,
-			searchVisibleUrls: search.searchVisibleUrls,
-			clicks: search.clicks,
-			impressions: search.impressions,
-			sessions: search.sessions,
-			totalUsers: search.totalUsers,
-			conversions: search.conversions,
-			avgPosition: search.avgPosition,
-			effort,
-			effortScore: EFFORT_SCORE[effort]
-		};
-		const valueWeight = VALUE_WEIGHT[group.severity];
-		const score = scoreFactors.severity + scoreFactors.affectedUrls * AFFECTED_URL_WEIGHT[group.severity] + scoreFactors.searchVisibleUrls * 100 * valueWeight + scoreFactors.clicks * 20 * valueWeight + Math.min(scoreFactors.impressions, 1e4) / 25 * valueWeight + scoreFactors.sessions * 2 * valueWeight + (scoreFactors.conversions ?? 0) * 100 * valueWeight + scoreFactors.effortScore;
-		const rule = explainRule(group.ruleId);
-		const recommendation = recommendationForRule(group.ruleId);
-		return {
-			...group,
-			recommendation,
-			score: Math.round(score),
-			scoreFactors,
-			whyThisRanks: whyThisRanks(scoreFactors, recommendation, report.dataSources?.searchConsole.status, report.dataSources?.analytics.provider),
-			howToFix: rule?.howToFix ?? "",
-			howToVerify: rule?.howToVerify ?? "",
-			verification: {
-				command: verificationCommand(report, group.severity),
-				expected: rule?.howToVerify ?? ""
-			}
-		};
-	}).sort((a, b) => reachRank(a.ruleId) - reachRank(b.ruleId) || b.score - a.score || SEVERITY_SCORE[b.severity] - SEVERITY_SCORE[a.severity] || b.count - a.count || a.ruleId.localeCompare(b.ruleId)).slice(0, filters.limit ?? 10);
-}
-function reviewObservations(report, filters = {}) {
-	return topFixes(report, {
-		...filters,
-		recommendation: "review"
-	});
-}
+	take() {
+		const first = this.#items[0];
+		if (!first) return void 0;
+		const last = this.#items.pop();
+		this.#positions.delete(first.url);
+		if (last && last !== first) {
+			this.#items[0] = last;
+			this.#positions.set(last.url, 0);
+			this.#bubbleDown(0);
+		}
+		return first;
+	}
+	#swap(left, right) {
+		const leftItem = this.#items[left];
+		const rightItem = this.#items[right];
+		if (!leftItem || !rightItem) return;
+		this.#items[left] = rightItem;
+		this.#items[right] = leftItem;
+		this.#positions.set(rightItem.url, left);
+		this.#positions.set(leftItem.url, right);
+	}
+	#bubbleUp(start) {
+		let index = start;
+		while (index > 0) {
+			const parent = Math.floor((index - 1) / 2);
+			const item = this.#items[index];
+			const parentItem = this.#items[parent];
+			if (!item || !parentItem || !before(item, parentItem)) break;
+			this.#swap(index, parent);
+			index = parent;
+		}
+	}
+	#bubbleDown(start) {
+		let index = start;
+		while (true) {
+			const left = index * 2 + 1;
+			const right = left + 1;
+			let next = index;
+			const item = this.#items[next];
+			const leftItem = this.#items[left];
+			const rightItem = this.#items[right];
+			if (leftItem && item && before(leftItem, item)) next = left;
+			const nextItem = this.#items[next];
+			if (rightItem && nextItem && before(rightItem, nextItem)) next = right;
+			if (next === index) return;
+			this.#swap(index, next);
+			index = next;
+		}
+	}
+};
 // packages/core/dist/analyze/crawler/report.js
 const MAX_CRAWL_PAGES = 1e4;
 const MAX_CRAWL_CONCURRENCY = 16;
@@ -23560,6 +23495,10 @@ function sanitizeMessages(values = []) {
 function compareText$15(left, right) {
 	return left < right ? -1 : left > right ? 1 : 0;
 }
+function compareCaveats(left, right) {
+	const rank = (value) => value.startsWith("Coverage incomplete:") ? 0 : 1;
+	return rank(left) - rank(right) || compareText$15(left, right);
+}
 function sortPages(pages) {
 	return [...pages].sort((left, right) => compareText$15(left.url, right.url));
 }
@@ -23794,9 +23733,229 @@ function createCrawlReport(input) {
 		...input.externalLinkVerification ? { externalLinkVerification: sanitizeTenantValue(input.externalLinkVerification) } : {},
 		...input.siteChecks ? { siteChecks: sanitizeTenantValue(input.siteChecks) } : {},
 		warnings: sanitizeMessages(input.warnings).sort(compareText$15),
-		caveats: sanitizeMessages(input.caveats).sort(compareText$15)
+		caveats: sanitizeMessages(input.caveats).sort(compareCaveats)
+	};
+}
+// packages/core/dist/analyze/crawler/crawl-caveats.js
+/** Reports rank a caveat with this prefix first and lead with it. */
+const CRAWL_COVERAGE_CAVEAT_PREFIX = "Coverage incomplete:";
+function rateLimitRerunCommand(config, intervalMs) {
+	const scope = config.strategy === "health" ? " --health" : config.mode === "site" ? "" : ` --mode ${config.mode}`;
+	const sitemap = config.sitemapUrl ? ` --sitemap-url ${config.sitemapUrl}` : "";
+	const slowMs = Math.min(8e3, Math.max(2e3, intervalMs * 2));
+	return `seo crawl ${config.url}${scope}${sitemap} --max-pages ${config.maxPages} --concurrency 1 --fetch-interval-cap 1 --fetch-interval-ms ${slowMs}`;
+}
+/** Set when the sitemaps list more URLs than the page limit let the crawl read. */
+function sitemapLimitCaveatInput(config, discovery, defaulted) {
+	if (!discovery || discovery.urlsReturned < config.maxPages || !discovery.roots.some((root) => root.possiblyTruncated)) return;
+	return {
+		maxPages: config.maxPages,
+		defaulted,
+		ceiling: MAX_CRAWL_PAGES,
+		health: config.strategy === "health",
+		url: config.url
+	};
+}
+function crawlCaveats(input) {
+	const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
+	if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
+	if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
+	if (input.rateLimit) {
+		const { readPages, skippedUrls, rerunCommand } = input.rateLimit;
+		caveats.push(`${CRAWL_COVERAGE_CAVEAT_PREFIX} read ${readPages} of ${readPages + skippedUrls} pages because the origin kept rate limiting the crawl (HTTP 429) after it backed off. Rerun slower: ${rerunCommand}`);
+	}
+	if (input.sitemapLimit) {
+		const { maxPages, defaulted, ceiling, health, url } = input.sitemapLimit;
+		caveats.push(`Checked only the first ${maxPages} sitemap URLs because the page limit is ${maxPages}${defaulted ? " by default" : ""}, and the sitemaps list more. Raise --max-pages, up to ${ceiling.toLocaleString("en-US")}, to check more: seo crawl ${url}${health ? " --health" : ""} --max-pages ${ceiling}`);
+	}
+	if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
+	const renderingFallbackUrls = input.renderingFallbackUrls ?? 0;
+	if (renderingFallbackUrls > 0) caveats.push(`JavaScript rendering could not run for ${renderingFallbackUrls} page${renderingFallbackUrls === 1 ? "" : "s"}, so ${renderingFallbackUrls === 1 ? "it was" : "they were"} audited from raw HTML. Client-rendered content, metadata and links may be missing. Install Chromium or set SEO_BROWSER_EXECUTABLE_PATH, then rerun.`);
+	const queryVariantSkippedUrls = input.queryVariantSkippedUrls ?? 0;
+	if (queryVariantSkippedUrls > 0) caveats.push(`Left ${queryVariantSkippedUrls} query-string variant${queryVariantSkippedUrls === 1 ? "" : "s"} unqueued after 5 variants of the same path, so tracking parameters do not fill the crawl.`);
+	for (const issue of input.reachIssues ?? []) if (issue.detail) caveats.push(issue.detail);
+	return caveats;
+}
+/**
+* A site crawl that found no internal links on the start page and no sitemap
+* URLs checked one page, so its other findings cannot stand for the site.
+*/
+function crawlReachIssues(input) {
+	const [page] = input.pages;
+	if (input.crawledPages !== 1 || input.pages.length !== 1 || !page || page.outgoingInternalCount > 0 || input.sitemapUrlsReturned > 0) return [];
+	const rule = explainRule("crawl_single_page_reach");
+	if (!rule) throw new Error("Missing rule guidance for crawl_single_page_reach");
+	return [{
+		ruleId: "crawl_single_page_reach",
+		title: rule.title,
+		category: rule.category,
+		severity: rule.defaultSeverity,
+		url: page.url,
+		detail: page.usedJs ? "The rendered start page had 0 internal links and no sitemap returned URLs, so the crawl checked only this page. List the pages to check with --mode list." : "The start page had 0 internal links in its HTML and no sitemap returned URLs, so the crawl checked only this page. It probably builds its links with JavaScript. Rerun with --rendering on, or list the pages with --mode list.",
+		evidence: {
+			startUrl: input.startUrl,
+			crawledPages: 1,
+			outgoingInternalCount: 0,
+			sitemapUrlsReturned: 0,
+			rendered: Boolean(page.usedJs)
+		}
+	}];
+}
+// packages/core/dist/analyze/crawler/top-fixes.js
+const SEVERITY_SCORE = {
+	high: 5e3,
+	medium: 1200,
+	low: 100
+};
+const EFFORT_BY_CATEGORY = {
+	canonical: "medium",
+	content: "high",
+	agents: "low",
+	headings: "low",
+	images: "medium",
+	indexability: "medium",
+	international: "low",
+	metadata: "low",
+	mobile: "low",
+	response: "medium",
+	security: "medium",
+	social: "low",
+	"structured-data": "medium"
+};
+const EFFORT_SCORE = {
+	low: 60,
+	medium: 30,
+	high: 0
+};
+const AFFECTED_URL_WEIGHT = {
+	high: 25,
+	medium: 15,
+	low: 2
+};
+const VALUE_WEIGHT = {
+	high: 1,
+	medium: .75,
+	low: .15
+};
+const CRAWL_REACH_RULES = /* @__PURE__ */ new Set(["crawl_single_page_reach"]);
+function reachRank(ruleId) {
+	return CRAWL_REACH_RULES.has(ruleId) ? 0 : 1;
+}
+function matchesPattern(pattern, value) {
+	if (!pattern.includes("*")) return value.includes(pattern);
+	const parts = pattern.split("*");
+	let position = 0;
+	for (const part of parts) {
+		if (!part) continue;
+		const found = value.slice(position).indexOf(part);
+		if (found === -1) return false;
+		position += found + part.length;
+	}
+	return true;
+}
+function filteredIssues(report, filters) {
+	return report.issues.filter((issue) => {
+		if (filters.category && issue.category !== filters.category) return false;
+		if (filters.severity && issue.severity !== filters.severity) return false;
+		if (filters.url && !matchesPattern(filters.url, issue.url)) return false;
+		if ((filters.recommendation ?? "fix") !== recommendationForRule(issue.ruleId)) return false;
+		return true;
+	});
+}
+function groupIssues(issues) {
+	const groups = /* @__PURE__ */ new Map();
+	for (const issue of issues) {
+		const existing = groups.get(issue.ruleId) ?? {
+			ruleId: issue.ruleId,
+			title: issue.title,
+			category: issue.category,
+			severity: issue.severity,
+			count: 0,
+			sampleUrls: []
+		};
+		existing.count += 1;
+		if (existing.sampleUrls.length < 10) existing.sampleUrls.push(issue.url);
+		groups.set(issue.ruleId, existing);
+	}
+	return [...groups.values()];
+}
+function searchValueForGroup(report, urls) {
+	const wanted = new Set(urls);
+	const pages = report.pages.filter((page) => wanted.has(page.url) || wanted.has(page.finalUrl));
+	const metrics = pages.map((page) => page.searchMetrics).filter((item) => Boolean(item));
+	const clicks = metrics.reduce((sum, item) => sum + item.clicks, 0);
+	const impressions = metrics.reduce((sum, item) => sum + item.impressions, 0);
+	const sessions = pages.reduce((sum, page) => sum + (page.analytics?.sessions ?? 0), 0);
+	const observedMetrics = new Set(report.dataSources?.analytics.observedMetrics ?? [
+		"sessions",
+		"totalUsers",
+		"conversions"
+	]);
+	const totalUsers = observedMetrics.has("totalUsers") ? pages.reduce((sum, page) => sum + (page.analytics?.totalUsers ?? 0), 0) : void 0;
+	const conversions = observedMetrics.has("conversions") ? pages.reduce((sum, page) => sum + (page.analytics?.conversions ?? 0), 0) : void 0;
+	const avgPosition = metrics.length ? metrics.reduce((sum, item) => sum + item.position, 0) / metrics.length : void 0;
+	return {
+		searchVisibleUrls: metrics.length,
+		clicks,
+		impressions,
+		sessions,
+		totalUsers,
+		conversions,
+		avgPosition
 	};
 }
+function whyThisRanks(input, recommendation, searchStatus, analyticsProvider) {
+	const visibility = input.searchVisibleUrls ? `${input.searchVisibleUrls} affected URLs have GSC visibility (${input.clicks} clicks, ${input.impressions} impressions).` : searchStatus && !["joined", "none"].includes(searchStatus) ? `No joined GSC visibility is available for these affected URLs; Search Console evidence is ${searchStatus}.` : "No affected URL has joined GSC visibility yet.";
+	const analyticsDetails = analyticsProvider ? analyticsProviderDetails(analyticsProvider) : analyticsProviderDetails("google");
+	const analyticsLabel = analyticsDetails.label;
+	const sessionLabel = analyticsDetails.landingMetricLabel;
+	return `${visibility}${input.sessions ? ` ${analyticsLabel} adds ${input.sessions} ${sessionLabel}${input.conversions === void 0 ? "" : ` and ${input.conversions} conversions`} from affected pages.` : ""} ${recommendation === "review" ? "This is a review observation. Confirm it before scheduling implementation work." : `This ranks as a fix. Severity is ${input.severity}; ${input.affectedUrls} URL${input.affectedUrls === 1 ? " is" : "s are"} affected; estimated effort is ${input.effort}.`}`;
+}
+function verificationCommand(report, severity) {
+	return `seo crawl ${report.config.url} --severity ${severity} --max-pages ${Math.min(report.config.maxPages, 100)}`;
+}
+function topFixes(report, filters = {}) {
+	return groupIssues(filteredIssues(report, filters)).map((group) => {
+		const search = searchValueForGroup(report, group.sampleUrls);
+		const effort = EFFORT_BY_CATEGORY[group.category] ?? "medium";
+		const scoreFactors = {
+			severity: SEVERITY_SCORE[group.severity],
+			affectedUrls: group.count,
+			searchVisibleUrls: search.searchVisibleUrls,
+			clicks: search.clicks,
+			impressions: search.impressions,
+			sessions: search.sessions,
+			totalUsers: search.totalUsers,
+			conversions: search.conversions,
+			avgPosition: search.avgPosition,
+			effort,
+			effortScore: EFFORT_SCORE[effort]
+		};
+		const valueWeight = VALUE_WEIGHT[group.severity];
+		const score = scoreFactors.severity + scoreFactors.affectedUrls * AFFECTED_URL_WEIGHT[group.severity] + scoreFactors.searchVisibleUrls * 100 * valueWeight + scoreFactors.clicks * 20 * valueWeight + Math.min(scoreFactors.impressions, 1e4) / 25 * valueWeight + scoreFactors.sessions * 2 * valueWeight + (scoreFactors.conversions ?? 0) * 100 * valueWeight + scoreFactors.effortScore;
+		const rule = explainRule(group.ruleId);
+		const recommendation = recommendationForRule(group.ruleId);
+		return {
+			...group,
+			recommendation,
+			score: Math.round(score),
+			scoreFactors,
+			whyThisRanks: whyThisRanks(scoreFactors, recommendation, report.dataSources?.searchConsole.status, report.dataSources?.analytics.provider),
+			howToFix: rule?.howToFix ?? "",
+			howToVerify: rule?.howToVerify ?? "",
+			verification: {
+				command: verificationCommand(report, group.severity),
+				expected: rule?.howToVerify ?? ""
+			}
+		};
+	}).sort((a, b) => reachRank(a.ruleId) - reachRank(b.ruleId) || b.score - a.score || SEVERITY_SCORE[b.severity] - SEVERITY_SCORE[a.severity] || b.count - a.count || a.ruleId.localeCompare(b.ruleId)).slice(0, filters.limit ?? 10);
+}
+function reviewObservations(report, filters = {}) {
+	return topFixes(report, {
+		...filters,
+		recommendation: "review"
+	});
+}
 // packages/core/dist/analyze/crawler/site-checks-schema.js
 const crawlSiteChecksSchema = z.object({ soft404: z.object({
 	status: z.enum([
@@ -24912,123 +25071,6 @@ function agentReadyIssues(input) {
 	}
 	return issues;
 }
-// packages/core/dist/analyze/crawler/crawl-url-queue.js
-function before(left, right) {
-	return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
-}
-var CrawlUrlQueue = class {
-	#items = [];
-	#positions = /* @__PURE__ */ new Map();
-	get size() {
-		return this.#items.length;
-	}
-	has(url) {
-		return this.#positions.has(url);
-	}
-	push(item) {
-		if (this.has(item.url)) throw new Error(`URL is already queued: ${item.url}`);
-		const index = this.#items.length;
-		this.#items.push(item);
-		this.#positions.set(item.url, index);
-		this.#bubbleUp(index);
-	}
-	decreaseDepth(url, depth) {
-		const index = this.#positions.get(url);
-		if (index === void 0) return false;
-		const item = this.#items[index];
-		if (!item || depth >= item.depth) return false;
-		item.depth = depth;
-		this.#bubbleUp(index);
-		return true;
-	}
-	take() {
-		const first = this.#items[0];
-		if (!first) return void 0;
-		const last = this.#items.pop();
-		this.#positions.delete(first.url);
-		if (last && last !== first) {
-			this.#items[0] = last;
-			this.#positions.set(last.url, 0);
-			this.#bubbleDown(0);
-		}
-		return first;
-	}
-	#swap(left, right) {
-		const leftItem = this.#items[left];
-		const rightItem = this.#items[right];
-		if (!leftItem || !rightItem) return;
-		this.#items[left] = rightItem;
-		this.#items[right] = leftItem;
-		this.#positions.set(rightItem.url, left);
-		this.#positions.set(leftItem.url, right);
-	}
-	#bubbleUp(start) {
-		let index = start;
-		while (index > 0) {
-			const parent = Math.floor((index - 1) / 2);
-			const item = this.#items[index];
-			const parentItem = this.#items[parent];
-			if (!item || !parentItem || !before(item, parentItem)) break;
-			this.#swap(index, parent);
-			index = parent;
-		}
-	}
-	#bubbleDown(start) {
-		let index = start;
-		while (true) {
-			const left = index * 2 + 1;
-			const right = left + 1;
-			let next = index;
-			const item = this.#items[next];
-			const leftItem = this.#items[left];
-			const rightItem = this.#items[right];
-			if (leftItem && item && before(leftItem, item)) next = left;
-			const nextItem = this.#items[next];
-			if (rightItem && nextItem && before(rightItem, nextItem)) next = right;
-			if (next === index) return;
-			this.#swap(index, next);
-			index = next;
-		}
-	}
-};
-// packages/core/dist/analyze/crawler/crawl-caveats.js
-function crawlCaveats(input) {
-	const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
-	if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
-	if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
-	if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
-	const renderingFallbackUrls = input.renderingFallbackUrls ?? 0;
-	if (renderingFallbackUrls > 0) caveats.push(`JavaScript rendering could not run for ${renderingFallbackUrls} page${renderingFallbackUrls === 1 ? "" : "s"}, so ${renderingFallbackUrls === 1 ? "it was" : "they were"} audited from raw HTML. Client-rendered content, metadata and links may be missing. Install Chromium or set SEO_BROWSER_EXECUTABLE_PATH, then rerun.`);
-	const queryVariantSkippedUrls = input.queryVariantSkippedUrls ?? 0;
-	if (queryVariantSkippedUrls > 0) caveats.push(`Left ${queryVariantSkippedUrls} query-string variant${queryVariantSkippedUrls === 1 ? "" : "s"} unqueued after 5 variants of the same path, so tracking parameters do not fill the crawl.`);
-	for (const issue of input.reachIssues ?? []) if (issue.detail) caveats.push(issue.detail);
-	return caveats;
-}
-/**
-* A site crawl that found no internal links on the start page and no sitemap
-* URLs checked one page, so its other findings cannot stand for the site.
-*/
-function crawlReachIssues(input) {
-	const [page] = input.pages;
-	if (input.crawledPages !== 1 || input.pages.length !== 1 || !page || page.outgoingInternalCount > 0 || input.sitemapUrlsReturned > 0) return [];
-	const rule = explainRule("crawl_single_page_reach");
-	if (!rule) throw new Error("Missing rule guidance for crawl_single_page_reach");
-	return [{
-		ruleId: "crawl_single_page_reach",
-		title: rule.title,
-		category: rule.category,
-		severity: rule.defaultSeverity,
-		url: page.url,
-		detail: page.usedJs ? "The rendered start page had 0 internal links and no sitemap returned URLs, so the crawl checked only this page. List the pages to check with --mode list." : "The start page had 0 internal links in its HTML and no sitemap returned URLs, so the crawl checked only this page. It probably builds its links with JavaScript. Rerun with --rendering on, or list the pages with --mode list.",
-		evidence: {
-			startUrl: input.startUrl,
-			crawledPages: 1,
-			outgoingInternalCount: 0,
-			sitemapUrlsReturned: 0,
-			rendered: Boolean(page.usedJs)
-		}
-	}];
-}
 // packages/core/dist/analyze/crawler/crawl-control.js
 const CRAWL_CANCELLED = Symbol("crawl-cancelled");
 function abortController(input) {
@@ -25070,6 +25112,96 @@ function crawlRssLimitBytes(totalMemoryBytes) {
 function crawlMemoryPressure(input) {
 	return input.memoryUsage.heapUsed + input.memoryUsage.external >= crawlMemoryLimitBytes(input.totalMemoryBytes) || input.memoryUsage.rss >= crawlRssLimitBytes(input.totalMemoryBytes);
 }
+// packages/core/dist/analyze/crawler/crawl-rate-limit.js
+const RETRIES_PER_URL = 3;
+const BASE_BACKOFF_MS = 2e3;
+const MAX_BACKOFF_MS = 3e4;
+const COOLDOWN_BUDGET_MS = 18e4;
+const LIMITED_INTERVAL_MS = 1e3;
+const LIMITED_MAX_INTERVAL_MS = 4e3;
+function rateLimitEvidence(result, status, nowMs) {
+	if (status !== 429 && status !== 503) return void 0;
+	const header = retryAfterMs(result.page?.responseHeaders?.["retry-after"]);
+	const cooldownUntil = Date.parse(result.page?.fetchDiagnostics?.backpressure?.cooldownUntil ?? "");
+	const fromCooldown = Number.isFinite(cooldownUntil) ? Math.max(0, cooldownUntil - nowMs) : void 0;
+	const wait = header ?? fromCooldown;
+	if (status === 503 && wait === void 0) return void 0;
+	return wait === void 0 ? {} : { retryAfterMs: Math.min(wait, MAX_BACKOFF_MS) };
+}
+/**
+* Rate-limit state for one crawl. After the first 429 the crawl drops to one
+* request at a time with a per-host interval and never speeds back up.
+*/
+var CrawlRateLimit = class {
+	rate;
+	responses = 0;
+	skippedUrls = 0;
+	pendingMs = 0;
+	spentMs = 0;
+	#level = 0;
+	#budgetSpent = false;
+	#retries = /* @__PURE__ */ new Map();
+	constructor(rate) {
+		this.rate = rate;
+	}
+	get limited() {
+		return this.responses > 0;
+	}
+	get intervalMs() {
+		return this.rate.intervalMs ?? LIMITED_INTERVAL_MS;
+	}
+	/** The cool-down before probes that run after the crawl. */
+	get settleMs() {
+		return Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** this.#level);
+	}
+	/**
+	* `retry` requeues the URL after `pendingMs`. `skip` gives it up, and
+	* `stop` gives up the rest of the queue too once the budget is spent.
+	*/
+	observe(url, result, status, nowMs) {
+		const evidence = rateLimitEvidence(result, status, nowMs);
+		if (!evidence) return void 0;
+		this.responses += 1;
+		if (this.pendingMs === 0) this.#level += 1;
+		const step = 2 ** Math.max(0, this.#level - 1);
+		const waitMs = evidence.retryAfterMs ?? Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * step);
+		this.rate = {
+			...this.rate,
+			concurrency: 1,
+			intervalCap: 1,
+			intervalMs: Math.min(LIMITED_MAX_INTERVAL_MS, Math.max(this.rate.intervalMs ?? 0, LIMITED_INTERVAL_MS * step))
+		};
+		const nextMs = Math.max(this.pendingMs, waitMs);
+		if (this.spentMs + nextMs > COOLDOWN_BUDGET_MS) this.#budgetSpent = true;
+		const retries = this.#retries.get(url) ?? 0;
+		if (!this.#budgetSpent && retries < RETRIES_PER_URL) {
+			this.#retries.set(url, retries + 1);
+			this.pendingMs = nextMs;
+			return "retry";
+		}
+		this.skippedUrls += 1;
+		return this.#budgetSpent ? "stop" : "skip";
+	}
+	async settle(sleep, signal) {
+		if (this.pendingMs <= 0) return;
+		await sleep(this.pendingMs, signal);
+		this.spentMs += this.pendingMs;
+		this.pendingMs = 0;
+	}
+	caveatInput(config, readPages) {
+		if (this.skippedUrls === 0) return void 0;
+		return {
+			readPages,
+			skippedUrls: this.skippedUrls,
+			rerunCommand: rateLimitRerunCommand(config, this.intervalMs)
+		};
+	}
+	warning() {
+		if (!this.limited) return void 0;
+		const plural = (count, noun) => `${count} ${noun}${count === 1 ? "" : "s"}`;
+		return `The origin rate limited ${plural(this.responses, "request")}. The crawl slowed to one request at a time, at least ${Math.round(this.intervalMs / 1e3)}s apart, and waited ${Math.round(this.spentMs / 1e3)}s in cool-downs${this.skippedUrls > 0 ? `, then left ${plural(this.skippedUrls, "URL")} unread` : " before reading every queued URL"}.`;
+	}
+};
 // packages/core/dist/analyze/crawler/external-link-checks.js
 const EXTERNAL_LINK_SELECTION_LIMIT = 200;
 const EXTERNAL_LINK_STATES = [
@@ -25330,21 +25462,32 @@ function probeObservation(url, result) {
 		error: request.outcome === "failure" ? request.error : `Probe skipped: ${request.reason}`
 	};
 }
+function isConclusive(probe) {
+	return probe.status !== void 0 && !probe.accessBlocked && probe.status !== 429 && probe.status !== 503;
+}
 async function collectCrawlSiteChecks(input) {
+	const probe = async (url) => probeObservation(url, await input.fetchStatusPage(url, {
+		js: "off",
+		refresh: true,
+		timeoutMs: input.timeoutMs,
+		signal: input.signal,
+		writeCache: false,
+		respectRobots: input.respectRobots,
+		...input.rate ? { rate: input.rate } : {}
+	}));
 	const probes = [];
 	for (const url of probeUrls(input.startUrl)) {
 		if (input.signal?.aborted) break;
-		const result = await input.fetchStatusPage(url, {
-			js: "off",
-			refresh: true,
-			timeoutMs: input.timeoutMs,
-			signal: input.signal,
-			writeCache: false,
-			respectRobots: input.respectRobots
-		});
-		probes.push(probeObservation(url, result));
+		probes.push(await probe(url));
 	}
-	const conclusive = probes.filter((probe) => probe.status !== void 0 && !probe.accessBlocked);
+	if (input.sleep && !input.signal?.aborted && probes.some((item) => !isConclusive(item))) {
+		await input.sleep(input.retryDelayMs ?? 2e3, input.signal);
+		for (const [index, item] of probes.entries()) {
+			if (input.signal?.aborted) break;
+			if (!isConclusive(item)) probes[index] = await probe(item.url);
+		}
+	}
+	const conclusive = probes.filter(isConclusive);
 	const successful = conclusive.filter((probe) => (probe.status ?? 0) >= 200 && (probe.status ?? 0) < 400);
 	const correctMissing = conclusive.filter((probe) => probe.status === 404 || probe.status === 410);
 	const status = successful.length > 0 ? "warning" : correctMissing.length === PROBE_COUNT ? "pass" : "unknown";
@@ -25352,7 +25495,7 @@ async function collectCrawlSiteChecks(input) {
 		status,
 		probes,
 		probeLimit: PROBE_COUNT,
-		complete: probes.length === PROBE_COUNT
+		complete: conclusive.length === PROBE_COUNT
 	} };
 	if (status !== "warning") return {
 		checks,
@@ -27800,6 +27943,18 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
 	}[connection.provider](connection);
 }
 // packages/core/dist/analyze/crawler/site-crawl-dependencies.js
+function sleep$2(ms, signal) {
+	return new Promise((resolve) => {
+		if (signal?.aborted) return resolve();
+		const done = () => {
+			clearTimeout(timer);
+			signal?.removeEventListener("abort", done);
+			resolve();
+		};
+		const timer = setTimeout(done, ms);
+		signal?.addEventListener("abort", done, { once: true });
+	});
+}
 function resolveCrawlSiteDependencies(dependencies = {}) {
 	const hasInjectedPerPageSearchProvider = Boolean(dependencies.queryPageMetrics || dependencies.queryPageTopQuery);
 	const hasInjectedLegacyBulkSearchProvider = Boolean(dependencies.queryPagesMetrics || dependencies.queryPagesTopQueries);
@@ -27820,6 +27975,7 @@ function resolveCrawlSiteDependencies(dependencies = {}) {
 		fetchLandingPageValues: dependencies.fetchLandingPageValues ?? fetchLandingPageValues,
 		landingValueForUrl: dependencies.landingValueForUrl ?? landingValueForUrl,
 		now: dependencies.now ?? (() => /* @__PURE__ */ new Date()),
+		sleep: dependencies.sleep ?? sleep$2,
 		memoryUsage: dependencies.memoryUsage ?? (() => process.memoryUsage()),
 		totalMemory: dependencies.totalMemory ?? totalmem
 	};
@@ -28038,6 +28194,7 @@ async function crawlSite(input, dependencies) {
 	const queryVariantsByPath = /* @__PURE__ */ new Map();
 	const queryVariantSkipped = /* @__PURE__ */ new Set();
 	let renderingFallbackUrls = 0;
+	const rateLimit = new CrawlRateLimit(config.fetchRate);
 	let failedUrls = 0;
 	let observedInternalLinks = 0;
 	let memoryLimited = false;
@@ -28122,6 +28279,15 @@ async function crawlSite(input, dependencies) {
 				});
 				return;
 			}
+			if (isCdnUtilityUrl(normalized)) {
+				recordSkip("cdn-utility-url");
+				emitStatus("url_skipped", {
+					url: normalized,
+					depth,
+					reason: "cdn_utility_url"
+				});
+				return;
+			}
 			if (isLikelyAsset(normalized)) {
 				recordSkip("asset-url");
 				emitStatus("url_skipped", {
@@ -28226,7 +28392,7 @@ async function crawlSite(input, dependencies) {
 					js: config.js,
 					refresh: config.refresh,
 					timeoutMs: config.timeoutMs,
-					rate: config.fetchRate,
+					rate: rateLimit.rate,
 					signal,
 					writeCache: false,
 					respectRobots: config.respectRobots,
@@ -28257,7 +28423,11 @@ async function crawlSite(input, dependencies) {
 			});
 		};
 		while (!isCancelled() && (queue.size || inFlight.size) && (!memoryLimited || inFlight.size > 0) && pages.length < config.maxPages) {
-			while (!isCancelled() && !memoryLimited && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
+			if (rateLimit.pendingMs > 0 && inFlight.size === 0) {
+				await rateLimit.settle(deps.sleep, signal);
+				if (isCancelled()) break;
+			}
+			while (!isCancelled() && !memoryLimited && rateLimit.pendingMs === 0 && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
 				const item = nextQueueItem();
 				if (!item) break;
 				startTask(item);
@@ -28277,11 +28447,46 @@ async function crawlSite(input, dependencies) {
 				error: result.warning ?? "Crawler returned no page snapshot.",
 				extraction: "not-applicable"
 			});
+			const limited = rateLimit.observe(task.url, result, request.outcome === "response" ? request.status : void 0, deps.now().getTime());
+			if (limited) {
+				activeConcurrency = 1;
+				healthyProbeStreak = 0;
+			}
+			if (limited === "retry") {
+				visited.delete(task.url);
+				if (!queue.has(task.url)) queue.push({
+					url: task.url,
+					depth: task.depth
+				});
+				emitStatus("url_queued", {
+					url: task.url,
+					depth: task.depth,
+					reason: "rate_limited_retry",
+					message: `The origin rate limited ${task.url}. Waiting ${Math.ceil(rateLimit.pendingMs / 1e3)}s, then retrying one request at a time.`
+				});
+				continue;
+			}
+			if (limited) {
+				requests.push(request);
+				recordSkip("origin-backpressure");
+				emitStatus("page_skipped", {
+					url: task.url,
+					depth: task.depth,
+					reason: "origin-backpressure"
+				});
+				if (limited === "stop") {
+					let left = 0;
+					while (queue.take()) left += 1;
+					recordSkips("origin-backpressure", left);
+					rateLimit.skippedUrls += left;
+				}
+				continue;
+			}
 			requests.push(request);
 			if (healthStrategy) {
 				if (request.outcome === "response" && request.status < 400 && !request.accessBlock) {
 					healthyProbeStreak += 1;
-					activeConcurrency = Math.min(config.concurrency, 1 + Math.floor(healthyProbeStreak / 20));
+					activeConcurrency = Math.min(rateLimit.limited ? 1 : config.concurrency, 1 + Math.floor(healthyProbeStreak / 20));
 				} else {
 					healthyProbeStreak = 0;
 					activeConcurrency = 1;
@@ -28354,7 +28559,7 @@ async function crawlSite(input, dependencies) {
 			const storeDocument = (urls) => {
 				const sourceRequest = normalizeUrl$1(task.url, config.url) ?? task.url;
 				const directlyRequested = sourceRequest === finalUrl;
-				const normalizedLinks = urls.map((url) => normalizeUrl$1(url, finalUrl)).filter((url) => Boolean(url));
+				const normalizedLinks = urls.map((url) => normalizeUrl$1(url, finalUrl)).filter((url) => Boolean(url) && !isCdnUtilityUrl(url ?? ""));
 				const syncLinkEvidence = (target, links) => {
 					const sortedLinks = [...links].sort((left, right) => left < right ? -1 : left > right ? 1 : 0);
 					target.outgoingInternalCount = sortedLinks.length;
@@ -28433,7 +28638,7 @@ async function crawlSite(input, dependencies) {
 			if (followLinks && task.depth < config.maxDepth) {
 				if (memoryLimited) for (const value of result.urls) {
 					const normalized = normalizeUrl$1(value, finalUrl);
-					if (normalized && sameOrigin(normalized, origin) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
+					if (normalized && sameOrigin(normalized, origin) && !isCdnUtilityUrl(normalized) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
 						discovered.add(normalized);
 						memoryDeferredUrls.add(normalized);
 					}
@@ -28458,6 +28663,8 @@ async function crawlSite(input, dependencies) {
 		cancelled = isCancelled();
 		const robotsDisallowed = skipReasonCounts["robots-disallowed"] ?? 0;
 		if (robotsDisallowed > 0) warnings.push(`${robotsDisallowed} URL${robotsDisallowed === 1 ? "" : "s"} disallowed ${SEO_CRAWLER_USER_AGENT} in robots.txt. If the audit should reach them, allow the SEO-Skill token on only those public paths.`);
+		const rateLimitWarning = rateLimit.warning();
+		if (rateLimitWarning) warnings.push(rateLimitWarning);
 		if (cancelled) {
 			warnings.push("Crawl cancelled; returning a partial report.");
 			emitStatus("cancelled", {
@@ -28469,7 +28676,7 @@ async function crawlSite(input, dependencies) {
 		const sitemapEvidencePartial = sitemapDiscovery?.dataStatus === "partial";
 		const accessEvidencePartial = requests.some((request) => request.outcome === "response" && request.accessBlock);
 		const requiredSitemapUnavailable = config.mode === "sitemap" && sitemapDiscovery?.dataStatus === "unavailable";
-		const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
+		const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || rateLimit.skippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
 		const failed = !cancelled && pages.length === 0 && (requiredSitemapUnavailable || requests.length > 0 && requests.every((request) => request.outcome === "failure"));
 		const requestEvidenceStatus = cancelled && inFlight.size > 0 ? "partial" : "available";
 		const dataSources = await crawlDataSources({
@@ -28516,7 +28723,10 @@ async function crawlSite(input, dependencies) {
 			timeoutMs: config.timeoutMs,
 			fetchStatusPage: deps.fetchStatusPage,
 			respectRobots: config.respectRobots,
-			signal
+			signal,
+			rate: rateLimit.rate,
+			sleep: deps.sleep,
+			retryDelayMs: rateLimit.settleMs
 		}) : void 0;
 		for (const page of pages) {
 			if (!page.geo || !llmsTxt) continue;
@@ -28585,6 +28795,8 @@ async function crawlSite(input, dependencies) {
 				queueSafetySkippedUrls,
 				queueSafetyInventoryCapped,
 				originBackpressureSkippedUrls,
+				rateLimit: rateLimit.caveatInput(config, pages.length),
+				sitemapLimit: sitemapLimitCaveatInput(config, sitemapDiscovery, input.maxPages === void 0),
 				memoryPressureSkippedUrls,
 				queryVariantSkippedUrls: queryVariantSkipped.size,
 				renderingFallbackUrls,
@@ -42279,6 +42491,31 @@ async function dataForSeoRows(keys, input, spent) {
 }
 const GAP_DOMAINS = 1e3;
 const GAP_COMPETITORS = 5;
+const SHORTENERS = [
+	"bit.ly",
+	"tinyurl.com",
+	"t.co",
+	"ow.ly",
+	"buff.ly",
+	"is.gd",
+	"cutt.ly",
+	"rebrand.ly",
+	"shorturl.at",
+	"rb.gy",
+	"lnkd.in"
+];
+const FREE_APP_HOSTS = [
+	"web.app",
+	"firebaseapp.com",
+	"pages.dev"
+];
+const LINK_LIST_NAME = /(subdomain|whois|login|signin|coupon|promo|voucher|discount|sitelike|similarsite|sitesimilar|siteprice|sitevalue|siteworth|websiteworth|hypestat|statshow|websiteoutlook|siteinfo|domaininfo|domainstat|domaintools|ipaddress|dnslookup|linkdirectory|webdirectory|alternativeto-)/;
+function isLinkList(domain) {
+	const within = (name) => domain === name || domain.endsWith(`.${name}`);
+	if (SHORTENERS.some(within)) return true;
+	if (FREE_APP_HOSTS.some((name) => domain.endsWith(`.${name}`))) return true;
+	return LINK_LIST_NAME.test(domain.replace(/[-.]/g, ""));
+}
 async function linkGap(provider, headers, input) {
 	const target = bareTarget(input.target).replace(/^www\./, "");
 	const competitors = [...new Set((input.competitors ?? []).map((site) => bareTarget(site).replace(/^www\./, "")))].filter((site) => site && site !== target);
@@ -42319,8 +42556,10 @@ async function linkGap(provider, headers, input) {
 			found.set(domain, entry);
 		}
 	});
+	const gap = [...found.entries()].filter(([, entry]) => entry.sites.length >= 2 && (floor === null || entry.value >= floor));
+	const linkLists = gap.map(([domain]) => domain).filter(isLinkList).sort((a, b) => a.localeCompare(b));
 	return {
-		rows: [...found.entries()].filter(([, entry]) => entry.sites.length >= 2 && (floor === null || entry.value >= floor)).sort(([a, left], [b, right]) => right.value - left.value || right.sites.length - left.sites.length || a.localeCompare(b)).map(([domain, entry]) => ({
+		rows: gap.filter(([domain]) => !isLinkList(domain)).sort(([a, left], [b, right]) => right.value - left.value || right.sites.length - left.sites.length || a.localeCompare(b)).map(([domain, entry]) => ({
 			domain,
 			[score]: entry.value,
 			competitors_linking: entry.sites.length,
@@ -42333,7 +42572,8 @@ async function linkGap(provider, headers, input) {
 		...floor !== null ? { floor: {
 			field: score,
 			value: floor
-		} } : {}
+		} } : {},
+		...linkLists.length ? { leftOut: linkLists } : {}
 	};
 }
 function isPage(target) {
@@ -42700,6 +42940,7 @@ async function runAiAnswers(input, headers, deps, more) {
 		question,
 		impressions,
 		task: null,
+		ref: null,
 		cost_usd: 0,
 		...empty
 	});
@@ -42711,7 +42952,8 @@ async function runAiAnswers(input, headers, deps, more) {
 			...blank("", null),
 			engine: engine.name,
 			result: "pending",
-			task: value
+			task: value,
+			ref: value
 		};
 		rows.push(row);
 		waiting.push({
@@ -42764,6 +43006,7 @@ async function runAiAnswers(input, headers, deps, more) {
 				}
 				row.result = "pending";
 				row.task = `${engine.name}:${task.id}`;
+				row.ref = row.task;
 				waiting.push({
 					row,
 					engine,
@@ -42804,6 +43047,7 @@ async function runAiAnswers(input, headers, deps, more) {
 	const cost = usd(rows.reduce((sum, row) => sum + row.cost_usd, 0));
 	const count = (result) => rows.filter((row) => row.result === result).length;
 	const left = rows.filter((row) => row.task).map((row) => row.task);
+	const refs = rows.flatMap((row) => row.ref ? [row.ref] : []);
 	const cites = /* @__PURE__ */ new Map();
 	for (const row of rows) for (const host of row.sources) cites.set(host, (cites.get(host) ?? 0) + 1);
 	const topCited = [...cites].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10);
@@ -42837,7 +43081,7 @@ async function runAiAnswers(input, headers, deps, more) {
 			limit: 50,
 			detail: input.detail,
 			notes,
-			more
+			more: (offset) => more(offset, refs)
 		}),
 		data: {
 			provider: "dataforseo",
@@ -42851,6 +43095,7 @@ async function runAiAnswers(input, headers, deps, more) {
 			questions: [...new Set(rows.map((row) => row.question))],
 			costUsd: cost,
 			pendingTasks: left,
+			tasks: refs,
 			citedSites: topCited.map(([host, answers]) => ({
 				host,
 				answers
@@ -42965,7 +43210,7 @@ function matchGa4WebStreams(site, candidates) {
 }
 // packages/core/dist/commands/analytics-site.js
 const STREAM_CHECKS = 30;
-const AT_ONCE = 5;
+const AT_ONCE$1 = 5;
 const TIED = 3;
 const hostOf$1 = (site) => {
 	const value = site.replace(/^sc-domain:/, "");
@@ -43030,8 +43275,8 @@ async function lookProperty(access, site) {
 	}
 	const order = ranked(site, candidates).slice(0, STREAM_CHECKS);
 	let failures = 0;
-	for (let start = 0; start < order.length; start += AT_ONCE) {
-		const batch = order.slice(start, start + AT_ONCE);
+	for (let start = 0; start < order.length; start += AT_ONCE$1) {
+		const batch = order.slice(start, start + AT_ONCE$1);
 		const streams = await Promise.all(batch.map((candidate) => listAnalyticsDataStreams(candidate.google, candidate.property, { signIn: access.signIn }).catch(() => {
 			failures += 1;
 			return [];
@@ -44164,7 +44409,8 @@ async function runClickyRows(input, keys, options) {
 // packages/core/dist/commands/domain-rating.js
 const DR_ATTRIBUTION = "domain_rating: Domain Rating by Ahrefs (https://ahrefs.com/).";
 const ENDPOINT = "https://api.ahrefs.com/v3/public/domain-rating-free";
-const MAX_DOMAINS = 25;
+const MAX_DOMAINS = 50;
+const AT_ONCE = 25;
 const TTL_SECONDS = 86400;
 const DOMAIN_FIELDS = [
 	"domain",
@@ -44195,23 +44441,26 @@ async function lookup(domain, key) {
 	}
 	return body.domain_rating?.domain_rating ?? null;
 }
-async function domainRatings(domains, key) {
+async function domainRatings(domains, key, max = MAX_DOMAINS) {
 	const found = /* @__PURE__ */ new Map();
 	if (!key) return found;
-	const unique = [...new Set(domains.map(bareDomain).filter(Boolean))].slice(0, MAX_DOMAINS);
-	const values = await Promise.all(unique.map((domain) => lookup(domain, key).catch(() => null)));
-	unique.forEach((domain, index) => {
-		found.set(domain, values[index] ?? null);
-	});
+	const unique = [...new Set(domains.map(bareDomain).filter(Boolean))].slice(0, max);
+	for (let start = 0; start < unique.length; start += AT_ONCE) {
+		const batch = unique.slice(start, start + AT_ONCE);
+		const values = await Promise.all(batch.map((domain) => lookup(domain, key).catch(() => null)));
+		batch.forEach((domain, index) => {
+			found.set(domain, values[index] ?? null);
+		});
+	}
 	return found;
 }
-async function withDomainRating(rows, key) {
+async function withDomainRating(rows, key, start = 0) {
 	const field = DOMAIN_FIELDS.find((name) => rows.some((row) => typeof row[name] === "string"));
 	if (!key || !field || rows.some((row) => "domain_rating" in row)) return {
 		rows,
 		added: false
 	};
-	const ratings = await domainRatings(rows.map((row) => String(row[field] ?? "")), key);
+	const ratings = await domainRatings(rows.slice(start).map((row) => String(row[field] ?? "")), key);
 	return {
 		rows: rows.map((row) => {
 			const next = {};
@@ -46305,7 +46554,7 @@ async function runResearch(input, keys, options) {
 	const offset = input.offset ?? 0;
 	const language = input.provider === "dataforseo" ? dataForSeoLanguage(country) : null;
 	if (input.action === "ai_answers") throw new CommandError("INVALID_INPUT", "ai_answers runs on its own.");
-	const { rows: found, costUsd, partial, missed, noData, floor } = await research(input.provider, keys, {
+	const { rows: found, costUsd, partial, missed, noData, floor, leftOut } = await research(input.provider, keys, {
 		action: input.action,
 		keywords: input.keywords ?? [],
 		target,
@@ -46319,7 +46568,8 @@ async function runResearch(input, keys, options) {
 	};
 	const unresolved = rows.filter((row) => row.url === null && row.url_hint !== void 0).length;
 	const drKey = keys.get("x-ahrefs-key") ?? options.drKey;
-	const rated = await withDomainRating(rows, drKey);
+	const byDr = input.action === "link_gap" && input.provider === "dataforseo" && drKey;
+	const rated = byDr ? await sortedByDomainRating(rows, drKey) : await withDomainRating(rows, drKey, offset);
 	const targetDr = target ? (await domainRatings([target], drKey)).get(bareDomain(target)) : void 0;
 	const hasTargetDr = targetDr !== void 0 && targetDr !== null;
 	return {
@@ -46335,6 +46585,8 @@ async function runResearch(input, keys, options) {
 				...partial ? ["Partial: DataForSEO could not fetch some pages of results, so rows may be missing."] : [],
 				...missed.length ? [`Partial: ${input.provider} failed for ${missed.join(", ")}, so those keywords have no rows. Ask for them again.`] : [],
 				...noData.length ? [noDataNote(input, noData)] : [],
+				...byDr ? [rows.length > GAP_RATED ? `Sorted by Domain Rating for the top ${GAP_RATED} rows by DataForSEO rank; the ${rows.length - GAP_RATED} rows after them follow by rank, with no Domain Rating.` : "Sorted by Domain Rating, then DataForSEO rank."] : [],
+				...leftOut?.length ? [`Left out ${leftOut.length} link lists, URL shorteners and directories that link to most sites (a guess from the host name): ${leftOut.join(", ")}.`] : [],
 				...floor ? [`Cut at ${floor.field} ${floor.value}: at least one site has more than 1,000 referring domains, and only its top 1,000 were read, so domains below ${floor.field} ${floor.value} are left out rather than guessed.`] : [],
 				...unresolved ? [unresolvedNote(input, unresolved, rows.length)] : [],
 				...note ? [note] : [],
@@ -46358,7 +46610,8 @@ async function runResearch(input, keys, options) {
 			noDataKeywords: noData,
 			...input.action === "link_gap" ? {
 				competitors: input.competitors ?? [],
-				cutAt: floor ?? null
+				cutAt: floor ?? null,
+				leftOut: leftOut ?? []
 			} : {},
 			total: rated.rows.length,
 			offset,
@@ -46374,6 +46627,24 @@ function noDataNote(input, keywords) {
 function unresolvedNote(input, count, total) {
 	return `${count} of ${total} results have no exact page: ${input.provider} returned Google redirect links for them. Their domain and url_hint come from Google's breadcrumb, which is shortened.${input.provider === "dataforseo" ? " For exact pages, use provider ahrefs." : ""}`;
 }
+const GAP_RATED = 200;
+async function sortedByDomainRating(rows, key) {
+	const top = rows.slice(0, GAP_RATED);
+	const ratings = await domainRatings(top.map((row) => String(row.domain ?? "")), key, GAP_RATED);
+	const withDr = (row, rated) => {
+		const { domain = null, ...rest } = row;
+		return {
+			domain,
+			domain_rating: rated ? ratings.get(bareDomain(String(domain ?? ""))) ?? null : null,
+			...rest
+		};
+	};
+	const dr = (row) => Number(row.domain_rating ?? -1);
+	return {
+		rows: [...top.map((row) => withDr(row, true)).sort((a, b) => dr(b) - dr(a) || Number(b.rank ?? 0) - Number(a.rank ?? 0) || String(a.domain).localeCompare(String(b.domain))), ...rows.slice(GAP_RATED).map((row) => withDr(row, false))],
+		added: true
+	};
+}
 // packages/core/dist/commands/sitemap-urls.js
 const MAX_FILES = 20;
 const MAX_URLS = 5e4;
@@ -47264,7 +47535,7 @@ const SHARED_TOOLS = [
 					offset: input.offset,
 					questions: input.keywords,
 					tasks: input.tasks
-				}, await surface.keys(), surface.deps, more);
+				}, await surface.keys(), surface.deps, (offset, tasks) => `seo_research with the same provider, action and target, tasks ${JSON.stringify(tasks)}, detail "rows", offset ${offset}. Passing the tasks reads these answers back at no cost; without them every question is asked and paid for again`);
 			}
 			return runResearch(input, await surface.keys(), {
 				drKey: surface.drKey,