seo 0.17.1 → 0.17.2

package/dist/index.js

@@ -1,5 +1,5 @@
 import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
-import { n as version, t as load } from "./esm-G8B5TPzB.js";
+import { n as version, t as load } from "./esm-BR8_7rhM.js";
 import { SEO_PROVIDER_API_VERSION } from "./provider-sdk.js";
 import crypto$1, { createHash, randomUUID, sign } from "node:crypto";
 import { z } from "zod";
@@ -453,13 +453,20 @@ const RULE_DEFINITIONS = [
 		title: "Large HTML response",
 		category: "performance",
 		defaultSeverity: "low",
-		whyItMatters: "Large HTML takes longer to download and parse before the browser can even start the rest of the page.",
-		howToFix: "Trim duplicated markup, remove heavy inline data, split non-critical content, and move large scripts or styles out of the HTML response.",
-		impactIfIgnored: "Pages can feel slow on mobile connections and crawlers spend more time processing each URL.",
-		howToVerify: "Re-run the crawl and confirm sizeBytes is below the large HTML threshold for normal pages.",
+		whyItMatters: "Googlebot reads only the first 2MB of uncompressed HTML for Google Search and indexes the page as if the file ended there. When the h1 or main element, or most of the visible text, sits past that cut, Google may index the page without its main content. Large HTML is also slow to download and parse.",
+		howToFix: "Move the main content earlier and trim what comes before it: large inline CSS, inline SVG, JSON data and duplicated markup. Load non-critical styles and scripts as separate files.",
+		impactIfIgnored: "Content past the 2MB cut is not indexed, and every page of the template carries the same weight on slow connections.",
+		howToVerify: "Re-run the crawl and confirm sizeBytes is under 2,097,152, or that mainContentStartBytes is well before it and few visible words sit after it. The word split is a heuristic from the raw HTML.",
 		agentHints: {
-			evidenceFields: ["page.sizeBytes", "issue.evidence.thresholdBytes"],
-			suggestedCommands: ["seo crawl <url> --json"]
+			evidenceFields: [
+				"page.sizeBytes",
+				"issue.evidence.thresholdBytes",
+				"issue.evidence.mainContentStartBytes",
+				"issue.evidence.bytesAfterLimit",
+				"issue.evidence.visibleWordsAfterLimit",
+				"issue.evidence.indexingRisk"
+			],
+			suggestedCommands: ["seo crawl <url> --mode page --json"]
 		}
 	},
 	{
@@ -1097,17 +1104,17 @@ function isAgentFindings(value) {
 	const counts = record(candidate?.counts);
 	return candidate?.schemaVersion === 1 && candidate.scope === "returned-report" && typeof counts?.total === "number" && typeof counts.returned === "number" && Array.isArray(candidate.items) && candidate.items.every((value) => {
 		const item = record(value);
-		if (!text(item?.id) || item?.status !== "open" || !text(item.title) || !text(item.sourcePath) || !Array.isArray(item.allowedOutcomes)) return false;
-		if (item.type === "fix") return Boolean(text(record(item.fix)?.instruction));
+		if (!text$1(item?.id) || item?.status !== "open" || !text$1(item.title) || !text$1(item.sourcePath) || !Array.isArray(item.allowedOutcomes)) return false;
+		if (item.type === "fix") return Boolean(text$1(record(item.fix)?.instruction));
 		if (item.type === "review") {
 			const review = record(item.review);
 			if (!review) return false;
-			return Boolean(text(review.question) && text(review.changeOnlyIf) && text(review.ifConfirmed) && text(review.ifNotNeeded) && Array.isArray(review.doNot));
+			return Boolean(text$1(review.question) && text$1(review.changeOnlyIf) && text$1(review.ifConfirmed) && text$1(review.ifNotNeeded) && Array.isArray(review.doNot));
 		}
 		return false;
 	});
 }
-function text(value) {
+function text$1(value) {
 	return typeof value === "string" && value.trim() ? value.trim() : void 0;
 }
 function stableId(value) {
@@ -1130,7 +1137,7 @@ function findingId(item, path, action) {
 		"actionId",
 		"id"
 	]) {
-		const id = text(item[key]);
+		const id = text$1(item[key]);
 		if (id) return id;
 	}
 	return `action-${stableId(`${path}|${action}`)}`;
@@ -1140,9 +1147,9 @@ function findingType(path, item) {
 }
 function findingVerification(item, fallback) {
 	const supplied = record(item.verification);
-	const expected = text(supplied?.expected ?? item.howToVerify ?? fallback);
+	const expected = text$1(supplied?.expected ?? item.howToVerify ?? fallback);
 	if (!expected) return void 0;
-	const command = text(supplied?.command);
+	const command = text$1(supplied?.command);
 	return {
 		...command ? { command } : {},
 		expected
@@ -1150,14 +1157,14 @@ function findingVerification(item, fallback) {
 }
 function stringArray(value) {
 	if (!Array.isArray(value)) return [];
-	return value.flatMap((item) => text(item) ? [text(item)] : []);
+	return value.flatMap((item) => text$1(item) ? [text$1(item)] : []);
 }
 function affectedUrls(item, affectedCount) {
 	const items = stringArray(item.sampleUrls);
 	const details = record(item.affectedUrlsReport);
 	const params = record(details?.params);
-	const reportId = details?.id === "affected-urls" ? text(params?.reportId) : void 0;
-	const ruleId = details?.id === "affected-urls" ? text(params?.ruleId) : void 0;
+	const reportId = details?.id === "affected-urls" ? text$1(params?.reportId) : void 0;
+	const ruleId = details?.id === "affected-urls" ? text$1(params?.ruleId) : void 0;
 	if (!items.length && !reportId) return void 0;
 	const total = affectedCount ?? items.length;
 	return {
@@ -1177,13 +1184,13 @@ function affectedUrls(item, affectedCount) {
 function reviewGuidance(item, title, action) {
 	const supplied = record(item.review);
 	const evidence = record(item.evidence);
-	const ruleId = text(item.ruleId ?? evidence?.ruleId);
+	const ruleId = text$1(item.ruleId ?? evidence?.ruleId);
 	const defined = (ruleId ? explainRule(ruleId) : void 0)?.review;
 	return {
-		question: text(supplied?.question) ?? defined?.question ?? `Does the retained evidence and confirmed intent show that "${title}" needs a change?`,
-		changeOnlyIf: text(supplied?.changeOnlyIf) ?? defined?.changeOnlyIf ?? "Make a change only when the retained evidence and required intent support it.",
-		ifConfirmed: text(supplied?.ifConfirmed) ?? action,
-		ifNotNeeded: text(supplied?.ifNotNeeded) ?? defined?.ifNotNeeded ?? "Record no change and cite the evidence or confirmed intent that makes a change unnecessary.",
+		question: text$1(supplied?.question) ?? defined?.question ?? `Does the retained evidence and confirmed intent show that "${title}" needs a change?`,
+		changeOnlyIf: text$1(supplied?.changeOnlyIf) ?? defined?.changeOnlyIf ?? "Make a change only when the retained evidence and required intent support it.",
+		ifConfirmed: text$1(supplied?.ifConfirmed) ?? action,
+		ifNotNeeded: text$1(supplied?.ifNotNeeded) ?? defined?.ifNotNeeded ?? "Record no change and cite the evidence or confirmed intent that makes a change unnecessary.",
 		doNot: stringArray(supplied?.doNot).length ? stringArray(supplied?.doNot) : [...defined?.doNot ?? []]
 	};
 }
@@ -1202,7 +1209,7 @@ function findingEvidence(item) {
 }
 function normalizeFinding(value, path, index, options) {
 	if (typeof value === "string") {
-		const action = text(value);
+		const action = text$1(value);
 		if (!action) return void 0;
 		const common = {
 			id: `action-${stableId(`${options.reportId ?? ""}|${path}|${action}`)}`,
@@ -1234,8 +1241,8 @@ function normalizeFinding(value, path, index, options) {
 	}
 	const item = record(value);
 	if (!item) return void 0;
-	const action = text(item.action) ?? text(item.howToFix) ?? (item.recommendation === "fix" || item.recommendation === "review" ? void 0 : text(item.recommendation)) ?? text(item.nextStep);
-	const title = text(item.title) ?? text(item.headline) ?? text(item.name) ?? action;
+	const action = text$1(item.action) ?? text$1(item.howToFix) ?? (item.recommendation === "fix" || item.recommendation === "review" ? void 0 : text$1(item.recommendation)) ?? text$1(item.nextStep);
+	const title = text$1(item.title) ?? text$1(item.headline) ?? text$1(item.name) ?? action;
 	if (!action || !title) return void 0;
 	const confidence = [
 		"high",
@@ -1253,7 +1260,7 @@ function normalizeFinding(value, path, index, options) {
 		status: "open",
 		title,
 		...confidence ? { confidence } : {},
-		...text(item.severity) ? { severity: text(item.severity) } : {},
+		...text$1(item.severity) ? { severity: text$1(item.severity) } : {},
 		...typeof affectedCount === "number" && Number.isFinite(affectedCount) ? { affectedCount } : {},
 		...urls ? { affectedUrls: urls } : {},
 		sourcePath: `${path}[${index}]`,
@@ -1315,7 +1322,7 @@ function inventory(value, path, reportId) {
 	const totalItems = typeof (source.totalPages ?? source.totalItems) === "number" ? Number(source.totalPages ?? source.totalItems) : source.rows.length;
 	const items = source.rows.map((value, index) => {
 		const row = record(value) ?? { value };
-		const identity = text(row.id) ?? text(row.url) ?? text(row.path) ?? `${path}[${index}]`;
+		const identity = text$1(row.id) ?? text$1(row.url) ?? text$1(row.path) ?? `${path}[${index}]`;
 		return {
 			...row,
 			id: `inventory-item-${stableId(`${reportId ?? ""}|${path}|${identity}`)}`,
@@ -1334,7 +1341,7 @@ function inventory(value, path, reportId) {
 		returnedItems: items.length,
 		complete: source.capped === false && items.length >= totalItems,
 		...source.criteria !== void 0 ? { criteria: source.criteria } : {},
-		...text(source.note) ? { note: text(source.note) } : {},
+		...text$1(source.note) ? { note: text$1(source.note) } : {},
 		...pageCount !== void 0 ? { pagination: {
 			page,
 			pageCount,
@@ -2802,7 +2809,7 @@ async function readableSites(google, signIn) {
 		permissionLevel: entry.permissionLevel ?? ""
 	})).sort((a, b) => a.siteUrl.localeCompare(b.siteUrl));
 }
-const hostOf$3 = (value) => {
+const hostOf$4 = (value) => {
 	try {
 		return new URL(value.includes("://") ? value : `https://${value.replace(/^sc-domain:/, "")}`).hostname.replace(/^www\./, "");
 	} catch {
@@ -2820,14 +2827,14 @@ async function runSites(input) {
 		};
 	})), input.bing ? listBingSites(input.bing).catch((error) => error instanceof Error ? error : new Error(String(error))) : Promise.resolve(null)]);
 	const bingSites = Array.isArray(bing) ? bing.filter((site) => site.verified) : [];
-	const bingByHost = new Map(bingSites.map((site) => [hostOf$3(site.url), site]));
+	const bingByHost = new Map(bingSites.map((site) => [hostOf$4(site.url), site]));
 	const properties = google.flatMap((account) => account.properties instanceof Error ? [] : account.properties.map((property) => ({
 		...property,
 		...named ? { account: account.email ?? "" } : {},
-		...Array.isArray(bing) ? { bing: bingByHost.get(hostOf$3(property.siteUrl))?.url ?? "" } : {}
+		...Array.isArray(bing) ? { bing: bingByHost.get(hostOf$4(property.siteUrl))?.url ?? "" } : {}
 	})));
-	const googleHosts = new Set(properties.map((property) => hostOf$3(property.siteUrl)));
-	const bingOnly = bingSites.filter((site) => !googleHosts.has(hostOf$3(site.url)));
+	const googleHosts = new Set(properties.map((property) => hostOf$4(property.siteUrl)));
+	const bingOnly = bingSites.filter((site) => !googleHosts.has(hostOf$4(site.url)));
 	const failed = google.filter((account) => account.properties instanceof Error);
 	const analytics = google.flatMap((account) => (account.analytics ?? []).flatMap((summary) => summary.propertySummaries.map((property) => ({
 		property: property.property.replace("properties/", ""),
@@ -11477,6 +11484,98 @@ function contentOptimizationFromPageOpportunities(sourceReport) {
 async function contentOptimizationReport(input) {
 	return contentOptimizationFromPageOpportunities(await pageOpportunitiesReport(input));
 }
+// packages/core/src/analyze/crawler/html-size.ts
+const GOOGLEBOT_HTML_LIMIT_BYTES = 2097152;
+const NEAR_LIMIT_SHARE = .9;
+const HIDDEN_BLOCKS = /<(script|style|svg|noscript|template)\b[\s\S]*?(?:<\/\1\s*>|$)|<!--[\s\S]*?(?:-->|$)/gi;
+function wordCount(text) {
+	return text.match(/[^\s&;]+/g)?.length ?? 0;
+}
+/** Where a page over Googlebot's limit puts its main content. */
+function googlebotCutEvidence(html) {
+	const bytes = Buffer.from(html, "utf8");
+	if (bytes.byteLength <= 2097152) return void 0;
+	const marker = /<(h1|main)[\s>/]/i.exec(html);
+	const text = html.replace(HIDDEN_BLOCKS, (match) => " ".repeat(match.length)).replace(/<[^>]*>?/g, (match) => " ".repeat(match.length));
+	const limitIndex = bytes.subarray(0, GOOGLEBOT_HTML_LIMIT_BYTES).toString("utf8").length;
+	const startBytes = marker ? Buffer.byteLength(html.slice(0, marker.index)) : void 0;
+	return {
+		limitBytes: GOOGLEBOT_HTML_LIMIT_BYTES,
+		bytesAfterLimit: bytes.byteLength - GOOGLEBOT_HTML_LIMIT_BYTES,
+		...marker && startBytes !== void 0 ? {
+			mainContentStartBytes: startBytes,
+			mainContentMarker: marker[1]?.toLowerCase()
+		} : {},
+		mainContentStart: startBytes === void 0 ? "not-found" : startBytes >= 2097152 ? "after-limit" : startBytes >= 2097152 * NEAR_LIMIT_SHARE ? "near-limit" : "before-limit",
+		visibleWordsBeforeLimit: wordCount(text.slice(0, limitIndex)),
+		visibleWordsAfterLimit: wordCount(text.slice(limitIndex))
+	};
+}
+const formatBytes = (value) => value.toLocaleString("en-US");
+function largeHtmlIssue(rule, page) {
+	const sizeBytes = page.sizeBytes ?? 0;
+	const cut = page.googlebotCut;
+	const htmlSource = page.usedJs ? "rendered" : "raw";
+	const base = {
+		...rule,
+		detail: `${Math.round(sizeBytes / 1024)} KB`,
+		evidence: {
+			sizeBytes,
+			thresholdBytes: GOOGLEBOT_HTML_LIMIT_BYTES,
+			htmlSource
+		}
+	};
+	if (!cut) return base;
+	const words = cut.visibleWordsBeforeLimit + cut.visibleWordsAfterLimit;
+	const wordsAfterShare = words ? Math.round(cut.visibleWordsAfterLimit / words * 100) / 100 : void 0;
+	const indexingRisk = htmlSource === "rendered" ? "unknown" : cut.mainContentStart === "after-limit" || cut.mainContentStart === "near-limit" || (wordsAfterShare ?? 0) > .5 ? "main-content-past-limit" : "none-found";
+	const start = cut.mainContentStartBytes === void 0 ? "no <h1> or <main> found" : `main content (<${cut.mainContentMarker}>) starts at byte ${formatBytes(cut.mainContentStartBytes)}, ${cut.mainContentStart === "after-limit" ? "after" : cut.mainContentStart === "near-limit" ? "just before" : "before"} the cut`;
+	const wordsNote = wordsAfterShare === void 0 ? "" : `, with about ${Math.round(wordsAfterShare * 100)}% of the visible words`;
+	return {
+		...base,
+		severity: indexingRisk === "main-content-past-limit" ? "high" : base.severity,
+		detail: `${Math.round(sizeBytes / 1024)} KB; ${start}; ${formatBytes(cut.bytesAfterLimit)} bytes sit past Googlebot's ${formatBytes(cut.limitBytes)}-byte cut${wordsNote}.${indexingRisk === "main-content-past-limit" ? " Googlebot stops reading at the cut and indexes only what came before it (the word split is a heuristic)." : ""}`,
+		evidence: {
+			...base.evidence,
+			googlebotLimitBytes: cut.limitBytes,
+			bytesAfterLimit: cut.bytesAfterLimit,
+			mainContentStartBytes: cut.mainContentStartBytes ?? null,
+			mainContentMarker: cut.mainContentMarker ?? null,
+			mainContentStart: cut.mainContentStart,
+			visibleWordsBeforeLimit: cut.visibleWordsBeforeLimit,
+			visibleWordsAfterLimit: cut.visibleWordsAfterLimit,
+			visibleWordsAfterLimitShare: wordsAfterShare ?? null,
+			indexingRisk
+		}
+	};
+}
+const RESPONSE_CAP_ERROR = /^Page response exceeds the (\d+)-byte response limit/;
+function responseCapIssue(request) {
+	if (request.outcome !== "failure") return void 0;
+	const capBytes = Number(RESPONSE_CAP_ERROR.exec(request.error)?.[1]);
+	if (!capBytes) return void 0;
+	const rule = explainRule("large_html");
+	if (!rule) throw new Error("Missing rule guidance for large_html");
+	return {
+		ruleId: rule.id,
+		title: rule.title,
+		category: rule.category,
+		severity: rule.defaultSeverity,
+		url: request.requestedUrl,
+		detail: `Over ${Math.round(capBytes / 1024)} KB; the crawler stopped reading at its ${formatBytes(capBytes)}-byte cap, so where the main content starts is unknown.`,
+		evidence: {
+			dataStatus: "partial",
+			sizeBytes: null,
+			sizeBytesAtLeast: capBytes,
+			thresholdBytes: GOOGLEBOT_HTML_LIMIT_BYTES,
+			googlebotLimitBytes: GOOGLEBOT_HTML_LIMIT_BYTES,
+			bytesAfterLimitAtLeast: capBytes - GOOGLEBOT_HTML_LIMIT_BYTES,
+			mainContentStartBytes: null,
+			mainContentStart: "unknown",
+			indexingRisk: "unknown"
+		}
+	};
+}
 // packages/core/src/analyze/crawler/page-eligibility.ts
 function comparableUrl(value) {
 	if (!value) return void 0;
@@ -11515,6 +11614,21 @@ function isCdnUtilityUrl(value) {
 		return false;
 	}
 }
+function globMatch(pattern, value) {
+	if (!pattern.includes("*")) return value.includes(pattern);
+	const parts = pattern.split("*");
+	let position = 0;
+	for (const part of parts) {
+		if (!part) continue;
+		const found = value.slice(position).indexOf(part);
+		if (found === -1) return false;
+		position += found + part.length;
+	}
+	return true;
+}
+function passesFilters(url, include, exclude) {
+	return (!include.length || include.some((pattern) => globMatch(pattern, url))) && !exclude.some((pattern) => globMatch(pattern, url));
+}
 // packages/core/src/analyze/crawler/audit.ts
 function sameUrl(a, b) {
 	if (!a || !b) return false;
@@ -11557,7 +11671,7 @@ function urlKey(value) {
 	}
 }
 const SLOW_RESPONSE_MS = 2e3;
-const LARGE_HTML_BYTES = 2097152;
+const LARGE_HTML_BYTES = GOOGLEBOT_HTML_LIMIT_BYTES;
 const COMPRESSION_MIN_BYTES = 4096;
 const DEEP_PAGE_DEPTH = 4;
 const WEAK_VALUABLE_INLINKS = 1;
@@ -11729,6 +11843,11 @@ function auditCrawlRequests(requests) {
 	for (const request of requests) {
 		if (request.outcome === "failure") {
 			if (request.failureKind === "aborted") continue;
+			const overCap = responseCapIssue(request);
+			if (overCap) {
+				issues.push(overCap);
+				continue;
+			}
 			issues.push(requestIssue("connection_error", request, request.error, {
 				outcome: request.outcome,
 				failureKind: request.failureKind,
@@ -11841,10 +11960,7 @@ function auditCrawlPages(pages, opts = {}) {
 			outgoingInternalCount: page.outgoingInternalCount,
 			rendering: page.fetchDiagnostics?.rendering
 		}));
-		if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(issue$1("large_html", page, `${Math.round((page.sizeBytes ?? 0) / 1024)} KB`, {
-			sizeBytes: page.sizeBytes,
-			thresholdBytes: LARGE_HTML_BYTES
-		}));
+		if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(largeHtmlIssue(issue$1("large_html", page), page));
 		if (isHtmlPage(page) && !page.compression && (page.sizeBytes ?? 0) > COMPRESSION_MIN_BYTES) issues.push(issue$1("no_compression", page, void 0, {
 			sizeBytes: page.sizeBytes,
 			thresholdBytes: COMPRESSION_MIN_BYTES,
@@ -12661,7 +12777,11 @@ function refreshPrioritiesCsvFiles(report) {
 }
 // packages/core/src/analyze/crawler/crawl-url-queue.ts
 function before(left, right) {
-	return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
+	if (left.depth !== right.depth) return left.depth < right.depth;
+	const leftRank = left.rank ?? 0;
+	const rightRank = right.rank ?? 0;
+	if (leftRank !== rightRank) return leftRank < rightRank;
+	return left.url < right.url;
 }
 var CrawlUrlQueue = class {
 	#items = [];
@@ -13191,14 +13311,20 @@ function rateLimitRerunCommand(config, intervalMs) {
 	return `seo crawl ${config.url}${scope}${sitemap} --max-pages ${config.maxPages} --concurrency 1 --fetch-interval-cap 1 --fetch-interval-ms ${slowMs}`;
 }
 /** Set when the sitemaps list more URLs than the page limit let the crawl read. */
-function sitemapLimitCaveatInput(config, discovery, defaulted) {
-	if (!discovery || discovery.urlsReturned < config.maxPages || !discovery.roots.some((root) => root.possiblyTruncated)) return;
+function sitemapLimitCaveatInput(config, discovery, defaulted, sample) {
+	if (!discovery) return void 0;
+	const truncated = discovery.roots.some((root) => root.possiblyTruncated);
+	if (!sample && (discovery.urlsReturned < config.maxPages || !truncated)) return;
 	return {
 		maxPages: config.maxPages,
 		defaulted,
 		ceiling: MAX_CRAWL_PAGES,
 		health: config.strategy === "health",
-		url: config.url
+		url: config.url,
+		...sample ? { sample: {
+			...sample,
+			readCapped: truncated
+		} } : {}
 	};
 }
 function crawlCaveats(input) {
@@ -13210,8 +13336,9 @@ function crawlCaveats(input) {
 		caveats.push(`${CRAWL_COVERAGE_CAVEAT_PREFIX} read ${readPages} of ${readPages + skippedUrls} pages because the origin kept rate limiting the crawl (HTTP 429) after it backed off. Rerun slower: ${rerunCommand}`);
 	}
 	if (input.sitemapLimit) {
-		const { maxPages, defaulted, ceiling, health, url } = input.sitemapLimit;
-		caveats.push(`Checked only the first ${maxPages} sitemap URLs because the page limit is ${maxPages}${defaulted ? " by default" : ""}, and the sitemaps list more. Raise --max-pages, up to ${ceiling.toLocaleString("en-US")}, to check more: seo crawl ${url}${health ? " --health" : ""} --max-pages ${ceiling}`);
+		const { maxPages, defaulted, ceiling, health, url, sample } = input.sitemapLimit;
+		const rerun = `Raise --max-pages, up to ${ceiling.toLocaleString("en-US")}, to check more: seo crawl ${url}${health ? " --health" : ""} --max-pages ${ceiling}`;
+		caveats.push(sample ? `Checked a sample of ${sample.checked.toLocaleString("en-US")} of the ${sample.read.toLocaleString("en-US")}${sample.readCapped ? "+" : ""} sitemap URLs read because the page limit is ${maxPages}${defaulted ? " by default" : ""}. The sample takes URLs in turn from each of ${sample.sections} sections (the first path segment), shortest paths first, so no one section fills the crawl.${sample.readCapped ? " The sitemaps list more URLs than the crawl read." : ""} ${rerun}` : `Checked only the first ${maxPages} sitemap URLs because the page limit is ${maxPages}${defaulted ? " by default" : ""}, and the sitemaps list more. ${rerun}`);
 	}
 	if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
 	const renderingFallbackUrls = input.renderingFallbackUrls ?? 0;
@@ -14057,6 +14184,20 @@ const crawlPageSnapshotSchema = z.object({
 	responseHeaders: z.record(z.string(), z.string()).optional(),
 	responseTimeMs: z.number().optional(),
 	sizeBytes: z.number().int().optional(),
+	googlebotCut: z.object({
+		limitBytes: z.number().int().positive(),
+		bytesAfterLimit: z.number().int().nonnegative(),
+		mainContentStartBytes: z.number().int().nonnegative().optional(),
+		mainContentMarker: z.enum(["h1", "main"]).optional(),
+		mainContentStart: z.enum([
+			"before-limit",
+			"near-limit",
+			"after-limit",
+			"not-found"
+		]),
+		visibleWordsBeforeLimit: z.number().int().nonnegative(),
+		visibleWordsAfterLimit: z.number().int().nonnegative()
+	}).optional(),
 	usedJs: z.boolean().optional(),
 	fetchSource: z.enum([
 		"cache",
@@ -15551,6 +15692,7 @@ async function snapshotFetchedPage(url, fetched) {
 			responseTimeMs: fetched.diagnostics.durationMs,
 			sizeBytes: Buffer.byteLength(fetched.html),
 			usedJs: fetched.usedJs,
+			googlebotCut: googlebotCutEvidence(fetched.html),
 			fetchSource: fetched.diagnostics.source,
 			cacheState: fetched.diagnostics.cache,
 			fetchDiagnostics: fetched.diagnostics,
@@ -17996,6 +18138,74 @@ function sitemapDiscoveryStatus(roots) {
 	if (!roots.some((root) => root.dataStatus !== "unavailable")) return "unavailable";
 	return roots.every((root) => root.dataStatus === "complete") ? "complete" : "partial";
 }
+/** How many sitemap URLs a crawl reads to pick its sample from. */
+function sitemapReadLimit(maxPages) {
+	return Math.min(5e4, Math.max(1e4, maxPages * 10));
+}
+function sectionOf(url) {
+	try {
+		return new URL(url).pathname.split("/")[1]?.toLowerCase() ?? "";
+	} catch {
+		return "";
+	}
+}
+function pathDepth(url) {
+	try {
+		return new URL(url).pathname.split("/").filter(Boolean).length;
+	} catch {
+		return 0;
+	}
+}
+/**
+* Picks up to `limit` URLs round-robin across sections (the first path
+* segment), so one large section such as a forum cannot fill a capped crawl.
+* Within a section, shallower paths come first, then URL order.
+*/
+function spreadSitemapSample(urls, limit) {
+	const groups = /* @__PURE__ */ new Map();
+	for (const url of urls) {
+		const key = sectionOf(url);
+		const group = groups.get(key);
+		if (group) group.push(url);
+		else groups.set(key, [url]);
+	}
+	const ordered = [...groups.entries()].sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([, group]) => group.map((url) => ({
+		url,
+		depth: pathDepth(url)
+	})).sort((left, right) => left.depth - right.depth || (left.url < right.url ? -1 : left.url > right.url ? 1 : 0)).map((entry) => entry.url));
+	const picked = [];
+	for (let index = 0; picked.length < limit; index += 1) {
+		let added = false;
+		for (const group of ordered) {
+			const url = group[index];
+			if (url === void 0) continue;
+			added = true;
+			picked.push(url);
+			if (picked.length >= limit) break;
+		}
+		if (!added) break;
+	}
+	return {
+		urls: picked,
+		sections: groups.size
+	};
+}
+/**
+* The sitemap URLs a capped crawl checks, leaving room for the seed URLs, and
+* how they were picked when the sitemaps listed more.
+*/
+function sitemapCrawlSample(urls, maxPages, seedUrls, normalize) {
+	const candidates = urls.filter((url) => !seedUrls.has(normalize(url) ?? url));
+	const sample = spreadSitemapSample(candidates, Math.max(0, maxPages - seedUrls.size));
+	return candidates.length > sample.urls.length ? {
+		urls: sample.urls,
+		caveat: {
+			checked: sample.urls.length,
+			read: candidates.length,
+			sections: sample.sections
+		}
+	} : { urls: sample.urls };
+}
 async function sitemapSeeds(input) {
 	const roots = [];
 	const urls = /* @__PURE__ */ new Set();
@@ -18075,21 +18285,6 @@ function isLikelyAsset(url) {
 	const path = new URL(url).pathname.toLowerCase();
 	return /\.(avif|bmp|css|csv|dmg|eot|gif|gz|ico|jpeg|jpg|js|json|map|mov|mp3|mp4|ogg|otf|pdf|png|svg|tar|ttf|wav|webm|webp|woff|woff2|zip)$/.test(path);
 }
-function globMatch(pattern, value) {
-	if (!pattern.includes("*")) return value.includes(pattern);
-	const parts = pattern.split("*");
-	let position = 0;
-	for (const part of parts) {
-		if (!part) continue;
-		const found = value.slice(position).indexOf(part);
-		if (found === -1) return false;
-		position += found + part.length;
-	}
-	return true;
-}
-function passesFilters(url, include, exclude) {
-	return (!include.length || include.some((pattern) => globMatch(pattern, url))) && !exclude.some((pattern) => globMatch(pattern, url));
-}
 async function crawlSite(input, dependencies) {
 	const deps = resolveCrawlSiteDependencies(dependencies);
 	const config = assertCrawlConfigLimits(normalizeCrawlConfig(input));
@@ -18144,6 +18339,7 @@ async function crawlSite(input, dependencies) {
 		});
 	};
 	let sitemapDiscovery;
+	let sitemapSample;
 	let externalLinkVerification;
 	let statusEventChain = Promise.resolve();
 	const emitStatus = (phase, event = {}) => {
@@ -18191,7 +18387,7 @@ async function crawlSite(input, dependencies) {
 		const llmsTxt = aiSignals?.signals.llmsTxt;
 		const healthSitemapUrls = healthStrategy && !config.sitemapUrl && !isCancelled() ? await robotsSession?.sitemapUrls(origin) : void 0;
 		cancelled = isCancelled();
-		const enqueue = (value, depth) => {
+		const enqueue = (value, depth, rank) => {
 			const normalized = normalizeUrl$1(value, config.url);
 			if (!normalized) {
 				recordSkip("invalid-url");
@@ -18277,7 +18473,8 @@ async function crawlSite(input, dependencies) {
 			queuedUrls += 1;
 			queue.push({
 				url: normalized,
-				depth
+				depth,
+				rank
 			});
 			emitStatus("url_queued", {
 				url: normalized,
@@ -18297,12 +18494,16 @@ async function crawlSite(input, dependencies) {
 			const sitemap = await sitemapSeeds({
 				url: config.url,
 				sitemapUrl: config.sitemapUrl,
-				maxPages: config.maxPages,
+				maxPages: sitemapReadLimit(config.maxPages),
 				declaredSitemapUrls: aiSignals?.signals.robotsTxt?.sitemapUrls ?? healthSitemapUrls ?? [],
 				warnings,
 				fetchSitemapUrls: deps.fetchSitemapUrls
 			});
-			for (const url of sitemap.urls) enqueue(url, 0);
+			const sample = sitemapCrawlSample(sitemap.urls, config.maxPages, seedUrls, (url) => normalizeUrl$1(url, config.url));
+			sitemapSample = sample.caveat;
+			sample.urls.forEach((url, index) => {
+				enqueue(url, 0, index + 1);
+			});
 			sitemapDiscovery = sitemap.discovery;
 		}
 		checkMemoryPressure();
@@ -18463,7 +18664,7 @@ async function crawlSite(input, dependencies) {
 				});
 				continue;
 			}
-			const page = result.page;
+			const page = structuredClone(result.page);
 			const rendering = page.fetchDiagnostics?.rendering;
 			if (rendering?.status === "unavailable" && config.js === "on") throw new SeoError("INVALID_INPUT", `--rendering on needs a local Chrome or Chromium browser, and none could start. ${rendering.error ?? ""} Install Chromium, or set SEO_BROWSER_EXECUTABLE_PATH to a Chromium executable, then rerun. Use --rendering auto to accept raw HTML.`.replace(/\s+/g, " ").trim());
 			if (rendering?.status === "unavailable" || rendering?.status === "failed") renderingFallbackUrls += 1;
@@ -18733,7 +18934,7 @@ async function crawlSite(input, dependencies) {
 				queueSafetyInventoryCapped,
 				originBackpressureSkippedUrls,
 				rateLimit: rateLimit.caveatInput(config, pages.length),
-				sitemapLimit: sitemapLimitCaveatInput(config, sitemapDiscovery, input.maxPages === void 0),
+				sitemapLimit: sitemapLimitCaveatInput(config, sitemapDiscovery, input.maxPages === void 0, sitemapSample),
 				memoryPressureSkippedUrls,
 				queryVariantSkippedUrls: queryVariantSkipped.size,
 				renderingFallbackUrls,
@@ -31198,6 +31399,12 @@ function plainFor(word) {
 	for (const [key, value] of Object.entries(PLAIN)) if (lower.startsWith(key.slice(0, Math.max(4, key.length - 2)))) return value;
 	return "a plain word";
 }
+function swapFix(word) {
+	const plain = plainFor(word).replace(/\s*\([^)]*\)/g, "");
+	if (plain === "a plain word") return "Use a plain word.";
+	if (/^(?:say|name|give)\b/.test(plain)) return `Cut "${word}", or swap it for a detail the text already gives.`;
+	return `Use ${plain.split(", ").map((swap) => `"${swap}"`).join(" or ")}.`;
+}
 const PHRASE_RULES = [
 	{
 		id: "chatbot-residue",
@@ -31356,7 +31563,7 @@ const PHRASE_RULES = [
 			/\bstunning(?:ly)?\b|\benchanting\b/gi
 		],
 		message: (match) => `"${match.text}" is far more common in AI text than in human writing.`,
-		fix: (match) => `Use ${plainFor(match.text.replace(/^(?:rich|a) /i, ""))}.`
+		fix: (match) => swapFix(match.text.replace(/^(?:rich|a) /i, ""))
 	},
 	{
 		id: "banned-word",
@@ -31376,7 +31583,7 @@ const PHRASE_RULES = [
 			/\bgets? you to the next fix\b/gi
 		],
 		message: (match) => `"${match.text}" is on the house list of words to cut.`,
-		fix: (match) => /source of truth/i.test(match.text) ? "Name the actual file, field or result." : /hype|everything you need|next fix/i.test(match.text) ? "Say what the page or tool actually gives the reader." : `Use ${plainFor(match.text)}.`
+		fix: (match) => /source of truth/i.test(match.text) ? "Name the actual file, field or result." : /hype|everything you need|next fix/i.test(match.text) ? "Say what the page or tool actually gives the reader." : swapFix(match.text)
 	},
 	{
 		id: "buzzword",
@@ -31621,7 +31828,7 @@ const PHRASE_RULES = [
 			/(?:^|[.!?:]\s+)(?:But|And) only (?:part|half|some|partly)\.(?=\s|$)/g
 		],
 		message: "A two-word denial used for rhythm. It makes the reader wait.",
-		fix: "Join it to the sentence before, such as \"Nothing technical was holding these pages back.\""
+		fix: "Join it to the sentence before, so the point is stated once."
 	},
 	{
 		id: "strawman",
@@ -32419,10 +32626,10 @@ function stockCloser(ctx) {
 	for (const block of ctx.blocks(RHYTHM_KINDS)) for (const sentence of ctx.sentences(block)) {
 		const text = sentence.text;
 		if (CLOSERS.some((re) => re.test(text))) {
-			ctx.hit(rule, block, sentence.start, sentence.end, "A stock kicker that tells the reader how to feel about the last point.", "Cut it. If the point needs support, give the number or fact that makes it true.");
+			ctx.hit(rule, block, sentence.start, sentence.end, "A stock kicker that tells the reader how to feel about the last point.", "Cut it. If the text already has the number or fact that makes the point, end on that.");
 			continue;
 		}
-		if (/^(?:but |and )?(?:that|this)(?:'s| is) not (?:how|what|why|where|a|an|the)\b/i.test(text) && words$2(text).length <= 10) ctx.hit(reframe, block, sentence.start, sentence.end, "A short \"That's not ...\" line dismisses a view instead of stating one.", "Say what is true instead, such as \"Agents can do this work in a day.\"");
+		if (/^(?:but |and )?(?:that|this)(?:'s| is) not (?:how|what|why|where|a|an|the)\b/i.test(text) && words$2(text).length <= 10) ctx.hit(reframe, block, sentence.start, sentence.end, "A short \"That's not ...\" line dismisses a view instead of stating one.", "Cut it, or say what is true with a fact the text already gives.");
 	}
 }
 const FIGURATIVE_NOUNS = "fights?|battles?|hills?|wars?|bets?|gambles?|races?|risks?|horses?|roads?|bridges?|doors?|games?|ships?|cards?";
@@ -33212,15 +33419,24 @@ async function ahrefs(key, input, missed, totals) {
 		order_by: "refdomains_target:desc"
 	})).pages ?? [];
 	if (input.action === "serp") return eachKeyword(input.keywords.slice(0, serpKeywords(input)), missed, async (keyword) => {
-		return ((await get("serp-overview/serp-overview", {
+		const data = await get("serp-overview/serp-overview", {
 			select: "position,type,url,title,domain_rating,traffic,keywords",
 			keyword,
 			country: input.country,
 			top_positions: String(serpDepth(input))
-		})).positions ?? []).map((row) => ({
-			keyword,
-			...row
-		}));
+		});
+		let organic = 0;
+		return (data.positions ?? []).map((row) => {
+			const { position, type, ...rest } = row;
+			const kinds = Array.isArray(type) ? type.map(String) : [String(type ?? "")];
+			return {
+				keyword,
+				position: kinds.includes("organic") ? ++organic : null,
+				ahrefs_position: position ?? null,
+				type: kinds.join(",") || null,
+				...rest
+			};
+		});
 	});
 	if (input.action === "backlinks") return (await get("site-explorer/refdomains", {
 		select: "domain,domain_rating,links_to_target,dofollow_links,first_seen,last_seen",
@@ -33248,7 +33464,55 @@ function flattenIntents(row) {
 		intents: names.join(" ") || null
 	};
 }
+// packages/core/src/commands/research-serp.ts
+const GOTO = /^https?:\/\/(www\.)?google\.[a-z.]+\/goto\b/;
+const hostOf$3 = (url) => {
+	try {
+		return new URL(url).hostname;
+	} catch {
+		return "";
+	}
+};
+const text = (value) => typeof value === "string" && value ? value : null;
+function dataForSeoSerp(keyword, items) {
+	const found = items.find((item) => item.type === "ai_overview");
+	const overview = found ? {
+		keyword,
+		references: (found.references ?? []).slice(0, 20).map((ref) => {
+			const url = String(ref.url ?? "");
+			const goto = GOTO.test(url);
+			return {
+				domain: goto ? null : text(ref.domain) ?? (hostOf$3(url) || null),
+				url: goto || !url ? null : url,
+				title: text(ref.title)
+			};
+		})
+	} : null;
+	return {
+		rows: items.filter((item) => item.type === "organic" || item.type === "featured_snippet").map((item) => {
+			const row = {
+				keyword,
+				position: item.type === "organic" ? Number(item.rank_group) : null,
+				rank_absolute: Number(item.rank_absolute),
+				type: String(item.type),
+				url: text(item.url),
+				domain: text(item.domain),
+				title: text(item.title)
+			};
+			if (GOTO.test(String(row.url))) {
+				const crumb = String(item.breadcrumb ?? "");
+				const site = /^https?:\/\/([^/\s›]+)/.exec(crumb);
+				row.url = null;
+				row.domain = site?.[1] ?? null;
+				row.url_hint = site ? crumb.replace(/^https?:\/\//, "") : null;
+			}
+			return row;
+		}),
+		overview
+	};
+}
 // packages/core/src/commands/research-providers.ts
+const RESEARCH_DEVICES = ["desktop", "mobile"];
 const RESEARCH_SCOPES = [
 	"domain",
 	"prefix",
@@ -33372,7 +33636,25 @@ async function dataForSeoList([login, password], path) {
 	if (!task || task.status_code >= 4e4) throw new CommandError("PROVIDER_ERROR", `dataforseo: no list at ${path}`);
 	return task.result ?? [];
 }
-async function dataForSeoTask([login, password], path, task) {
+async function dataForSeoTask(keys, path, task) {
+	let spent = 0;
+	for (let attempt = 1;; attempt += 1) try {
+		const done = await dataForSeoTaskOnce(keys, path, task);
+		return {
+			...done,
+			cost: done.cost + spent
+		};
+	} catch (error) {
+		const failed = error;
+		spent += failed.cost ?? 0;
+		if (!SE_ERRORS.has(failed.status ?? 0) || attempt >= SE_TRIES) throw Object.assign(error, {
+			cost: spent,
+			...attempt > 1 ? { tries: attempt } : {}
+		});
+		await wait(SE_RETRY_MS[attempt - 1] ?? 3e3);
+	}
+}
+async function dataForSeoTaskOnce([login, password], path, task) {
 	const result = (await readJson(await call("dataforseo", new Request(`https://api.dataforseo.com/${path}`, {
 		method: "POST",
 		headers: {
@@ -33387,7 +33669,10 @@ async function dataForSeoTask([login, password], path, task) {
 		partial: false,
 		noResults: true
 	};
-	if (!result || result.status_code >= 4e4 && result.status_code !== PARTIAL) throw Object.assign(new CommandError("PROVIDER_ERROR", `dataforseo: ${result?.status_message ?? "no task result"}`), { cost: result?.cost ?? 0 });
+	if (!result || result.status_code >= 4e4 && result.status_code !== PARTIAL) throw Object.assign(new CommandError("PROVIDER_ERROR", `dataforseo: ${result?.status_message ?? "no task result"}`), {
+		cost: result?.cost ?? 0,
+		status: result?.status_code ?? 0
+	});
 	return {
 		cost: result.cost ?? 0,
 		result: result.result?.[0] ?? null,
@@ -33396,6 +33681,9 @@ async function dataForSeoTask([login, password], path, task) {
 }
 const NO_RESULTS = 40102;
 const PARTIAL = 40106;
+const SE_ERRORS = /* @__PURE__ */ new Set([40101, 5e4]);
+const SE_TRIES = 3;
+const SE_RETRY_MS = [1e3, 3e3];
 const queued = (task) => ({
 	id: String(task.id ?? ""),
 	status_code: Number(task.status_code ?? 0),
@@ -33435,6 +33723,7 @@ async function dataForSeo(keys, input) {
 		noData: withoutData(input, rows, spent.missed, 700),
 		...spent.available !== void 0 ? { available: spent.available } : {},
 		...spent.siteTotal ? { siteTotal: spent.siteTotal } : {},
+		...spent.aiOverviews ? { aiOverviews: spent.aiOverviews } : {},
 		...spent.rankFailed ? { rankFailed: true } : {}
 	};
 }
@@ -33523,7 +33812,9 @@ async function dataForSeoRows(keys, input, spent) {
 		}
 		return items.map((item) => pick(item, {
 			keyword: "keyword_data.keyword",
-			position: "ranked_serp_element.serp_item.rank_absolute",
+			position: "ranked_serp_element.serp_item.rank_group",
+			rank_absolute: "ranked_serp_element.serp_item.rank_absolute",
+			type: "ranked_serp_element.serp_item.type",
 			url: "ranked_serp_element.serp_item.url",
 			search_volume: "keyword_data.keyword_info.search_volume",
 			etv: "ranked_serp_element.serp_item.etv",
@@ -33533,28 +33824,13 @@ async function dataForSeoRows(keys, input, spent) {
 	}
 	if (input.action === "serp") {
 		const rows = await eachKeyword(input.keywords.slice(0, serpKeywords(input)), spent.missed, async (keyword) => {
-			return (await post("v3/serp/google/organic/live/advanced", {
+			const { rows, overview } = dataForSeoSerp(keyword, await post("v3/serp/google/organic/live/advanced", {
 				keyword,
-				depth: serpDepth(input)
-			}, 2)).filter((item) => item.type === "organic").map((item) => {
-				const row = pick(item, {
-					position: "rank_absolute",
-					url: "url",
-					domain: "domain",
-					title: "title"
-				});
-				if (/^https?:\/\/(www\.)?google\.[a-z.]+\/goto\b/.test(String(row.url))) {
-					const crumb = String(item.breadcrumb ?? "");
-					const site = /^https?:\/\/([^/\s›]+)/.exec(crumb);
-					row.url = null;
-					row.domain = site?.[1] ?? null;
-					row.url_hint = site ? crumb.replace(/^https?:\/\//, "") : null;
-				}
-				return {
-					keyword,
-					...row
-				};
-			});
+				depth: serpDepth(input),
+				device: input.device ?? "desktop"
+			}, 2));
+			if (overview) spent.aiOverviews = [...spent.aiOverviews ?? [], overview];
+			return rows;
 		});
 		return input.noRank ? rows : withDomainRank(keys, rows, spent);
 	}
@@ -33672,7 +33948,7 @@ async function serpOverlap(provider, headers, input) {
 	const keywords = [...new Set(input.keywords.slice(0, OVERLAP_KEYWORDS))].filter((keyword) => !serps.missed.includes(keyword) && !serps.noData.includes(keyword));
 	const host = (value) => String(value ?? "").toLowerCase().replace(/^www\./, "");
 	const top = new Map(keywords.map((keyword) => {
-		const rows = serps.rows.filter((row) => row.keyword === keyword).slice(0, OVERLAP_DEPTH);
+		const rows = serps.rows.filter((row) => row.keyword === keyword && row.type !== "featured_snippet").slice(0, OVERLAP_DEPTH);
 		const pages = new Set(rows.map((row) => row.url).filter((url) => typeof url === "string" && url !== "").map((url) => url.replace(/\/$/, "").toLowerCase()));
 		const sites = new Set(rows.map((row) => host(row.domain ?? siteOf(row.url))).filter(Boolean));
 		return [keyword, {
@@ -37779,6 +38055,121 @@ async function runBingReport(input, deps) {
 		data: report
 	};
 }
+const DEVICES = ["desktop", "mobile"];
+const DEPTH = 10;
+const bare = (host) => host.toLowerCase().replace(/^www\./, "");
+function key(url) {
+	try {
+		const parsed = new URL(url);
+		return `${bare(parsed.hostname)}${parsed.pathname.replace(/\/+$/, "")}`.toLowerCase();
+	} catch {
+		return "";
+	}
+}
+function hintKey(hint) {
+	const parts = hint.split("›").map((part) => part.trim());
+	if (!parts[0] || parts.some((part) => /^\.\.\.$|…/.test(part))) return null;
+	return `${bare(parts[0])}${parts.length > 1 ? `/${parts.slice(1).join("/")}` : ""}`.toLowerCase();
+}
+function phraseFrom(text, words = 2) {
+	const cleaned = (text ?? "").split(/\s+[|\u2013\u2014-]\s+/)[0]?.replace(/["“”]/g, "").trim();
+	if (!cleaned || cleaned.split(/\s+/).length < words) return null;
+	return cleaned.split(/\s+/).slice(0, 12).join(" ");
+}
+async function runIndexCheck(input, headers) {
+	let target;
+	try {
+		target = new URL(/^https?:\/\//i.test(input.url) ? input.url : `https://${input.url}`);
+	} catch {
+		throw new CommandError("INVALID_INPUT", "index_check needs a page URL as target.");
+	}
+	const keys = dataForSeoKeys(headers);
+	const country = input.country ?? "us";
+	const market = dataForSeoMarket(country, dataForSeoLanguage(country));
+	const page = key(target.href);
+	const host = bare(target.hostname);
+	let phrases = (input.phrases ?? []).slice(0, 2);
+	let phraseSource = phrases.length ? "given" : null;
+	if (!phrases.length) {
+		const live = await liveCheck(target.href).catch(() => null);
+		const sentence = (live?.last?.description ?? "").split(/(?<=[.!?])\s/)[0] ?? "";
+		const found = [phraseFrom(live?.last?.h1 ?? null) ?? phraseFrom(live?.last?.title ?? null), phraseFrom(sentence.replace(/[.!?]$/, ""), 5)].filter((phrase) => Boolean(phrase));
+		phrases = [...new Set(found)].slice(0, 2);
+		phraseSource = phrases.length ? "the page’s H1 or title and its meta description" : null;
+	}
+	const queries = [`site:${host}${target.pathname === "/" ? "" : target.pathname.replace(/\/+$/, "")}`, ...phrases.map((phrase) => `"${phrase.replace(/"/g, "")}"`)];
+	const rows = [];
+	let cost = 0;
+	for (const query of queries) for (const device of DEVICES) {
+		const row = {
+			query,
+			device,
+			result: "empty",
+			position: null,
+			rank_absolute: null,
+			results: 0,
+			top_domains: "",
+			error: null
+		};
+		try {
+			const done = await dataForSeoTask(keys, "v3/serp/google/organic/live/advanced", {
+				...market,
+				keyword: query,
+				depth: DEPTH,
+				device
+			});
+			cost += done.cost;
+			const results = dataForSeoSerp(query, done.result?.items ?? []).rows;
+			row.results = results.length;
+			row.top_domains = [...new Set(results.map((r) => String(r.domain ?? "?")))].slice(0, 3).join(", ");
+			const onSite = results.filter((r) => {
+				const domain = bare(String(r.domain ?? ""));
+				return domain === host || domain.endsWith(`.${host}`);
+			});
+			const match = onSite.find((r) => typeof r.url === "string" && key(r.url) === page || typeof r.url_hint === "string" && hintKey(r.url_hint) === page);
+			if (match) {
+				row.result = "page";
+				row.position = typeof match.position === "number" ? match.position : null;
+				row.rank_absolute = Number(match.rank_absolute) || null;
+			} else if (onSite.some((r) => r.url === null && (typeof r.url_hint !== "string" || hintKey(r.url_hint) === null || hintKey(r.url_hint) === host))) row.result = "unresolved";
+			else if (onSite.length) row.result = "other_pages";
+			else if (results.length) row.result = query.startsWith("site:") ? "fallback" : "other_sites";
+		} catch (error) {
+			if (error instanceof CommandError && error.code === "NO_KEY") throw error;
+			cost += error.cost ?? 0;
+			row.result = "error";
+			row.error = error.message;
+		}
+		rows.push(row);
+	}
+	const has = (result) => rows.some((row) => row.result === result);
+	const verdict = has("page") ? "indexed" : has("error") || has("unresolved") ? "unknown" : "not_found";
+	const reason = verdict === "indexed" ? `Google returned the page itself for ${rows.filter((row) => row.result === "page").length} of ${rows.length} searches.` : verdict === "unknown" ? has("error") ? `${rows.filter((row) => row.result === "error").length} of ${rows.length} searches failed after retries, and none of the others returned the page. Ask again later.` : "Google returned results from the site, but DataForSEO gave Google redirect links whose page it could not resolve, so they may or may not be this page." : `None of ${rows.length} searches returned the page.${has("fallback") ? " Where the site: search showed other sites, Google had dropped the filter because nothing matched; those are not rankings." : ""} It may be missing from the index, or indexed without this text. Check it with URL Inspection (seo page or seo index) if the site is yours.`;
+	const usd = Math.round(cost * 1e4) / 1e4;
+	return {
+		text: [
+			`Index check for ${target.href} (${country}, desktop and mobile): ${verdict.replace("_", " ")}.`,
+			reason,
+			phraseSource ? `Exact phrase from ${phraseSource}.` : "No exact phrase: none was given and the page’s H1 and title could not be read, so only site: was asked.",
+			"",
+			...rows.map((row) => `${row.device.padEnd(7)}  ${row.result}${row.position !== null ? ` (organic ${row.position}, absolute ${row.rank_absolute})` : ""}  ${row.query}${row.results ? `  [${row.results} results: ${row.top_domains}]` : ""}${row.error ? `  ${row.error}` : ""}`),
+			"",
+			"This asks Google search, not Search Console. Google can index a page and still not show it for a site: or phrase search, so not found is evidence, not proof.",
+			`DataForSEO cost: $${usd}.`
+		].join("\n"),
+		data: {
+			provider: "dataforseo",
+			action: "index_check",
+			url: target.href,
+			country,
+			verdict,
+			reason,
+			phrases,
+			checks: rows,
+			costUsd: usd
+		}
+	};
+}
 // packages/core/src/commands/research-cache.ts
 const RESEARCH_TTL_SECONDS = 86400;
 function workersResearchStore() {
@@ -37823,10 +38214,11 @@ const RESEARCH_ACTIONS = [
 	"top_pages",
 	"pages_by_backlinks",
 	"keyword_ideas",
-	"serp_overlap"
+	"serp_overlap",
+	"index_check"
 ];
 const RESEARCH_PROVIDERS = ["dataforseo", "ahrefs"];
-const RESEARCH_HELP = `provider is optional: without it the call uses the saved key, DataForSEO when both are saved. The target is a domain (the whole site), a path ending in / such as example.com/blog/ (every URL under it), or one page; scope overrides that. keyword_metrics (keywords: volume, difficulty, CPC), keyword_ideas (up to 5 seed keywords: searches that contain each seed, by volume), ranked_keywords (target: keywords a site, subfolder or page ranks for, with the provider's traffic total for the whole target), top_pages (target: the pages with the most estimated organic traffic), serp (keywords, up to 5: the ranking pages, each with its domain's authority: DataForSEO domain_rank 0 to 100, Ahrefs domain_rating; depth sets results per keyword, default 20; rows run position by position across the keywords), serp_overlap (2 to 10 keywords: for each pair, how many of their top 10 pages and sites are the same, to decide one page or several; results without a real page are counted, not compared), competitors (target: domains that share keywords), backlinks (target: referring domains), pages_by_backlinks (target: the pages with the most referring domains), link_gap (target and 2 to 5 competitors: referring domains that link to two or more competitors and not the target, highest authority first, from each site's top 1,000 referring domains; costs one backlinks call per site), ai_answers (DataForSEO only; target: the site. Without keywords it takes the site’s top 10 chat-style and long questions from Search Console’s AI-shaped queries itself. With up to 10 questions in keywords it asks those instead, and the target can be any domain, such as a competitor's. Each answer row also lists the other sites it cites, and the notes rank them. Answers come from DataForSEO's queue: answers not ready after about 40 seconds are returned as tasks, already paid for; call again with the same target and those tasks to read them at no cost. It asks ChatGPT search, Google AI Mode and Google AI Overviews each one and says whether the answer cites or mentions the site. DataForSEO’s queue prices are about $0.0072 a question across the three engines, so about $0.07 a run: tell the person before running it). Rows with a domain, and the target, get a free Ahrefs domain_rating column. Numbers are provider estimates, not Search Console data.`;
+const RESEARCH_HELP = `provider is optional: without it the call uses the saved key, DataForSEO when both are saved. The target is a domain (the whole site), a path ending in / such as example.com/blog/ (every URL under it), or one page; scope overrides that. keyword_metrics (keywords: volume, difficulty, CPC), keyword_ideas (up to 5 seed keywords: searches that contain each seed, by volume), ranked_keywords (target: keywords a site, subfolder or page ranks for, with the provider's traffic total for the whole target), top_pages (target: the pages with the most estimated organic traffic), serp (keywords, up to 5: the ranking pages with their organic position beside DataForSEO's rank_absolute, which also counts AI Overviews and other blocks, plus the AI Overview's sources; device desktop or mobile, DataForSEO only; each with its domain's authority: DataForSEO domain_rank 0 to 100, Ahrefs domain_rating; depth sets results per keyword, default 20; rows run position by position across the keywords), index_check (DataForSEO only; target: one page URL; keywords: up to 2 exact phrases from the page, default its H1 or title. Searches Google for site:<url> and each "<phrase>" on desktop and mobile, retries failed searches, treats Google's fallback to unrelated sites as no match, and says indexed, not found or unknown with each search's evidence; about $0.01 a page), serp_overlap (2 to 10 keywords: for each pair, how many of their top 10 pages and sites are the same, to decide one page or several; results without a real page are counted, not compared), competitors (target: domains that share keywords), backlinks (target: referring domains), pages_by_backlinks (target: the pages with the most referring domains), link_gap (target and 2 to 5 competitors: referring domains that link to two or more competitors and not the target, highest authority first, from each site's top 1,000 referring domains; costs one backlinks call per site), ai_answers (DataForSEO only; target: the site. Without keywords it takes the site’s top 10 chat-style and long questions from Search Console’s AI-shaped queries itself. With up to 10 questions in keywords it asks those instead, and the target can be any domain, such as a competitor's. Each answer row also lists the other sites it cites, and the notes rank them. Answers come from DataForSEO's queue: answers not ready after about 40 seconds are returned as tasks, already paid for; call again with the same target and those tasks to read them at no cost. It asks ChatGPT search, Google AI Mode and Google AI Overviews each one and says whether the answer cites or mentions the site. DataForSEO’s queue prices are about $0.0072 a question across the three engines, so about $0.07 a run: tell the person before running it). Rows with a domain, and the target, get a free Ahrefs domain_rating column. Numbers are provider estimates, not Search Console data.`;
 const GENERIC_PLATFORMS = [
 	"youtube.com",
 	"facebook.com",
@@ -37888,6 +38280,17 @@ async function runResearch(input, keys, options) {
 	const offset = input.offset ?? 0;
 	if (input.action === "ai_answers") throw new CommandError("INVALID_INPUT", "ai_answers runs on its own.");
 	const provider = chooseProvider(input.provider, keys);
+	if (input.action === "index_check") {
+		if (provider === "ahrefs") throw new CommandError("INVALID_INPUT", "index_check uses DataForSEO: it runs live Google searches, which Ahrefs does not offer.");
+		return runIndexCheck({
+			url: target,
+			phrases: input.keywords,
+			country
+		}, keys);
+	}
+	if (input.device && input.action !== "serp") throw new CommandError("INVALID_INPUT", "device works with serp only. ranked_keywords and the other database actions have one ranking per keyword, from desktop results.");
+	if (input.device === "mobile" && provider === "ahrefs") throw new CommandError("INVALID_INPUT", "Ahrefs' SERP overview has no device choice, so it cannot show mobile results. Use provider dataforseo for device mobile.");
+	const device = input.action === "serp" ? input.device ?? "desktop" : null;
 	const language = provider === "dataforseo" ? dataForSeoLanguage(country) : null;
 	const scope = [
 		"ranked_keywords",
@@ -37903,7 +38306,8 @@ async function runResearch(input, keys, options) {
 		...scope ? { scope } : {},
 		competitors: input.competitors ?? [],
 		country,
-		...input.action === "serp" ? { depth: input.depth ?? 20 } : {}
+		...input.action === "serp" ? { depth: input.depth ?? 20 } : {},
+		...device === "mobile" ? { device } : {}
 	};
 	const store = options.store ?? workersResearchStore();
 	const key = await researchKey([
@@ -37921,10 +38325,10 @@ async function runResearch(input, keys, options) {
 		savedAt: (/* @__PURE__ */ new Date()).toISOString(),
 		result
 	});
-	const { rows: fetched, costUsd: paid, partial, missed, noData, floor, leftOut, available, siteTotal, rankFailed, compared } = result;
+	const { rows: fetched, costUsd: paid, partial, missed, noData, floor, leftOut, available, siteTotal, rankFailed, compared, aiOverviews } = result;
 	const costUsd = saved ? 0 : paid;
 	const order = new Map(request.keywords.map((word, index) => [word, index]));
-	const found = input.action === "serp" && request.keywords.length > 1 ? [...fetched].sort((a, b) => Number(a.position ?? 1e9) - Number(b.position ?? 1e9) || (order.get(String(a.keyword)) ?? 0) - (order.get(String(b.keyword)) ?? 0)) : fetched;
+	const found = input.action === "serp" && request.keywords.length > 1 ? [...fetched].sort((a, b) => Number(a.position ?? 0) - Number(b.position ?? 0) || (order.get(String(a.keyword)) ?? 0) - (order.get(String(b.keyword)) ?? 0)) : fetched;
 	const { rows, note } = input.action === "competitors" ? realCompetitors(found, target) : {
 		rows: found,
 		note: null
@@ -37942,7 +38346,7 @@ async function runResearch(input, keys, options) {
 	const perKeyword = input.action === "serp" ? Object.fromEntries(request.keywords.map((word) => [word, rated.rows.filter((row) => row.keyword === word).length])) : null;
 	return {
 		text: rowsText({
-			title: `${provider} ${input.action}${target ? ` for ${target}` : ""}${scope ? ` (${SCOPE_TEXT[scope]})` : ""}${LINK_ACTIONS.includes(input.action) ? ", all countries" : `, ${country}${language ? `, language ${language}` : ""}`}.`,
+			title: `${provider} ${input.action}${device && provider === "dataforseo" ? ` (${device})` : ""}${target ? ` for ${target}` : ""}${scope ? ` (${SCOPE_TEXT[scope]})` : ""}${LINK_ACTIONS.includes(input.action) ? ", all countries" : `, ${country}${language ? `, language ${language}` : ""}`}.`,
 			rows: rated.rows,
 			offset,
 			limit: size,
@@ -37951,6 +38355,8 @@ async function runResearch(input, keys, options) {
 				`${provider} estimates, in the provider’s own units.`,
 				...compared ? [`Compared: ${compared.withPage} of ${compared.results} top results had a real page, so shared_pages counts only those; ${compared.results - compared.withPage} came back as Google redirect links and could not be compared by page. ${compared.withSite} had a known site for shared_sites. Unique pages and sites are counted per keyword.${compared.withPage < compared.results / 2 && provider === "dataforseo" ? " With this few pages, shared_pages cannot settle one page or several; provider ahrefs returns real pages." : ""}`] : [],
 				...VOLUME_ACTIONS.includes(input.action) ? [`Volume model: ${VOLUME_MODEL[provider]} ${VOLUMES_DIFFER}`] : [],
+				...input.action === "serp" || input.action === "ranked_keywords" ? [positionNote(provider)] : [],
+				...aiOverviews?.length ? [aiOverviewNote(aiOverviews, request.keywords.length)] : input.action === "serp" && provider === "dataforseo" && rows.length ? ["No AI Overview on these results pages."] : [],
 				...perKeyword ? [`Results per keyword: ${Object.entries(perKeyword).map(([word, count]) => `${word} ${count}`).join(", ")}. Rows run position by position across the keywords.`] : [],
 				...siteTotal ? [siteTotalNote(provider, siteTotal, rows.length)] : [],
 				...available !== void 0 && available > found.length ? [`Capped: ${provider} has ${available.toLocaleString("en-US")} rows for this; this read the first ${found.length.toLocaleString("en-US")} in its order.`] : [],
@@ -37984,6 +38390,8 @@ async function runResearch(input, keys, options) {
 			available: available ?? null,
 			country,
 			language,
+			...device ? { device: provider === "dataforseo" ? device : null } : {},
+			...input.action === "serp" && provider === "dataforseo" ? { aiOverviews: aiOverviews ?? [] } : {},
 			targetDomainRating: hasTargetDr ? targetDr : null,
 			attribution: rated.added || hasTargetDr ? DR_ATTRIBUTION : null,
 			leftOut: note,
@@ -38007,6 +38415,17 @@ async function runResearch(input, keys, options) {
 		}
 	};
 }
+function positionNote(provider) {
+	return provider === "dataforseo" ? "position is the organic position: the nth organic link (DataForSEO rank_group). rank_absolute counts every block on the page from the top, AI Overviews, People Also Ask, videos and images included, so it is higher. A featured snippet has type featured_snippet and no organic position." : "position is the organic position: the nth row Ahrefs marks organic. ahrefs_position is Ahrefs' own position field, and type lists what each row is.";
+}
+function aiOverviewNote(overviews, keywords) {
+	const each = overviews.map((overview) => {
+		const sites = [...new Set(overview.references.map((ref) => ref.domain).filter((domain) => Boolean(domain)))];
+		const unknown = overview.references.filter((ref) => !ref.domain).length;
+		return `${overview.keyword}: ${overview.references.length} sources${sites.length ? ` (${sites.slice(0, 6).join(", ")}${sites.length > 6 ? ", …" : ""})` : ""}${unknown ? `, ${unknown} behind Google redirect links` : ""}`;
+	});
+	return `AI Overview shown for ${overviews.length} of ${keywords} keyword${keywords === 1 ? "" : "s"}. ${each.join("; ")}. The full list is in the JSON as aiOverviews.`;
+}
 const LINK_ACTIONS = [
 	"backlinks",
 	"pages_by_backlinks",
@@ -38634,7 +39053,7 @@ const percent = (site) => {
 	const clicks = site.clicks;
 	return Math.round(Math.abs(clicks.clicks - clicks.previous) / clicks.previous * 100);
 };
-function issues(site, startMs, endMs) {
+function issues(site, startMs, endMs, first = false) {
 	const found = [];
 	if (site.index?.dropped) found.push({
 		rank: 0,
@@ -38656,11 +39075,11 @@ function issues(site, startMs, endMs) {
 	});
 	if (crawled && crawl.new_issues > 0) found.push({
 		rank: 3,
-		heading: plural$1(crawl.new_issues, "new issue")
+		heading: plural$1(crawl.new_issues, first ? "open finding" : "new issue")
 	});
 	return found;
 }
-function siteLines(site, startMs, endMs) {
+function siteLines(site, startMs, endMs, first = false) {
 	const lines = [];
 	const index = site.index;
 	if (!index) lines.push("Google index: no checks ran this week yet.");
@@ -38677,7 +39096,7 @@ function siteLines(site, startMs, endMs) {
 	if (crawled && crawl.status === "failed") lines.push(`Weekly crawl: stopped after ${plural$1(crawl.pages, "page")}${crawl.error ? `: ${sentence(crawl.error)}` : "."} It tries again next week.`);
 	else if (crawled) {
 		const rule = crawl.top_rule ? AUDIT_RULES[crawl.top_rule] : void 0;
-		lines.push(crawl.new_issues ? `Weekly crawl: ${plural$1(crawl.pages, "page")}, ${plural$1(crawl.new_issues, "new issue")}${rule ? `, most of them ${rule.charAt(0).toLowerCase()}${rule.slice(1)}` : ""}.` : `Weekly crawl: ${plural$1(crawl.pages, "page")}, no new issues.`);
+		lines.push(crawl.new_issues ? `Weekly crawl: ${plural$1(crawl.pages, "page")}, ${plural$1(crawl.new_issues, first ? "open finding" : "new issue")}${rule ? `, most of them ${rule.charAt(0).toLowerCase()}${rule.slice(1)}` : ""}.` : `Weekly crawl: ${plural$1(crawl.pages, "page")}, no new issues.`);
 	} else if (site.added_at < startMs) lines.push(`Weekly crawl: none finished this week${site.start_error ? `: ${sentence(site.start_error)}` : "."} It tries again each hour.`);
 	else lines.push("Weekly crawl: the first crawl is still running.");
 	const clicks = site.clicks;
@@ -38745,7 +39164,7 @@ function weekMarkdown(data) {
 	const byName = (a, b) => siteName(a.property).localeCompare(siteName(b.property)) || a.property.localeCompare(b.property);
 	const rows = data.sites.map((site) => ({
 		site,
-		found: issues(site, startMs, endMs),
+		found: issues(site, startMs, endMs, data.first),
 		waiting: !site.crawl || !site.index
 	}));
 	const attention = rows.filter((row) => row.found.length);
@@ -38757,7 +39176,7 @@ function weekMarkdown(data) {
 		"",
 		attention.length ? total === 1 ? "Your site had changes worth a look this week." : `${attention.length} of your ${total} sites had changes worth a look this week.` : quiet.length ? total === 1 ? "Your site had a quiet week." : quiet.length === total ? `All ${total} of your sites had a quiet week.` : `${quiet.length} of your ${total} sites had a quiet week.` : "Your sites are still on their first checks."
 	];
-	for (const { site, found } of rows) out.push("", `## ${siteName(site.property)}${found.length ? `: ${list(found.map((item) => item.heading))}` : ""}`, "", ...siteLines(site, startMs, endMs).map((line) => `- ${line}`), ...opportunityLines(site), ...measuredLines(site), ...updateLines(site), ...site.link ? [`- [View site](${site.link})`] : []);
+	for (const { site, found } of rows) out.push("", `## ${siteName(site.property)}${found.length ? `: ${list(found.map((item) => item.heading))}` : ""}`, "", ...siteLines(site, startMs, endMs, data.first).map((line) => `- ${line}`), ...opportunityLines(site), ...measuredLines(site), ...updateLines(site), ...site.link ? [`- [View site](${site.link})`] : []);
 	return out.join("\n");
 }
 async function runAllSites(deps, sites) {
@@ -38995,6 +39414,7 @@ const SHARED_TOOLS = [
 			brand: z.array(z.string().min(1).max(60)).max(10).default([]).describe("ai_answers only: the brand names that count as a mention, such as Datadog for datadoghq.com. Default: each domain's name."),
 			country: z.string().regex(/^[a-z]{2}$/).default("us").describe("Two-letter country code."),
 			depth: z.number().int().min(1).max(100).optional().describe(`serp only: results per keyword, default 20. Deeper costs more.`),
+			device: z.enum(RESEARCH_DEVICES).optional().describe("serp only: desktop (default) or mobile. Google ranks pages differently on each. DataForSEO only."),
 			...detail
 		}),
 		run: async (surface, input) => {