seo 0.2.12 → 0.2.14

package/dist/index.js

@@ -1,5 +1,5 @@
 import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
-import { n as load, t as version } from "./package-B-M-4PZ_.js";
+import { n as load, t as version } from "./package-Cp8DwqCy.js";
 import { Headers, fetch as fetch$1 } from "undici";
 import { chmodSync, createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { arch, homedir, platform, totalmem } from "node:os";
@@ -2532,6 +2532,43 @@ function extractCanonicalEvidence($, headers, baseUrl) {
 	return evidenceFromCandidates([...htmlCandidates($, baseUrl), ...headerCandidates(headers, baseUrl)]);
 }
 //#endregion
+//#region packages/core/src/extract/content-sketch.ts
+const SHINGLE_SIZE = 5;
+const MAX_HASHES = 32;
+function words(value) {
+	return value.normalize("NFKC").toLocaleLowerCase("en").match(/[\p{L}\p{N}]+/gu) ?? [];
+}
+function sampleIndexes(total, limit) {
+	if (total <= limit) return Array.from({ length: total }, (_, index) => index);
+	return Array.from({ length: limit }, (_, index) => Math.round(index * (total - 1) / (limit - 1)));
+}
+function digest(value) {
+	return createHash("sha256").update(value).digest("hex").slice(0, 16);
+}
+function contentSketch(value) {
+	const tokens = words(value);
+	const hashes = sampleIndexes(Math.max(0, tokens.length - SHINGLE_SIZE + 1), MAX_HASHES).map((index) => digest(tokens.slice(index, index + SHINGLE_SIZE).join(" ")));
+	return {
+		version: 1,
+		wordCount: tokens.length,
+		shingleSize: SHINGLE_SIZE,
+		sampledShingles: hashes.length,
+		hashes
+	};
+}
+function contentSketchCoverage(source, candidate) {
+	if (!source || source.hashes.length < 4) return null;
+	const targets = new Set(source.hashes);
+	const matched = /* @__PURE__ */ new Set();
+	const tokens = words(candidate);
+	for (let index = 0; index <= tokens.length - source.shingleSize; index += 1) {
+		const hash = digest(tokens.slice(index, index + source.shingleSize).join(" "));
+		if (targets.has(hash)) matched.add(hash);
+		if (matched.size === targets.size) break;
+	}
+	return Math.round(matched.size / targets.size * 1e3) / 1e3;
+}
+//#endregion
 //#region packages/core/src/extract/main-content.ts
 function readableError(error) {
 	return error instanceof Error ? error.message : String(error);
@@ -3139,6 +3176,22 @@ function extractStructuredData($, baseUrl, options = {}) {
 }
 //#endregion
 //#region packages/core/src/extract/page-extractor.ts
+function metaRefresh($, finalUrl) {
+	for (const element of $("meta[http-equiv]").toArray()) {
+		if ($(element).attr("http-equiv")?.trim().toLowerCase() !== "refresh") continue;
+		const match = ($(element).attr("content")?.trim())?.match(/^([0-9]+(?:\.[0-9]+)?)\s*;\s*url\s*=\s*(?:"([^"]*)"|'([^']*)'|(.+))$/iu);
+		if (!match) continue;
+		try {
+			const target = new URL((match[2] ?? match[3] ?? match[4] ?? "").trim(), finalUrl);
+			if (!["http:", "https:"].includes(target.protocol)) continue;
+			target.hash = "";
+			return {
+				targetUrl: target.toString(),
+				delaySeconds: Number(match[1])
+			};
+		} catch {}
+	}
+}
 const SOCIAL_PROFILE_HOSTS = [
 	"facebook.com",
 	"instagram.com",
@@ -3149,6 +3202,48 @@ const SOCIAL_PROFILE_HOSTS = [
 	"x.com",
 	"youtube.com"
 ];
+const LOGIN_PATH_PATTERN = /(?:^|\/)(?:auth|login|log-in|signin|sign-in|sso)(?:\/|$)/iu;
+const LOGIN_TITLE_PATTERN = /\b(?:authenticate|log[ -]?in|sign[ -]?in)\b/iu;
+function softAuthenticationGate($, finalUrl, title) {
+	const passwordForms = $("form").toArray().filter((form) => $(form).find("input[type=\"password\" i]").length > 0);
+	if (passwordForms.length === 0) return void 0;
+	const pagePath = new URL(finalUrl).pathname;
+	const action = $(passwordForms[0]).attr("action");
+	const actionUrl = action ? httpUrl$1(action, finalUrl) : void 0;
+	const actionPath = actionUrl ? new URL(actionUrl).pathname : void 0;
+	const indicators = [
+		...LOGIN_PATH_PATTERN.test(pagePath) ? ["login-path"] : [],
+		...LOGIN_TITLE_PATTERN.test(title ?? "") ? ["login-title"] : [],
+		...LOGIN_PATH_PATTERN.test(actionPath ?? "") ? ["login-form-action"] : []
+	];
+	if (indicators.length === 0) return void 0;
+	return {
+		kind: "login-form",
+		indicators,
+		...actionPath ? { formActionPath: actionPath } : {}
+	};
+}
+function tabbedContent($) {
+	const groups = $("[role=\"tablist\" i]").length;
+	const panels = $("[role=\"tabpanel\" i]").toArray();
+	if (groups === 0 || panels.length === 0) return void 0;
+	const panelSketches = panels.slice(0, 12).map((panel) => {
+		const labelledBy = $(panel).attr("aria-labelledby");
+		const labelledElement = labelledBy ? $("[id]").toArray().find((element) => $(element).attr("id") === labelledBy) : void 0;
+		const label = safeText$1($(panel).attr("aria-label") ?? (labelledElement ? $(labelledElement).text() : void 0));
+		return {
+			...label ? { label: label.slice(0, 80) } : {},
+			sketch: contentSketch($(panel).text())
+		};
+	});
+	return {
+		groups,
+		panels: panels.length,
+		retainedPanels: panelSketches.length,
+		truncated: panels.length > panelSketches.length,
+		panelSketches
+	};
+}
 function safeText$1(value) {
 	const trimmed = value?.replace(/\s+/g, " ").trim();
 	return trimmed ? trimmed : void 0;
@@ -3375,10 +3470,13 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
 		html: sanitizedContentHtml($, fetchResult.html, fetchResult.finalUrl)
 	}, extractor, dependencies);
 	const { text, excerpt } = content;
+	const title = safeText$1($("title").first().text());
+	const crawlerMetaRefresh = extractor === "crawler" ? metaRefresh($, fetchResult.finalUrl) : void 0;
+	if (crawlerMetaRefresh && new URL(crawlerMetaRefresh.targetUrl).origin === url.origin) crawlerInternalLinks.add(crawlerMetaRefresh.targetUrl);
 	return {
 		url: fetchResult.url,
 		finalUrl: fetchResult.finalUrl,
-		title: safeText$1($("title").first().text()),
+		title,
 		metaDescription: safeText$1($("meta[name=\"description\"]").attr("content")),
 		metaRobots,
 		xRobotsTag: safeText$1(headerValue$4(fetchResult.headers, "x-robots-tag")),
@@ -3398,7 +3496,11 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
 			externalLinks: [...crawlerExternalLinks],
 			internalAnchorSamples,
 			externalAnchorSamples,
-			socialProfileLinks: [...socialProfileLinks]
+			socialProfileLinks: [...socialProfileLinks],
+			contentSketch: contentSketch(text),
+			metaRefresh: crawlerMetaRefresh,
+			softAuthenticationGate: softAuthenticationGate($, fetchResult.finalUrl, title),
+			tabbedContent: tabbedContent($)
 		} } : {},
 		markdownAlternates,
 		hreflang,
@@ -7631,6 +7733,8 @@ function markdownQuality(markdown, page) {
 	const codeFences = markdown.match(/^```/gmu)?.length ?? 0;
 	const wordCount = bodyWordCount(markdown);
 	const sourceWordCount = page.wordCount ?? 0;
+	const evaluatedPanels = (page.tabbedContent?.panelSketches ?? []).map((panel) => contentSketchCoverage(panel.sketch, markdown)).filter((coverage) => coverage !== null);
+	const retainedPanels = evaluatedPanels.filter((coverage) => coverage >= .6).length;
 	return {
 		frontmatterTitle: /^---[\s\S]*?^title:\s*.+$/mu.test(markdown),
 		h1Count: markdown.match(/^#\s+\S.+$/gmu)?.length ?? 0,
@@ -7647,7 +7751,15 @@ function markdownQuality(markdown, page) {
 		sourceWordCount,
 		wordRetentionRatio: sourceWordCount > 0 ? Math.round(wordCount / sourceWordCount * 1e3) / 1e3 : null,
 		introductoryCopyRetained: introProbe ? normalizedMarkdown.includes(introProbe) : null,
-		navigationOnly: bodyWordCount(markdown) < 25
+		navigationOnly: bodyWordCount(markdown) < 25,
+		contentSketchCoverage: contentSketchCoverage(page.contentSketch, markdown),
+		tabbedContent: {
+			detectedPanels: page.tabbedContent?.panels ?? 0,
+			evaluatedPanels: evaluatedPanels.length,
+			retainedPanels,
+			missingPanels: Math.max(0, evaluatedPanels.length - retainedPanels),
+			complete: evaluatedPanels.length === 0 ? null : retainedPanels === evaluatedPanels.length
+		}
 	};
 }
 function combinedSignal(timeoutMs, signal) {
@@ -7701,6 +7813,8 @@ async function fetchRepresentation(input) {
 				status: response.status,
 				contentType: response.headers.get("content-type") ?? void 0,
 				bytes: Buffer.byteLength(body),
+				characters: body.length,
+				estimatedTokens: Math.ceil(body.length / 4),
 				sha256: sha256(body),
 				canonicalUrl: link.find((entry) => entry.rel.includes("canonical"))?.url,
 				alternateUrl: link.find((entry) => entry.rel.includes("alternate") && entry.type === "text/markdown")?.url,
@@ -7759,11 +7873,15 @@ async function inspectMarkdownPage(input) {
 	if (explicit) observation.explicit = explicit.observation;
 	observation.negotiated = negotiated.observation;
 	observation.repeated = repeated.observation;
-	observation.explicitMatchesNegotiated = explicit?.observation.sha256 && negotiated.observation.sha256 ? explicit.observation.sha256 === negotiated.observation.sha256 : null;
-	const primary = explicit ?? negotiated;
-	observation.repeatedHashStable = primary.observation.sha256 && repeated.observation.sha256 ? primary.observation.sha256 === repeated.observation.sha256 : null;
-	observation.markdownCanonicalMatchesHtml = sameDocument(primary.observation.canonicalUrl, input.page.finalUrl);
-	if (primary.body) observation.quality = markdownQuality(primary.body, input.page);
+	const isMarkdown = (result) => result?.observation.status !== void 0 && result.observation.status >= 200 && result.observation.status < 300 && /^\s*text\/markdown\b/iu.test(result.observation.contentType ?? "");
+	const explicitMarkdown = isMarkdown(explicit) ? explicit : void 0;
+	const negotiatedMarkdown = isMarkdown(negotiated) ? negotiated : void 0;
+	const repeatedMarkdown = isMarkdown(repeated) ? repeated : void 0;
+	observation.explicitMatchesNegotiated = explicitMarkdown?.observation.sha256 && negotiatedMarkdown?.observation.sha256 ? explicitMarkdown.observation.sha256 === negotiatedMarkdown.observation.sha256 : null;
+	const primary = explicitMarkdown ?? negotiatedMarkdown;
+	observation.repeatedHashStable = primary?.observation.sha256 && repeatedMarkdown?.observation.sha256 ? primary.observation.sha256 === repeatedMarkdown.observation.sha256 : null;
+	observation.markdownCanonicalMatchesHtml = sameDocument(primary?.observation.canonicalUrl, input.page.finalUrl);
+	if (primary?.body) observation.quality = markdownQuality(primary.body, input.page);
 	return observation;
 }
 async function fetchText(input) {
@@ -8007,17 +8125,16 @@ async function inspectRouteManifest(input) {
 		const declared = Array.isArray(parsed.pages) ? parsed.pages : [];
 		const declaredHtmlRoutes = declared.map((page) => page.htmlPath).filter((path) => typeof path === "string").sort();
 		const declaredMarkdownRoutes = declared.map((page) => page.markdownPath).filter((path) => typeof path === "string").sort();
-		const crawledHtmlRoutes = new Set(input.pages.map((page) => new URL(page.finalUrl).pathname.replace(/\/$/u, "") || "/"));
-		const observedMarkdownRoutes = new Set(input.observations.flatMap((page) => page.advertisedUrls.map((value) => new URL(value).pathname)));
 		return {
 			url,
 			status: result.response.status,
 			valid: Array.isArray(parsed.pages),
 			declaredHtmlRoutes,
 			declaredMarkdownRoutes,
-			missingHtmlRoutes: declaredHtmlRoutes.filter((path) => !crawledHtmlRoutes.has(path.replace(/\/$/u, "") || "/")).sort(),
-			missingMarkdownRoutes: declaredMarkdownRoutes.filter((path) => !observedMarkdownRoutes.has(path)).sort(),
-			orphanMarkdownRoutes: [...observedMarkdownRoutes].filter((path) => !declaredMarkdownRoutes.includes(path)).sort()
+			missingHtmlRoutes: [],
+			missingMarkdownRoutes: [],
+			orphanMarkdownRoutes: [],
+			comparisonStatus: "partial"
 		};
 	} catch (error) {
 		return {
@@ -8028,6 +8145,7 @@ async function inspectRouteManifest(input) {
 			missingHtmlRoutes: [],
 			missingMarkdownRoutes: [],
 			orphanMarkdownRoutes: [],
+			comparisonStatus: "not-applicable",
 			error: safeError(error)
 		};
 	}
@@ -8071,9 +8189,7 @@ async function collectAgentDiscovery(input) {
 		}),
 		inspectRouteManifest({
 			...input,
-			origin,
-			pages,
-			observations
+			origin
 		}),
 		inspectAgentEndpoints({
 			...input,
@@ -8178,20 +8294,13 @@ async function collectAgentDiscovery(input) {
 	};
 }
 //#endregion
-//#region packages/core/src/analyze/crawler/agent-readiness.ts
-function check$1(section, input) {
+//#region packages/core/src/analyze/crawler/agent-readiness-markdown.ts
+function check$2(section, input) {
 	return {
 		section,
 		...input
 	};
 }
-function section$2(id, title, checks) {
-	return {
-		id,
-		title,
-		checks
-	};
-}
 function failedMarkdownUrls(discovery) {
 	return discovery.markdownAlternates.pages.filter((page) => {
 		const negotiatedOk = page.negotiated?.status === 200 && /^\s*text\/markdown\b/iu.test(page.negotiated.contentType ?? "") && page.negotiated.varyAccept;
@@ -8199,112 +8308,200 @@ function failedMarkdownUrls(discovery) {
 		return !negotiatedOk || !explicitOk || page.repeatedHashStable !== true;
 	}).map((page) => page.htmlUrl);
 }
+function unstableMarkdownUrls(discovery) {
+	return discovery.markdownAlternates.pages.filter((page) => confirmedMarkdown(page) !== void 0 && page.repeatedHashStable !== true).map((page) => page.htmlUrl);
+}
 function qualityFailureUrls(discovery) {
 	return discovery.markdownAlternates.pages.filter((page) => {
 		const quality = page.quality;
-		if (!quality) return true;
-		return !quality.frontmatterTitle || quality.h1Count !== 1 || !quality.codeFenceBalanced || quality.rawHtmlTags > 0 || quality.rawSvgTags > 0 || quality.rawScriptTags > 0 || quality.rawStyleTags > 0 || quality.navigationOnly || quality.repeatedLines > 0 || quality.wordRetentionRatio !== null && quality.wordRetentionRatio < .4;
+		if (!quality) return false;
+		return !quality.frontmatterTitle || quality.h1Count !== 1 || !quality.codeFenceBalanced || quality.rawHtmlTags > 0 || quality.rawSvgTags > 0 || quality.rawScriptTags > 0 || quality.rawStyleTags > 0 || quality.navigationOnly || quality.repeatedLines > 0 || quality.contentSketchCoverage !== null && quality.contentSketchCoverage !== void 0 && quality.contentSketchCoverage < .6 || quality.tabbedContent?.complete === false || quality.wordRetentionRatio !== null && quality.wordRetentionRatio < .4;
 	}).map((page) => page.htmlUrl);
 }
-function profileChecks(discovery) {
-	return [
-		"api",
-		"application",
-		"commerce"
-	].map((profile) => check$1("profile", {
-		id: `profile-${profile}`,
-		status: "notApplicable",
-		title: `${profile[0]?.toUpperCase()}${profile.slice(1)} checks are not applicable`,
-		plainEnglish: discovery.profileApplicability[profile].reason,
-		action: "Choose this profile only when the site exposes that capability publicly.",
-		evidence: {
-			profile,
-			selectedProfile: discovery.profile
-		}
-	}));
-}
-function summariseChecks(checks) {
-	const count = (status) => checks.filter((item) => item.status === status).length;
-	return {
-		checks: checks.length,
-		passed: count("pass"),
-		warnings: count("warning"),
-		failed: count("fail"),
-		unknown: count("unknown"),
-		information: count("info"),
-		notApplicable: count("notApplicable")
-	};
+function confirmedMarkdown(page) {
+	for (const response of [page.explicit, page.negotiated]) if (response?.status !== void 0 && response.status >= 200 && response.status < 300 && /^\s*text\/markdown\b/iu.test(response.contentType ?? "")) return response;
 }
 function markdownChecks(discovery) {
 	const markdown = discovery.markdownAlternates;
 	const failedUrls = failedMarkdownUrls(discovery);
+	const unstableUrls = unstableMarkdownUrls(discovery);
 	const qualityUrls = qualityFailureUrls(discovery);
+	const hasEligibleHtml = markdown.eligibleHtmlPages > 0;
+	const hasEvaluatedMarkdown = markdown.evaluatedPages > 0;
 	const negotiatedPages = markdown.pages.filter((page) => page.negotiated?.status === 200 && /^\s*text\/markdown\b/iu.test(page.negotiated.contentType ?? "")).length;
 	const explicitPages = markdown.pages.filter((page) => page.htmlAlternateUnique).length;
 	const coverageComplete = markdown.eligibleHtmlPages > 0 && markdown.evaluatedPages === markdown.eligibleHtmlPages;
 	const negotiationComplete = markdown.eligibleHtmlPages > 0 && negotiatedPages === markdown.eligibleHtmlPages && markdown.pages.every((page) => page.negotiated?.varyAccept) && markdown.pages.every((page) => !page.htmlAlternateUnique || page.explicitMatchesNegotiated === true);
-	const stable = markdown.eligibleHtmlPages > 0 && markdown.stableResponses === markdown.eligibleHtmlPages;
+	const stable = markdown.evaluatedPages > 0 && markdown.stableResponses === markdown.evaluatedPages;
 	const tokenHeaders = markdown.pages.filter((page) => {
-		return (page.explicit ?? page.negotiated)?.markdownTokens !== void 0 && (!page.explicit || !page.negotiated || page.explicit.markdownTokens === page.negotiated.markdownTokens);
+		return confirmedMarkdown(page)?.markdownTokens !== void 0 && (!page.explicit || !page.negotiated || page.explicit.markdownTokens === page.negotiated.markdownTokens);
 	}).length;
+	const sizedPages = markdown.pages.filter((page) => {
+		return confirmedMarkdown(page)?.characters !== void 0;
+	});
+	const largePages = sizedPages.filter((page) => {
+		return (confirmedMarkdown(page)?.characters ?? 0) > 5e4;
+	});
+	const veryLargePages = sizedPages.filter((page) => {
+		return (confirmedMarkdown(page)?.characters ?? 0) > 1e5;
+	});
+	const maxCharacters = Math.max(0, ...sizedPages.map((page) => confirmedMarkdown(page)?.characters ?? 0));
+	const maxEstimatedTokens = Math.max(0, ...sizedPages.map((page) => confirmedMarkdown(page)?.estimatedTokens ?? 0));
+	const parityPages = markdown.pages.filter((page) => page.quality?.contentSketchCoverage !== null && page.quality?.contentSketchCoverage !== void 0);
+	const parityFailures = parityPages.filter((page) => (page.quality?.contentSketchCoverage ?? 0) < .6);
+	const tabbedPages = markdown.pages.filter((page) => (page.quality?.tabbedContent?.detectedPanels ?? 0) > 0);
+	const tabFailures = tabbedPages.filter((page) => page.quality?.tabbedContent?.complete === false);
+	const tabUnknown = tabbedPages.filter((page) => page.quality?.tabbedContent?.complete === null);
 	return [
-		check$1("representations", {
+		check$2("representations", {
 			id: "markdown-coverage",
 			status: markdown.eligibleHtmlPages === 0 ? "unknown" : coverageComplete ? "pass" : "fail",
-			title: coverageComplete ? "Every successful HTML page has a Markdown representation" : "Markdown representation coverage is incomplete",
-			plainEnglish: `${markdown.evaluatedPages} of ${markdown.eligibleHtmlPages} successful HTML pages returned Markdown. ${explicitPages} advertised an explicit alternative and ${negotiatedPages} supported content negotiation.`,
-			action: coverageComplete ? "Keep the negotiated or explicit Markdown response stable as routes change." : "Return text/markdown through content negotiation or advertise one working Markdown alternative for each public HTML page.",
+			title: coverageComplete ? "Every successful HTML page has a Markdown representation" : !hasEligibleHtml ? "Markdown representation coverage was not evaluated" : "Markdown representation coverage is incomplete",
+			plainEnglish: hasEligibleHtml ? `${markdown.evaluatedPages} of ${markdown.eligibleHtmlPages} successful HTML pages returned Markdown. ${explicitPages} advertised an explicit alternative and ${negotiatedPages} supported content negotiation.` : "No successful HTML page was available, so Markdown representation coverage could not be evaluated.",
+			action: !hasEligibleHtml ? "Resolve the crawl or page-fetch failure, then run this check again." : coverageComplete ? "Keep the negotiated or explicit Markdown response stable as routes change." : "Return text/markdown through content negotiation or advertise one working Markdown alternative for each public HTML page.",
 			evidence: {
 				eligibleHtmlPages: markdown.eligibleHtmlPages,
 				advertisedPages: markdown.advertisedPages,
 				negotiatedPages,
 				evaluatedPages: markdown.evaluatedPages
 			},
-			urls: failedUrls.slice(0, 25)
+			urls: hasEligibleHtml ? failedUrls.slice(0, 25) : []
 		}),
-		check$1("representations", {
+		check$2("representations", {
 			id: "markdown-token-estimates",
-			status: markdown.evaluatedPages === 0 ? "unknown" : tokenHeaders === markdown.evaluatedPages ? "pass" : "warning",
-			title: tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes a token estimate" : "Some Markdown responses are missing a stable token estimate",
-			plainEnglish: `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned X-Markdown-Tokens. Paired explicit and negotiated responses agreed where both existed. The value is an estimate, not an exact model-specific token count.`,
-			action: "Return a stable token estimate and keep it consistent when both explicit and negotiated responses exist.",
+			status: markdown.evaluatedPages === 0 ? "unknown" : tokenHeaders === markdown.evaluatedPages ? "pass" : "info",
+			title: !hasEvaluatedMarkdown ? "Markdown token estimates were not evaluated" : tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes a token estimate" : "Some Markdown responses are missing a stable token estimate",
+			plainEnglish: hasEvaluatedMarkdown ? `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned X-Markdown-Tokens. Paired explicit and negotiated responses agreed where both existed. The value is an estimate, not an exact model-specific token count.` : "No confirmed Markdown response was available, so optional server token estimates could not be evaluated.",
+			action: !hasEvaluatedMarkdown ? "Confirm a successful text/markdown representation, then run this check again." : tokenHeaders === markdown.evaluatedPages ? "Keep the estimate stable when both explicit and negotiated responses exist." : "No change is required. X-Markdown-Tokens is optional; local size estimates are reported separately.",
 			urls: markdown.pages.filter((page) => {
-				return (page.explicit ?? page.negotiated)?.markdownTokens === void 0 || page.explicit !== void 0 && page.negotiated !== void 0 && page.explicit.markdownTokens !== page.negotiated.markdownTokens;
+				const primary = confirmedMarkdown(page);
+				if (!primary) return false;
+				const paired = [page.explicit, page.negotiated].filter((response) => response?.status !== void 0 && response.status >= 200 && response.status < 300 && /^\s*text\/markdown\b/iu.test(response.contentType ?? ""));
+				return primary?.markdownTokens === void 0 || paired.length === 2 && paired[0]?.markdownTokens !== paired[1]?.markdownTokens;
 			}).map((page) => page.htmlUrl).slice(0, 25)
 		}),
-		check$1("representations", {
+		check$2("representations", {
+			id: "markdown-size",
+			status: sizedPages.length === 0 ? "unknown" : largePages.length === 0 ? "pass" : "warning",
+			title: sizedPages.length === 0 ? "Markdown size was not measured" : largePages.length === 0 ? "Markdown representations stay within the review threshold" : "Some Markdown representations are large enough to review",
+			plainEnglish: `${sizedPages.length} Markdown responses were measured locally. ${largePages.length} exceeded 50,000 characters and ${veryLargePages.length} exceeded 100,000 characters. These are processing-risk thresholds, not universal model limits.`,
+			action: sizedPages.length === 0 ? "Confirm a successful text/markdown representation, then run this check again." : largePages.length === 0 ? "Keep important content early and retain stable section structure as pages grow." : "Split genuinely separate topics, keep navigation and repeated chrome out of Markdown, and put the most useful content early. Do not remove necessary detail just to hit a threshold.",
+			evidence: {
+				measuredPages: sizedPages.length,
+				reviewThresholdCharacters: 5e4,
+				highRiskThresholdCharacters: 1e5,
+				largePages: largePages.length,
+				veryLargePages: veryLargePages.length,
+				maxCharacters,
+				maxEstimatedTokens,
+				heuristic: true
+			},
+			urls: largePages.map((page) => page.htmlUrl).slice(0, 25)
+		}),
+		check$2("representations", {
+			id: "markdown-content-parity",
+			status: parityPages.length === 0 ? "unknown" : parityFailures.length === 0 ? "pass" : "warning",
+			title: parityPages.length === 0 ? "HTML and Markdown content parity was not measured" : parityFailures.length === 0 ? "Markdown retains sampled content from the HTML document" : "Some Markdown responses may omit important HTML content",
+			plainEnglish: `${parityPages.length} pages were compared using bounded hashes sampled across the main HTML content. ${parityFailures.length} retained less than 60% of that sample. This detects likely omissions without storing full page bodies.`,
+			action: parityPages.length === 0 ? "Confirm a successful text/markdown representation, then run this check again." : parityFailures.length === 0 ? "Keep the shared conversion path covered as templates and components change." : "Compare the affected HTML and Markdown around the missing sections, then fix the shared converter or component serialization.",
+			evidence: {
+				evaluatedPages: parityPages.length,
+				failedPages: parityFailures.length,
+				minimumCoverage: .6,
+				sampling: "bounded-content-shingles"
+			},
+			urls: parityFailures.map((page) => page.htmlUrl).slice(0, 25)
+		}),
+		check$2("representations", {
+			id: "markdown-tab-serialization",
+			status: tabbedPages.length === 0 ? "notApplicable" : tabFailures.length > 0 ? "warning" : tabUnknown.length > 0 ? "unknown" : "pass",
+			title: tabbedPages.length === 0 ? "No accessible tabbed content was detected" : tabFailures.length > 0 ? "Some tab panels are missing from Markdown" : tabUnknown.length > 0 ? "Some tab panels were too small to compare reliably" : "Markdown retains detected tab panels",
+			plainEnglish: tabbedPages.length === 0 ? "The crawled HTML did not contain ARIA tablists with tab panels, so this check does not apply." : `${tabbedPages.length} pages contained accessible tab panels. ${tabFailures.length} did not retain every evaluated panel in Markdown, and ${tabUnknown.length} had no panel large enough for a reliable comparison.`,
+			action: tabFailures.length === 0 ? tabUnknown.length > 0 ? "Review the affected page directly if the short tab labels or values are important to the document." : "Keep every panel serialized in document order even when only one tab is visible by default." : "Serialize all tab panels into the Markdown document with headings instead of exporting only the initially active panel.",
+			evidence: {
+				tabbedPages: tabbedPages.length,
+				failedPages: tabFailures.length,
+				unknownPages: tabUnknown.length
+			},
+			urls: [...tabFailures, ...tabUnknown].map((page) => page.htmlUrl).slice(0, 25)
+		}),
+		check$2("representations", {
 			id: "markdown-negotiation",
 			status: markdown.eligibleHtmlPages === 0 ? "unknown" : negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "pass" : "fail",
-			title: negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "Markdown content negotiation works across the site" : "Markdown content negotiation needs attention",
-			plainEnglish: `${negotiatedPages} of ${markdown.eligibleHtmlPages} pages returned negotiated Markdown. ${markdown.exactByteMatches} of ${explicitPages} paired explicit responses matched byte for byte. A request that refuses Markdown ${discovery.contentNegotiation.qZeroHonoured ? "received HTML" : "did not produce confirmed HTML evidence"}.`,
-			action: "Honour Accept q-values, send Vary: Accept, and keep paired explicit and negotiated responses identical when both exist.",
+			title: !hasEligibleHtml ? "Markdown content negotiation was not evaluated" : negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "Markdown content negotiation works across the site" : "Markdown content negotiation needs attention",
+			plainEnglish: hasEligibleHtml ? `${negotiatedPages} of ${markdown.eligibleHtmlPages} pages returned negotiated Markdown. ${markdown.exactByteMatches} of ${explicitPages} paired explicit responses matched byte for byte. A request that refuses Markdown ${discovery.contentNegotiation.qZeroHonoured ? "received HTML" : "did not produce confirmed HTML evidence"}.` : "No successful HTML page was available, so content negotiation could not be evaluated.",
+			action: hasEligibleHtml ? "Honour Accept q-values, send Vary: Accept, and keep paired explicit and negotiated responses identical when both exist." : "Resolve the crawl or page-fetch failure, then run this check again.",
 			evidence: {
 				negotiatedPages,
 				explicitPages,
 				exactByteMatches: markdown.exactByteMatches,
 				qZero: discovery.contentNegotiation
 			},
-			urls: failedUrls.slice(0, 25)
+			urls: hasEligibleHtml ? failedUrls.slice(0, 25) : []
 		}),
-		check$1("representations", {
+		check$2("representations", {
 			id: "markdown-determinism",
-			status: markdown.eligibleHtmlPages === 0 ? "unknown" : stable ? "pass" : "fail",
-			title: stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
-			plainEnglish: `${markdown.stableResponses} of ${markdown.eligibleHtmlPages} repeated Markdown responses kept the same SHA-256 digest.`,
-			action: "Remove timestamps, random output, or other runtime rewriting that changes the same Markdown response between requests.",
+			status: markdown.evaluatedPages === 0 ? "unknown" : stable ? "pass" : "fail",
+			title: !hasEvaluatedMarkdown ? "Markdown response stability was not evaluated" : stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
+			plainEnglish: hasEvaluatedMarkdown ? `${markdown.stableResponses} of ${markdown.evaluatedPages} repeated Markdown responses kept the same SHA-256 digest.` : "No confirmed Markdown response was available for a repeated stability check.",
+			action: !hasEvaluatedMarkdown ? "Confirm a successful text/markdown representation, then run this check again." : stable ? "Keep repeated Markdown responses byte-stable as templates and data change." : "Remove timestamps, random output, or other runtime rewriting that changes the same Markdown response between requests.",
 			evidence: { stableResponses: markdown.stableResponses },
-			urls: failedUrls.slice(0, 25)
+			urls: unstableUrls.slice(0, 25)
 		}),
-		check$1("representations", {
+		check$2("representations", {
 			id: "markdown-quality",
 			status: markdown.evaluatedPages === 0 ? "unknown" : qualityUrls.length === 0 ? "pass" : "warning",
-			title: qualityUrls.length === 0 ? "Markdown keeps the useful document structure cleanly" : "Some Markdown alternatives need an extraction review",
-			plainEnglish: qualityUrls.length === 0 ? "The evaluated alternatives retained one H1, frontmatter titles, balanced code fences, useful copy, and no leaked SVG, script, style, or layout tags." : `${qualityUrls.length} Markdown alternative${qualityUrls.length === 1 ? "" : "s"} lost important structure, looked navigation-only, or leaked presentation markup.`,
-			action: "Open the affected Markdown directly and fix the shared converter or component exclusion rules rather than patching generated files.",
+			title: !hasEvaluatedMarkdown ? "Markdown quality was not evaluated" : qualityUrls.length === 0 ? "Markdown keeps the useful document structure cleanly" : "Some Markdown alternatives need an extraction review",
+			plainEnglish: !hasEvaluatedMarkdown ? "No confirmed Markdown response was available for structure and extraction checks." : qualityUrls.length === 0 ? "The evaluated alternatives retained one H1, frontmatter titles, balanced code fences, useful copy, and no leaked SVG, script, style, or layout tags." : `${qualityUrls.length} Markdown alternative${qualityUrls.length === 1 ? "" : "s"} lost important structure, looked navigation-only, or leaked presentation markup.`,
+			action: !hasEvaluatedMarkdown ? "Confirm a successful text/markdown representation, then run this check again." : qualityUrls.length === 0 ? "Keep the shared Markdown conversion path covered as templates and components change." : "Open the affected Markdown directly and fix the shared converter or component exclusion rules rather than patching generated files.",
 			urls: qualityUrls.slice(0, 25)
 		})
 	];
 }
+//#endregion
+//#region packages/core/src/analyze/crawler/agent-readiness.ts
+function check$1(section, input) {
+	return {
+		section,
+		...input
+	};
+}
+function section$2(id, title, checks) {
+	return {
+		id,
+		title,
+		checks
+	};
+}
+function profileChecks(discovery) {
+	return [
+		"api",
+		"application",
+		"commerce"
+	].map((profile) => check$1("profile", {
+		id: `profile-${profile}`,
+		status: "notApplicable",
+		title: `${profile[0]?.toUpperCase()}${profile.slice(1)} checks are not applicable`,
+		plainEnglish: discovery.profileApplicability[profile].reason,
+		action: "Choose this profile only when the site exposes that capability publicly.",
+		evidence: {
+			profile,
+			selectedProfile: discovery.profile
+		}
+	}));
+}
+function summariseChecks(checks) {
+	const count = (status) => checks.filter((item) => item.status === status).length;
+	return {
+		checks: checks.length,
+		passed: count("pass"),
+		warnings: count("warning"),
+		failed: count("fail"),
+		unknown: count("unknown"),
+		information: count("info"),
+		notApplicable: count("notApplicable")
+	};
+}
 function contentSignalsCheck(discovery, robotsContentSignals) {
 	const signals = discovery.contentSignals;
 	const robotsDeclared = robotsContentSignals ?? [];
@@ -8405,8 +8602,8 @@ function discoveryChecks(discovery, robotsContentSignals) {
 		check$1("discovery", {
 			id: "route-manifest",
 			status: !discovery.routeManifest.valid ? "info" : discovery.routeManifest.missingHtmlRoutes.length === 0 && discovery.routeManifest.missingMarkdownRoutes.length === 0 && discovery.routeManifest.orphanMarkdownRoutes.length === 0 ? "pass" : "warning",
-			title: discovery.routeManifest.valid ? "The public route manifest agrees with the crawl" : "No public route manifest was available",
-			plainEnglish: discovery.routeManifest.valid ? `The manifest declared ${discovery.routeManifest.declaredHtmlRoutes.length} HTML routes and ${discovery.routeManifest.declaredMarkdownRoutes.length} Markdown routes. Missing or orphan routes are listed in the evidence.` : "A public route manifest is optional. Without one, this crawl cannot prove that no orphan Markdown files were deployed.",
+			title: discovery.routeManifest.valid ? "The public route manifest is valid" : "No public route manifest was available",
+			plainEnglish: discovery.routeManifest.valid ? `The manifest declared ${discovery.routeManifest.declaredHtmlRoutes.length} HTML routes and ${discovery.routeManifest.declaredMarkdownRoutes.length} Markdown routes. This bounded crawl did not treat unvisited routes as missing.` : "A public route manifest is optional. Without one, this crawl cannot prove that no orphan Markdown files were deployed.",
 			action: "Generate one deterministic route inventory from the build when exact HTML and Markdown parity matters.",
 			evidence: discovery.routeManifest
 		})
@@ -8537,25 +8734,42 @@ function accessChecks(report, discovery) {
 	const blocked = bots.filter((bot) => bot.allowed === false);
 	const robotsKnown = report.ai?.robotsTxt?.availability === "available" || report.ai?.robotsTxt?.availability === "absent";
 	const protocol = discovery.protocolVariants;
-	return [check$1("access", {
-		id: "crawler-access",
-		status: !robotsKnown ? "unknown" : blocked.length > 0 ? "warning" : "pass",
-		title: blocked.length ? "robots.txt blocks selected crawler tokens" : "No selected crawler token is blocked at the start URL",
-		plainEnglish: robotsKnown ? `${blocked.length} of ${bots.length} selected crawler tokens were blocked by the start URL robots.txt policy.` : "The robots.txt response was not stable enough to make a crawler-access claim.",
-		action: "Keep blocks intentional. A crawler token being allowed here does not prove that the service will crawl, index, or cite the site.",
-		evidence: {
-			availability: report.ai?.robotsTxt?.availability,
-			blocked,
-			botAccess: bots
-		}
-	}), check$1("access", {
-		id: "protocol-canonicalization",
-		status: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "pass" : protocol.http.permanentRedirectToHttps === null ? "unknown" : "warning",
-		title: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "HTTP redirects permanently to HTTPS and HTTPS sends HSTS" : "Protocol canonicalization needs attention or could not be proven",
-		plainEnglish: `The HTTP variant ${protocol.http.permanentRedirectToHttps === true ? "redirected permanently to HTTPS" : "did not produce a confirmed permanent HTTPS redirect"}. The HTTPS start page ${protocol.hstsOnStartPage ? "sent HSTS" : "did not provide confirmed HSTS evidence"}.`,
-		action: "Use one permanent HTTP-to-HTTPS redirect and send HSTS only after HTTPS is stable across the site.",
-		evidence: protocol
-	})];
+	const authenticationGates = report.issues.filter((issue) => issue.ruleId === "soft_authentication_gate");
+	const clientRenderedPages = report.issues.filter((issue) => issue.ruleId === "client_rendered_content");
+	return [
+		check$1("access", {
+			id: "crawler-access",
+			status: !robotsKnown ? "unknown" : blocked.length > 0 ? "warning" : "pass",
+			title: blocked.length ? "robots.txt blocks selected crawler tokens" : "No selected crawler token is blocked at the start URL",
+			plainEnglish: robotsKnown ? `${blocked.length} of ${bots.length} selected crawler tokens were blocked by the start URL robots.txt policy.` : "The robots.txt response was not stable enough to make a crawler-access claim.",
+			action: "Keep blocks intentional. A crawler token being allowed here does not prove that the service will crawl, index, or cite the site.",
+			evidence: {
+				availability: report.ai?.robotsTxt?.availability,
+				blocked,
+				botAccess: bots
+			}
+		}),
+		check$1("access", {
+			id: "public-content-access",
+			status: authenticationGates.length > 0 || clientRenderedPages.length > 0 ? "warning" : "pass",
+			title: authenticationGates.length > 0 || clientRenderedPages.length > 0 ? "Some public content may be difficult for constrained clients to reach" : "No login gate or empty initial document was detected",
+			plainEnglish: `${authenticationGates.length} crawled pages looked like login gates and ${clientRenderedPages.length} relied on JavaScript to replace an empty or near-empty initial document. These are general site-health observations reused here because they affect agents that do not maintain sessions or run JavaScript.`,
+			action: authenticationGates.length > 0 || clientRenderedPages.length > 0 ? "Make intended public content available without a session and include its important document content in the initial HTML or a complete server-generated representation." : "Keep important public content available without a session or client-side rendering requirement.",
+			evidence: {
+				authenticationGates: authenticationGates.length,
+				clientRenderedPages: clientRenderedPages.length
+			},
+			urls: [...authenticationGates, ...clientRenderedPages].map((issue) => issue.url).slice(0, 25)
+		}),
+		check$1("access", {
+			id: "protocol-canonicalization",
+			status: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "pass" : protocol.http.permanentRedirectToHttps === null ? "unknown" : "warning",
+			title: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "HTTP redirects permanently to HTTPS and HTTPS sends HSTS" : "Protocol canonicalization needs attention or could not be proven",
+			plainEnglish: `The HTTP variant ${protocol.http.permanentRedirectToHttps === true ? "redirected permanently to HTTPS" : "did not produce a confirmed permanent HTTPS redirect"}. The HTTPS start page ${protocol.hstsOnStartPage ? "sent HSTS" : "did not provide confirmed HSTS evidence"}.`,
+			action: "Use one permanent HTTP-to-HTTPS redirect and send HSTS only after HTTPS is stable across the site.",
+			evidence: protocol
+		})
+	];
 }
 function identityChecks(report) {
 	const types = new Set(report.pages.flatMap((page) => page.schemaTypes ?? []));
@@ -10088,6 +10302,48 @@ const RULE_DEFINITIONS = [
 			suggestedCommands: ["seo crawl --sitemap-url <url> --health --json"]
 		}
 	},
+	{
+		id: "soft_authentication_gate",
+		title: "Page appears to be an authentication gate",
+		category: "response",
+		defaultSeverity: "medium",
+		whyItMatters: "The URL returned a successful HTML response, but the visible document looks like a login gate rather than the intended public page. Users and crawlers may receive a 200 response without the content they expected.",
+		howToFix: "Confirm whether authentication is intentional. Public URLs should return their public content without a login form. Protected URLs should use a clear 401 or 403 response where appropriate and should not appear in public sitemaps or internal crawl paths.",
+		impactIfIgnored: "A login page can be mistaken for the intended content, producing misleading crawl results and weak search or agent access.",
+		howToVerify: "Fetch the affected URL without a session and confirm it returns the intended public page, or a deliberate authentication status for protected content.",
+		agentHints: {
+			evidenceFields: ["page.softAuthenticationGate", "page.finalUrl"],
+			suggestedCommands: ["seo crawl <url> --max-pages 1 --json"]
+		}
+	},
+	{
+		id: "client_rendered_content",
+		title: "Important content requires JavaScript rendering",
+		category: "response",
+		defaultSeverity: "medium",
+		whyItMatters: "The raw HTML contained very little content and the useful document appeared only after JavaScript rendering. Some crawlers, agents, previews, and constrained clients do not run the page JavaScript.",
+		howToFix: "Render the important page content into the initial HTML, or publish a complete server-generated representation such as negotiated Markdown. Keep navigation, headings, canonicals, and indexability signals available without client execution.",
+		impactIfIgnored: "Clients that do not execute JavaScript may see an empty shell or miss important content and links.",
+		howToVerify: "Re-run the crawl and confirm the raw and rendered content observations contain the same important document content.",
+		agentHints: {
+			evidenceFields: ["page.fetchDiagnostics.rendering.documentDifference", "page.usedJs"],
+			suggestedCommands: ["seo crawl <url> --json"]
+		}
+	},
+	{
+		id: "soft_404",
+		title: "Unknown URLs return a successful response",
+		category: "response",
+		defaultSeverity: "high",
+		whyItMatters: "Known-nonexistent URLs returned a successful response or redirected to another successful page. Search engines and users cannot reliably distinguish missing content from a real page.",
+		howToFix: "Return 404 or 410 for missing URLs. If a replacement exists, redirect only the specific retired URL to its closest relevant replacement rather than sending every unknown path to a generic page.",
+		impactIfIgnored: "Missing URLs can remain in crawl queues, waste crawl attention, and create confusing search or monitoring evidence.",
+		howToVerify: "Request several known-nonexistent paths and confirm each returns 404 or 410 without redirecting to a generic successful page.",
+		agentHints: {
+			evidenceFields: ["report.siteChecks.soft404"],
+			suggestedCommands: ["seo crawl <url> --json"]
+		}
+	},
 	{
 		id: "client_error",
 		title: "Client error",
@@ -10121,7 +10377,8 @@ const RULE_DEFINITIONS = [
 			evidenceFields: [
 				"page.url",
 				"page.finalUrl",
-				"page.status"
+				"page.status",
+				"page.metaRefresh"
 			],
 			suggestedCommands: ["seo redirect-trace --url <url> --json"]
 		}
@@ -10577,6 +10834,8 @@ const RULE_RECOMMENDATIONS = {
 	x_robots_noindex: "review",
 	robots_blocked: "review",
 	crawler_access_blocked: "review",
+	soft_authentication_gate: "review",
+	client_rendered_content: "review",
 	orphan_page: "review",
 	redirected_url: "review",
 	slow_response: "review",
@@ -10616,6 +10875,7 @@ function comparableUrl(value) {
 	}
 }
 function isRedirectedPage(page) {
+	if (page.metaRefresh) return true;
 	const requested = comparableUrl(page.url);
 	const final = comparableUrl(page.finalUrl);
 	return Boolean(requested && final && requested !== final);
@@ -10670,6 +10930,28 @@ const WEAK_VALUABLE_INLINKS = 1;
 const QUERY_COVERAGE_MIN_IMPRESSIONS = 50;
 const QUERY_COVERAGE_MIN = .6;
 const OVERSIZED_IMAGE_CANDIDATE_LIMIT = 2e3;
+const EMPTY_INITIAL_HTML_CHARACTERS = 200;
+const SUBSTANTIVE_RENDERED_CHARACTERS = 500;
+function clientRenderedContentEvidence(page) {
+	const rendering = page.fetchDiagnostics?.rendering;
+	const difference = rendering?.documentDifference;
+	if (rendering?.status !== "rendered" || !difference?.changed.includes("content")) return;
+	const rawCharacters = difference.raw.content.characters;
+	const renderedCharacters = difference.rendered.content.characters;
+	if (rawCharacters >= EMPTY_INITIAL_HTML_CHARACTERS || renderedCharacters < SUBSTANTIVE_RENDERED_CHARACTERS || renderedCharacters < rawCharacters * 3) return;
+	return {
+		rawCharacters,
+		rawWords: difference.raw.content.wordCount,
+		renderedCharacters,
+		renderedWords: difference.rendered.content.wordCount,
+		changed: difference.changed,
+		thresholds: {
+			emptyInitialHtmlCharacters: EMPTY_INITIAL_HTML_CHARACTERS,
+			substantiveRenderedCharacters: SUBSTANTIVE_RENDERED_CHARACTERS,
+			minimumGrowthRatio: 3
+		}
+	};
+}
 function issue(ruleId, page, detail, evidence) {
 	const rule = explainRule(ruleId);
 	if (!rule) throw new Error(`Missing rule guidance for ${ruleId}`);
@@ -10868,11 +11150,18 @@ function auditCrawlPages(pages, opts = {}) {
 		}
 		if (page.status < 200) continue;
 		const redirected = isRedirectedPage(page);
-		if (redirected) issues.push(issue("redirected_url", page, `Final URL: ${page.finalUrl}`, {
-			requestedUrl: page.url,
-			finalUrl: page.finalUrl,
-			status: page.status
-		}));
+		if (redirected) {
+			const redirectTarget = page.metaRefresh?.targetUrl ?? page.finalUrl;
+			issues.push(issue("redirected_url", page, `Final URL: ${redirectTarget}`, {
+				requestedUrl: page.url,
+				finalUrl: redirectTarget,
+				status: page.status,
+				...page.metaRefresh ? {
+					redirectType: "meta-refresh",
+					delaySeconds: page.metaRefresh.delaySeconds
+				} : {}
+			}));
+		}
 		const redirectChain = page.fetchDiagnostics?.redirectChain ?? [];
 		if (redirectChain.length > 1) issues.push(issue("redirect_chain", page, `${redirectChain.length} hops`, {
 			redirectChain,
@@ -10884,6 +11173,9 @@ function auditCrawlPages(pages, opts = {}) {
 		}));
 		if (page.auditScope === "status") continue;
 		if (redirected) continue;
+		if (page.softAuthenticationGate) issues.push(issue("soft_authentication_gate", page, "A password form and login-specific page signals were observed.", page.softAuthenticationGate));
+		const renderedContent = clientRenderedContentEvidence(page);
+		if (renderedContent) issues.push(issue("client_rendered_content", page, "The raw response was an empty or near-empty shell.", renderedContent));
 		if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(issue("large_html", page, `${Math.round((page.sizeBytes ?? 0) / 1024)} KB`, {
 			sizeBytes: page.sizeBytes,
 			thresholdBytes: LARGE_HTML_BYTES
@@ -11489,7 +11781,7 @@ function createCrawlReport(input) {
 	const requests = sortRequests(sanitizeTenantValue(input.requests ?? []));
 	const requestEvidenceStatus = input.requestEvidenceStatus ?? (input.requests ? "available" : "unavailable");
 	if (requestEvidenceStatus === "unavailable" && requests.length > 0) throw new Error("Unavailable request evidence cannot include request observations.");
-	const safeIssues = sanitizeIssues(input.issues ?? [...auditCrawlRequests(requests), ...auditCrawlPages(safePagesWithLinks, { startUrl: config.url })]);
+	const safeIssues = sanitizeIssues([...input.issues ?? [...auditCrawlRequests(requests), ...auditCrawlPages(safePagesWithLinks, { startUrl: config.url })], ...input.additionalIssues ?? []]);
 	const pages = safePagesWithLinks;
 	return {
 		id: input.id ?? crawlRunId(),
@@ -11518,6 +11810,7 @@ function createCrawlReport(input) {
 		...input.ai ? { ai: sanitizeTenantValue(input.ai) } : {},
 		...input.sitemapDiscovery ? { sitemapDiscovery: sanitizeTenantValue(input.sitemapDiscovery) } : {},
 		...input.externalLinkVerification ? { externalLinkVerification: sanitizeTenantValue(input.externalLinkVerification) } : {},
+		...input.siteChecks ? { siteChecks: sanitizeTenantValue(input.siteChecks) } : {},
 		...input.agentDiscovery ? { agentDiscovery: sanitizeTenantValue(input.agentDiscovery) } : {},
 		warnings: sanitizeMessages(input.warnings).sort(compareText$13),
 		caveats: sanitizeMessages(input.caveats).sort(compareText$13)
@@ -11724,7 +12017,7 @@ function latestCrawlReport(site, store = crawlReportStore) {
 //#region packages/core/src/analyze/crawler/crawl-caveats.ts
 function crawlCaveats(input) {
 	const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
-	if (input.queueSafetySkippedUrls > 0) caveats.push(`Left ${input.queueSafetySkippedUrls} eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
+	if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
 	if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
 	if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
 	return caveats;
@@ -12087,6 +12380,75 @@ function observationFromPage(requestedUrl, page) {
 	};
 }
 //#endregion
+//#region packages/core/src/analyze/crawler/site-checks.ts
+const PROBE_COUNT = 2;
+function probeUrls(startUrl) {
+	const origin = new URL(startUrl).origin;
+	const seed = createHash("sha256").update(origin).digest("hex").slice(0, 12);
+	return Array.from({ length: PROBE_COUNT }, (_, index) => new URL(`/.well-known/seo-audit/not-found-${seed}-${index + 1}`, origin).toString());
+}
+function probeObservation(url, result) {
+	const request = result.request;
+	if (!request) return {
+		url,
+		error: result.warning ?? "No response evidence."
+	};
+	if (request.outcome === "response") return {
+		url,
+		status: request.status,
+		finalUrl: request.finalUrl,
+		redirected: request.finalUrl !== url,
+		accessBlocked: Boolean(request.accessBlock)
+	};
+	return {
+		url,
+		error: request.outcome === "failure" ? request.error : `Probe skipped: ${request.reason}`
+	};
+}
+async function collectCrawlSiteChecks(input) {
+	const probes = [];
+	for (const url of probeUrls(input.startUrl)) {
+		if (input.signal?.aborted) break;
+		const result = await input.fetchStatusPage(url, {
+			js: "off",
+			refresh: true,
+			timeoutMs: input.timeoutMs,
+			signal: input.signal,
+			writeCache: false,
+			respectRobots: input.respectRobots
+		});
+		probes.push(probeObservation(url, result));
+	}
+	const conclusive = probes.filter((probe) => probe.status !== void 0 && !probe.accessBlocked);
+	const successful = conclusive.filter((probe) => (probe.status ?? 0) >= 200 && (probe.status ?? 0) < 400);
+	const correctMissing = conclusive.filter((probe) => probe.status === 404 || probe.status === 410);
+	const status = conclusive.length !== PROBE_COUNT ? "unknown" : successful.length === PROBE_COUNT ? "warning" : correctMissing.length === PROBE_COUNT ? "pass" : "unknown";
+	const checks = { soft404: {
+		status,
+		probes,
+		probeLimit: PROBE_COUNT,
+		complete: probes.length === PROBE_COUNT
+	} };
+	if (status !== "warning") return {
+		checks,
+		issues: []
+	};
+	const rule = explainRule("soft_404");
+	if (!rule) throw new Error("Missing rule guidance for soft_404");
+	return {
+		checks,
+		issues: [{
+			ruleId: "soft_404",
+			title: rule.title,
+			category: rule.category,
+			severity: rule.defaultSeverity,
+			url: new URL("/", input.startUrl).toString(),
+			detail: `${successful.length} known-nonexistent paths returned a successful response or redirect.`,
+			evidence: checks.soft404
+		}]
+	};
+}
+//#endregion
 //#region packages/core/src/analyze/crawler/site-crawl-ai-signals.ts
 const MAX_AUXILIARY_RESPONSE_BYTES = 2 * 1024 * 1024;
 const AI_BOT_USER_AGENTS = [
@@ -12806,6 +13168,10 @@ async function crawlOne(url, opts = {}) {
 			mainContentHash: hashText(extracted.contentText),
 			textRatio: fetched.html.length ? Math.min(1, extracted.contentText.length / fetched.html.length) : void 0,
 			contentSample: truncate$1(extracted.contentText, 300),
+			contentSketch: crawlerEvidence.contentSketch,
+			softAuthenticationGate: crawlerEvidence.softAuthenticationGate,
+			metaRefresh: crawlerEvidence.metaRefresh,
+			tabbedContent: crawlerEvidence.tabbedContent,
 			lang: extracted.lang,
 			hasViewport: extracted.hasViewport,
 			isHttps: new URL(extracted.finalUrl).protocol === "https:",
@@ -12855,9 +13221,9 @@ async function crawlOne(url, opts = {}) {
 				hasQapageSchema: extracted.schemaTypes.includes("QAPage")
 			}
 		};
-		page.indexability = indexabilityReason(page);
+		page.indexability = page.metaRefresh ? `Meta refresh to ${page.metaRefresh.targetUrl}` : indexabilityReason(page);
 		page.indexable = !page.indexability;
-		page.declaredIndexability = hasMetaRobotsDirective(page.metaRobots, "noindex") || hasXRobotsDirective(page.xRobotsTag, "noindex") ? "noindex" : page.robotsTxt?.allowed === false ? "robots-blocked" : page.canonicalStatus === "conflicting" ? "canonical-conflict" : page.canonical && !sameDocumentUrl(page.canonical, page.finalUrl) ? "canonical-hint-other" : "indexable-candidate";
+		page.declaredIndexability = page.metaRefresh ? "meta-refresh" : hasMetaRobotsDirective(page.metaRobots, "noindex") || hasXRobotsDirective(page.xRobotsTag, "noindex") ? "noindex" : page.robotsTxt?.allowed === false ? "robots-blocked" : page.canonicalStatus === "conflicting" ? "canonical-conflict" : page.canonical && !sameDocumentUrl(page.canonical, page.finalUrl) ? "canonical-hint-other" : "indexable-candidate";
 		return {
 			request: responseObservation(url, fetched, "complete"),
 			page,
@@ -13651,6 +14017,9 @@ async function crawlSite(input, dependencies) {
 	let robotsDeferredUrls = 0;
 	let originBackpressureSkippedUrls = 0;
 	let queueSafetySkippedUrls = 0;
+	const queueSafetyDeferredUrls = /* @__PURE__ */ new Set();
+	const maxTrackedQueueSafetyUrls = Math.min(5e4, Math.max(1e3, config.maxPages * 5));
+	let queueSafetyInventoryCapped = false;
 	let failedUrls = 0;
 	let observedInternalLinks = 0;
 	let memoryLimited = false;
@@ -13755,13 +14124,16 @@ async function crawlSite(input, dependencies) {
 				return;
 			}
 			if (queue.size + visited.size + inFlight.size >= config.maxPages * 5) {
-				recordSkip("queue-safety-limit");
-				queueSafetySkippedUrls += 1;
-				emitStatus("url_skipped", {
-					url: normalized,
-					depth,
-					reason: "queue_safety_limit"
-				});
+				if (!queueSafetyDeferredUrls.has(normalized)) if (queueSafetyDeferredUrls.size < maxTrackedQueueSafetyUrls) {
+					queueSafetyDeferredUrls.add(normalized);
+					queueSafetySkippedUrls = queueSafetyDeferredUrls.size;
+					recordSkip("queue-safety-limit");
+					emitStatus("url_skipped", {
+						url: normalized,
+						depth,
+						reason: "queue_safety_limit"
+					});
+				} else queueSafetyInventoryCapped = true;
 				return;
 			}
 			discovered.add(normalized);
@@ -14090,6 +14462,13 @@ async function crawlSite(input, dependencies) {
 			signal,
 			concurrency: Math.min(config.concurrency, 4)
 		}) : void 0;
+		const siteCheckResult = !healthStrategy && !cancelled && !stoppedForMemory ? await collectCrawlSiteChecks({
+			startUrl: config.url,
+			timeoutMs: config.timeoutMs,
+			fetchStatusPage: deps.fetchStatusPage,
+			respectRobots: config.respectRobots,
+			signal
+		}) : void 0;
 		for (const page of pages) {
 			if (!page.geo || !llmsTxt) continue;
 			page.geo = {
@@ -14115,6 +14494,10 @@ async function crawlSite(input, dependencies) {
 			} } : {},
 			...sitemapDiscovery ? { sitemapDiscovery } : {},
 			...externalLinkVerification ? { externalLinkVerification } : {},
+			...siteCheckResult ? {
+				siteChecks: siteCheckResult.checks,
+				additionalIssues: siteCheckResult.issues
+			} : {},
 			...agentDiscovery ? { agentDiscovery } : {},
 			status: reportStatus,
 			warnings,
@@ -14123,6 +14506,7 @@ async function crawlSite(input, dependencies) {
 				pageLimitReached,
 				maxPages: config.maxPages,
 				queueSafetySkippedUrls,
+				queueSafetyInventoryCapped,
 				originBackpressureSkippedUrls,
 				memoryPressureSkippedUrls
 			}),
@@ -16407,6 +16791,8 @@ const agentRepresentationResponseSchema = z.object({
 	status: z.number().int().optional(),
 	contentType: z.string().optional(),
 	bytes: z.number().int().nonnegative().optional(),
+	characters: z.number().int().nonnegative().optional(),
+	estimatedTokens: z.number().int().nonnegative().optional(),
 	sha256: z.string().optional(),
 	canonicalUrl: z.string().url().optional(),
 	alternateUrl: z.string().url().optional(),
@@ -16464,7 +16850,15 @@ const agentDiscoverySchema = z.object({
 				sourceWordCount: z.number().int().nonnegative(),
 				wordRetentionRatio: z.number().nonnegative().nullable(),
 				introductoryCopyRetained: z.boolean().nullable(),
-				navigationOnly: z.boolean()
+				navigationOnly: z.boolean(),
+				contentSketchCoverage: z.number().min(0).max(1).nullable().optional(),
+				tabbedContent: z.object({
+					detectedPanels: z.number().int().nonnegative(),
+					evaluatedPanels: z.number().int().nonnegative(),
+					retainedPanels: z.number().int().nonnegative(),
+					missingPanels: z.number().int().nonnegative(),
+					complete: z.boolean().nullable()
+				}).optional()
 			}).optional()
 		}))
 	}),
@@ -16483,6 +16877,11 @@ const agentDiscoverySchema = z.object({
 		missingHtmlRoutes: z.array(z.string()),
 		missingMarkdownRoutes: z.array(z.string()),
 		orphanMarkdownRoutes: z.array(z.string()),
+		comparisonStatus: z.enum([
+			"complete",
+			"partial",
+			"not-applicable"
+		]).optional(),
 		error: z.string().optional()
 	}),
 	agentSkills: z.object({
@@ -16588,6 +16987,25 @@ const agentDiscoverySchema = z.object({
 	warnings: z.array(z.string())
 });
 //#endregion
+//#region packages/core/src/analyze/crawler/site-checks-schema.ts
+const crawlSiteChecksSchema = z.object({ soft404: z.object({
+	status: z.enum([
+		"pass",
+		"warning",
+		"unknown"
+	]),
+	probes: z.array(z.object({
+		url: z.string().url(),
+		status: z.number().int().optional(),
+		finalUrl: z.string().url().optional(),
+		redirected: z.boolean().optional(),
+		accessBlocked: z.boolean().optional(),
+		error: z.string().optional()
+	})),
+	probeLimit: z.number().int().positive(),
+	complete: z.boolean()
+}) });
+//#endregion
 //#region packages/core/src/analyze/crawler/schemas.ts
 const crawlerRuleSeveritySchema = z.enum([
 	"low",
@@ -16983,6 +17401,7 @@ const crawlPageSnapshotSchema = z.object({
 		"robots-blocked",
 		"canonical-conflict",
 		"canonical-hint-other",
+		"meta-refresh",
 		"not-html",
 		"unknown"
 	]).optional(),
@@ -17017,6 +17436,38 @@ const crawlPageSnapshotSchema = z.object({
 	mainContentHash: z.string().optional(),
 	textRatio: z.number().min(0).max(1).optional(),
 	contentSample: z.string().optional(),
+	contentSketch: z.object({
+		version: z.literal(1),
+		wordCount: z.number().int().nonnegative(),
+		shingleSize: z.number().int().positive(),
+		sampledShingles: z.number().int().nonnegative(),
+		hashes: z.array(z.string())
+	}).optional(),
+	softAuthenticationGate: z.object({
+		kind: z.literal("login-form"),
+		indicators: z.array(z.string()),
+		formActionPath: z.string().optional()
+	}).optional(),
+	metaRefresh: z.object({
+		targetUrl: z.string().url(),
+		delaySeconds: z.number().nonnegative()
+	}).optional(),
+	tabbedContent: z.object({
+		groups: z.number().int().positive(),
+		panels: z.number().int().positive(),
+		retainedPanels: z.number().int().nonnegative(),
+		truncated: z.boolean(),
+		panelSketches: z.array(z.object({
+			label: z.string().optional(),
+			sketch: z.object({
+				version: z.literal(1),
+				wordCount: z.number().int().nonnegative(),
+				shingleSize: z.number().int().positive(),
+				sampledShingles: z.number().int().nonnegative(),
+				hashes: z.array(z.string())
+			})
+		}))
+	}).optional(),
 	lang: z.string().optional(),
 	hasViewport: z.boolean().optional(),
 	isHttps: z.boolean().optional(),
@@ -17508,6 +17959,7 @@ const crawlReportBaseSchema = z.object({
 	ai: crawlAiSignalsSchema.optional(),
 	sitemapDiscovery: crawlSitemapDiscoverySchema.optional(),
 	externalLinkVerification: crawlExternalLinkVerificationSchema.optional(),
+	siteChecks: crawlSiteChecksSchema.optional(),
 	agentDiscovery: agentDiscoverySchema.optional(),
 	warnings: z.array(z.string()),
 	caveats: z.array(z.string())