seo 0.2.13 → 0.2.14

package/dist/mcp.js

@@ -1,5 +1,5 @@
 import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
-import { n as load, t as version } from "./package-B_swz2Xd.js";
+import { n as load, t as version } from "./package-Cp8DwqCy.js";
 import { Headers, fetch as fetch$1 } from "undici";
 import { chmodSync, createReadStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { arch, homedir, platform, totalmem } from "node:os";
@@ -2778,6 +2778,22 @@ function extractStructuredData($, baseUrl, options = {}) {
 }
 //#endregion
 //#region packages/core/dist/extract/page-extractor.js
+function metaRefresh($, finalUrl) {
+	for (const element of $("meta[http-equiv]").toArray()) {
+		if ($(element).attr("http-equiv")?.trim().toLowerCase() !== "refresh") continue;
+		const match = ($(element).attr("content")?.trim())?.match(/^([0-9]+(?:\.[0-9]+)?)\s*;\s*url\s*=\s*(?:"([^"]*)"|'([^']*)'|(.+))$/iu);
+		if (!match) continue;
+		try {
+			const target = new URL((match[2] ?? match[3] ?? match[4] ?? "").trim(), finalUrl);
+			if (!["http:", "https:"].includes(target.protocol)) continue;
+			target.hash = "";
+			return {
+				targetUrl: target.toString(),
+				delaySeconds: Number(match[1])
+			};
+		} catch {}
+	}
+}
 const SOCIAL_PROFILE_HOSTS = [
 	"facebook.com",
 	"instagram.com",
@@ -3057,6 +3073,8 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
 	}, extractor, dependencies);
 	const { text, excerpt } = content;
 	const title = safeText$1($("title").first().text());
+	const crawlerMetaRefresh = extractor === "crawler" ? metaRefresh($, fetchResult.finalUrl) : void 0;
+	if (crawlerMetaRefresh && new URL(crawlerMetaRefresh.targetUrl).origin === url.origin) crawlerInternalLinks.add(crawlerMetaRefresh.targetUrl);
 	return {
 		url: fetchResult.url,
 		finalUrl: fetchResult.finalUrl,
@@ -3082,6 +3100,7 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
 			externalAnchorSamples,
 			socialProfileLinks: [...socialProfileLinks],
 			contentSketch: contentSketch(text),
+			metaRefresh: crawlerMetaRefresh,
 			softAuthenticationGate: softAuthenticationGate($, fetchResult.finalUrl, title),
 			tabbedContent: tabbedContent($)
 		} } : {},
@@ -7763,6 +7782,9 @@ function failedMarkdownUrls(discovery) {
 		return !negotiatedOk || !explicitOk || page.repeatedHashStable !== true;
 	}).map((page) => page.htmlUrl);
 }
+function unstableMarkdownUrls(discovery) {
+	return discovery.markdownAlternates.pages.filter((page) => confirmedMarkdown(page) !== void 0 && page.repeatedHashStable !== true).map((page) => page.htmlUrl);
+}
 function qualityFailureUrls(discovery) {
 	return discovery.markdownAlternates.pages.filter((page) => {
 		const quality = page.quality;
@@ -7776,7 +7798,10 @@ function confirmedMarkdown(page) {
 function markdownChecks(discovery) {
 	const markdown = discovery.markdownAlternates;
 	const failedUrls = failedMarkdownUrls(discovery);
+	const unstableUrls = unstableMarkdownUrls(discovery);
 	const qualityUrls = qualityFailureUrls(discovery);
+	const hasEligibleHtml = markdown.eligibleHtmlPages > 0;
+	const hasEvaluatedMarkdown = markdown.evaluatedPages > 0;
 	const negotiatedPages = markdown.pages.filter((page) => page.negotiated?.status === 200 && /^\s*text\/markdown\b/iu.test(page.negotiated.contentType ?? "")).length;
 	const explicitPages = markdown.pages.filter((page) => page.htmlAlternateUnique).length;
 	const coverageComplete = markdown.eligibleHtmlPages > 0 && markdown.evaluatedPages === markdown.eligibleHtmlPages;
@@ -7805,25 +7830,28 @@ function markdownChecks(discovery) {
 		check$2("representations", {
 			id: "markdown-coverage",
 			status: markdown.eligibleHtmlPages === 0 ? "unknown" : coverageComplete ? "pass" : "fail",
-			title: coverageComplete ? "Every successful HTML page has a Markdown representation" : "Markdown representation coverage is incomplete",
-			plainEnglish: `${markdown.evaluatedPages} of ${markdown.eligibleHtmlPages} successful HTML pages returned Markdown. ${explicitPages} advertised an explicit alternative and ${negotiatedPages} supported content negotiation.`,
-			action: coverageComplete ? "Keep the negotiated or explicit Markdown response stable as routes change." : "Return text/markdown through content negotiation or advertise one working Markdown alternative for each public HTML page.",
+			title: coverageComplete ? "Every successful HTML page has a Markdown representation" : !hasEligibleHtml ? "Markdown representation coverage was not evaluated" : "Markdown representation coverage is incomplete",
+			plainEnglish: hasEligibleHtml ? `${markdown.evaluatedPages} of ${markdown.eligibleHtmlPages} successful HTML pages returned Markdown. ${explicitPages} advertised an explicit alternative and ${negotiatedPages} supported content negotiation.` : "No successful HTML page was available, so Markdown representation coverage could not be evaluated.",
+			action: !hasEligibleHtml ? "Resolve the crawl or page-fetch failure, then run this check again." : coverageComplete ? "Keep the negotiated or explicit Markdown response stable as routes change." : "Return text/markdown through content negotiation or advertise one working Markdown alternative for each public HTML page.",
 			evidence: {
 				eligibleHtmlPages: markdown.eligibleHtmlPages,
 				advertisedPages: markdown.advertisedPages,
 				negotiatedPages,
 				evaluatedPages: markdown.evaluatedPages
 			},
-			urls: failedUrls.slice(0, 25)
+			urls: hasEligibleHtml ? failedUrls.slice(0, 25) : []
 		}),
 		check$2("representations", {
 			id: "markdown-token-estimates",
 			status: markdown.evaluatedPages === 0 ? "unknown" : tokenHeaders === markdown.evaluatedPages ? "pass" : "info",
-			title: tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes a token estimate" : "Some Markdown responses are missing a stable token estimate",
-			plainEnglish: `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned X-Markdown-Tokens. Paired explicit and negotiated responses agreed where both existed. The value is an estimate, not an exact model-specific token count.`,
-			action: tokenHeaders === markdown.evaluatedPages ? "Keep the estimate stable when both explicit and negotiated responses exist." : "No change is required. X-Markdown-Tokens is optional; local size estimates are reported separately.",
+			title: !hasEvaluatedMarkdown ? "Markdown token estimates were not evaluated" : tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes a token estimate" : "Some Markdown responses are missing a stable token estimate",
+			plainEnglish: hasEvaluatedMarkdown ? `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned X-Markdown-Tokens. Paired explicit and negotiated responses agreed where both existed. The value is an estimate, not an exact model-specific token count.` : "No confirmed Markdown response was available, so optional server token estimates could not be evaluated.",
+			action: !hasEvaluatedMarkdown ? "Confirm a successful text/markdown representation, then run this check again." : tokenHeaders === markdown.evaluatedPages ? "Keep the estimate stable when both explicit and negotiated responses exist." : "No change is required. X-Markdown-Tokens is optional; local size estimates are reported separately.",
 			urls: markdown.pages.filter((page) => {
-				return confirmedMarkdown(page)?.markdownTokens === void 0 || page.explicit !== void 0 && page.negotiated !== void 0 && page.explicit.markdownTokens !== page.negotiated.markdownTokens;
+				const primary = confirmedMarkdown(page);
+				if (!primary) return false;
+				const paired = [page.explicit, page.negotiated].filter((response) => response?.status !== void 0 && response.status >= 200 && response.status < 300 && /^\s*text\/markdown\b/iu.test(response.contentType ?? ""));
+				return primary?.markdownTokens === void 0 || paired.length === 2 && paired[0]?.markdownTokens !== paired[1]?.markdownTokens;
 			}).map((page) => page.htmlUrl).slice(0, 25)
 		}),
 		check$2("representations", {
@@ -7831,7 +7859,7 @@ function markdownChecks(discovery) {
 			status: sizedPages.length === 0 ? "unknown" : largePages.length === 0 ? "pass" : "warning",
 			title: sizedPages.length === 0 ? "Markdown size was not measured" : largePages.length === 0 ? "Markdown representations stay within the review threshold" : "Some Markdown representations are large enough to review",
 			plainEnglish: `${sizedPages.length} Markdown responses were measured locally. ${largePages.length} exceeded 50,000 characters and ${veryLargePages.length} exceeded 100,000 characters. These are processing-risk thresholds, not universal model limits.`,
-			action: largePages.length === 0 ? "Keep important content early and retain stable section structure as pages grow." : "Split genuinely separate topics, keep navigation and repeated chrome out of Markdown, and put the most useful content early. Do not remove necessary detail just to hit a threshold.",
+			action: sizedPages.length === 0 ? "Confirm a successful text/markdown representation, then run this check again." : largePages.length === 0 ? "Keep important content early and retain stable section structure as pages grow." : "Split genuinely separate topics, keep navigation and repeated chrome out of Markdown, and put the most useful content early. Do not remove necessary detail just to hit a threshold.",
 			evidence: {
 				measuredPages: sizedPages.length,
 				reviewThresholdCharacters: 5e4,
@@ -7849,7 +7877,7 @@ function markdownChecks(discovery) {
 			status: parityPages.length === 0 ? "unknown" : parityFailures.length === 0 ? "pass" : "warning",
 			title: parityPages.length === 0 ? "HTML and Markdown content parity was not measured" : parityFailures.length === 0 ? "Markdown retains sampled content from the HTML document" : "Some Markdown responses may omit important HTML content",
 			plainEnglish: `${parityPages.length} pages were compared using bounded hashes sampled across the main HTML content. ${parityFailures.length} retained less than 60% of that sample. This detects likely omissions without storing full page bodies.`,
-			action: parityFailures.length === 0 ? "Keep the shared conversion path covered as templates and components change." : "Compare the affected HTML and Markdown around the missing sections, then fix the shared converter or component serialization.",
+			action: parityPages.length === 0 ? "Confirm a successful text/markdown representation, then run this check again." : parityFailures.length === 0 ? "Keep the shared conversion path covered as templates and components change." : "Compare the affected HTML and Markdown around the missing sections, then fix the shared converter or component serialization.",
 			evidence: {
 				evaluatedPages: parityPages.length,
 				failedPages: parityFailures.length,
@@ -7874,32 +7902,32 @@ function markdownChecks(discovery) {
 		check$2("representations", {
 			id: "markdown-negotiation",
 			status: markdown.eligibleHtmlPages === 0 ? "unknown" : negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "pass" : "fail",
-			title: negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "Markdown content negotiation works across the site" : "Markdown content negotiation needs attention",
-			plainEnglish: `${negotiatedPages} of ${markdown.eligibleHtmlPages} pages returned negotiated Markdown. ${markdown.exactByteMatches} of ${explicitPages} paired explicit responses matched byte for byte. A request that refuses Markdown ${discovery.contentNegotiation.qZeroHonoured ? "received HTML" : "did not produce confirmed HTML evidence"}.`,
-			action: "Honour Accept q-values, send Vary: Accept, and keep paired explicit and negotiated responses identical when both exist.",
+			title: !hasEligibleHtml ? "Markdown content negotiation was not evaluated" : negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "Markdown content negotiation works across the site" : "Markdown content negotiation needs attention",
+			plainEnglish: hasEligibleHtml ? `${negotiatedPages} of ${markdown.eligibleHtmlPages} pages returned negotiated Markdown. ${markdown.exactByteMatches} of ${explicitPages} paired explicit responses matched byte for byte. A request that refuses Markdown ${discovery.contentNegotiation.qZeroHonoured ? "received HTML" : "did not produce confirmed HTML evidence"}.` : "No successful HTML page was available, so content negotiation could not be evaluated.",
+			action: hasEligibleHtml ? "Honour Accept q-values, send Vary: Accept, and keep paired explicit and negotiated responses identical when both exist." : "Resolve the crawl or page-fetch failure, then run this check again.",
 			evidence: {
 				negotiatedPages,
 				explicitPages,
 				exactByteMatches: markdown.exactByteMatches,
 				qZero: discovery.contentNegotiation
 			},
-			urls: failedUrls.slice(0, 25)
+			urls: hasEligibleHtml ? failedUrls.slice(0, 25) : []
 		}),
 		check$2("representations", {
 			id: "markdown-determinism",
 			status: markdown.evaluatedPages === 0 ? "unknown" : stable ? "pass" : "fail",
-			title: stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
-			plainEnglish: `${markdown.stableResponses} of ${markdown.evaluatedPages} repeated Markdown responses kept the same SHA-256 digest.`,
-			action: "Remove timestamps, random output, or other runtime rewriting that changes the same Markdown response between requests.",
+			title: !hasEvaluatedMarkdown ? "Markdown response stability was not evaluated" : stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
+			plainEnglish: hasEvaluatedMarkdown ? `${markdown.stableResponses} of ${markdown.evaluatedPages} repeated Markdown responses kept the same SHA-256 digest.` : "No confirmed Markdown response was available for a repeated stability check.",
+			action: !hasEvaluatedMarkdown ? "Confirm a successful text/markdown representation, then run this check again." : stable ? "Keep repeated Markdown responses byte-stable as templates and data change." : "Remove timestamps, random output, or other runtime rewriting that changes the same Markdown response between requests.",
 			evidence: { stableResponses: markdown.stableResponses },
-			urls: failedUrls.slice(0, 25)
+			urls: unstableUrls.slice(0, 25)
 		}),
 		check$2("representations", {
 			id: "markdown-quality",
 			status: markdown.evaluatedPages === 0 ? "unknown" : qualityUrls.length === 0 ? "pass" : "warning",
-			title: qualityUrls.length === 0 ? "Markdown keeps the useful document structure cleanly" : "Some Markdown alternatives need an extraction review",
-			plainEnglish: qualityUrls.length === 0 ? "The evaluated alternatives retained one H1, frontmatter titles, balanced code fences, useful copy, and no leaked SVG, script, style, or layout tags." : `${qualityUrls.length} Markdown alternative${qualityUrls.length === 1 ? "" : "s"} lost important structure, looked navigation-only, or leaked presentation markup.`,
-			action: "Open the affected Markdown directly and fix the shared converter or component exclusion rules rather than patching generated files.",
+			title: !hasEvaluatedMarkdown ? "Markdown quality was not evaluated" : qualityUrls.length === 0 ? "Markdown keeps the useful document structure cleanly" : "Some Markdown alternatives need an extraction review",
+			plainEnglish: !hasEvaluatedMarkdown ? "No confirmed Markdown response was available for structure and extraction checks." : qualityUrls.length === 0 ? "The evaluated alternatives retained one H1, frontmatter titles, balanced code fences, useful copy, and no leaked SVG, script, style, or layout tags." : `${qualityUrls.length} Markdown alternative${qualityUrls.length === 1 ? "" : "s"} lost important structure, looked navigation-only, or leaked presentation markup.`,
+			action: !hasEvaluatedMarkdown ? "Confirm a successful text/markdown representation, then run this check again." : qualityUrls.length === 0 ? "Keep the shared Markdown conversion path covered as templates and components change." : "Open the affected Markdown directly and fix the shared converter or component exclusion rules rather than patching generated files.",
 			urls: qualityUrls.slice(0, 25)
 		})
 	];
@@ -9819,7 +9847,8 @@ const RULE_DEFINITIONS = [
 			evidenceFields: [
 				"page.url",
 				"page.finalUrl",
-				"page.status"
+				"page.status",
+				"page.metaRefresh"
 			],
 			suggestedCommands: ["seo redirect-trace --url <url> --json"]
 		}
@@ -10313,6 +10342,7 @@ function comparableUrl(value) {
 	}
 }
 function isRedirectedPage(page) {
+	if (page.metaRefresh) return true;
 	const requested = comparableUrl(page.url);
 	const final = comparableUrl(page.finalUrl);
 	return Boolean(requested && final && requested !== final);
@@ -10587,11 +10617,18 @@ function auditCrawlPages(pages, opts = {}) {
 		}
 		if (page.status < 200) continue;
 		const redirected = isRedirectedPage(page);
-		if (redirected) issues.push(issue("redirected_url", page, `Final URL: ${page.finalUrl}`, {
-			requestedUrl: page.url,
-			finalUrl: page.finalUrl,
-			status: page.status
-		}));
+		if (redirected) {
+			const redirectTarget = page.metaRefresh?.targetUrl ?? page.finalUrl;
+			issues.push(issue("redirected_url", page, `Final URL: ${redirectTarget}`, {
+				requestedUrl: page.url,
+				finalUrl: redirectTarget,
+				status: page.status,
+				...page.metaRefresh ? {
+					redirectType: "meta-refresh",
+					delaySeconds: page.metaRefresh.delaySeconds
+				} : {}
+			}));
+		}
 		const redirectChain = page.fetchDiagnostics?.redirectChain ?? [];
 		if (redirectChain.length > 1) issues.push(issue("redirect_chain", page, `${redirectChain.length} hops`, {
 			redirectChain,
@@ -11437,7 +11474,7 @@ function latestCrawlReport(site, store = crawlReportStore) {
 //#region packages/core/dist/analyze/crawler/crawl-caveats.js
 function crawlCaveats(input) {
 	const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
-	if (input.queueSafetySkippedUrls > 0) caveats.push(`Left ${input.queueSafetySkippedUrls} eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
+	if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
 	if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
 	if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
 	return caveats;
@@ -12590,6 +12627,7 @@ async function crawlOne(url, opts = {}) {
 			contentSample: truncate$1(extracted.contentText, 300),
 			contentSketch: crawlerEvidence.contentSketch,
 			softAuthenticationGate: crawlerEvidence.softAuthenticationGate,
+			metaRefresh: crawlerEvidence.metaRefresh,
 			tabbedContent: crawlerEvidence.tabbedContent,
 			lang: extracted.lang,
 			hasViewport: extracted.hasViewport,
@@ -12640,9 +12678,9 @@ async function crawlOne(url, opts = {}) {
 				hasQapageSchema: extracted.schemaTypes.includes("QAPage")
 			}
 		};
-		page.indexability = indexabilityReason(page);
+		page.indexability = page.metaRefresh ? `Meta refresh to ${page.metaRefresh.targetUrl}` : indexabilityReason(page);
 		page.indexable = !page.indexability;
-		page.declaredIndexability = hasMetaRobotsDirective(page.metaRobots, "noindex") || hasXRobotsDirective(page.xRobotsTag, "noindex") ? "noindex" : page.robotsTxt?.allowed === false ? "robots-blocked" : page.canonicalStatus === "conflicting" ? "canonical-conflict" : page.canonical && !sameDocumentUrl(page.canonical, page.finalUrl) ? "canonical-hint-other" : "indexable-candidate";
+		page.declaredIndexability = page.metaRefresh ? "meta-refresh" : hasMetaRobotsDirective(page.metaRobots, "noindex") || hasXRobotsDirective(page.xRobotsTag, "noindex") ? "noindex" : page.robotsTxt?.allowed === false ? "robots-blocked" : page.canonicalStatus === "conflicting" ? "canonical-conflict" : page.canonical && !sameDocumentUrl(page.canonical, page.finalUrl) ? "canonical-hint-other" : "indexable-candidate";
 		return {
 			request: responseObservation(url, fetched, "complete"),
 			page,
@@ -13436,6 +13474,9 @@ async function crawlSite(input, dependencies) {
 	let robotsDeferredUrls = 0;
 	let originBackpressureSkippedUrls = 0;
 	let queueSafetySkippedUrls = 0;
+	const queueSafetyDeferredUrls = /* @__PURE__ */ new Set();
+	const maxTrackedQueueSafetyUrls = Math.min(5e4, Math.max(1e3, config.maxPages * 5));
+	let queueSafetyInventoryCapped = false;
 	let failedUrls = 0;
 	let observedInternalLinks = 0;
 	let memoryLimited = false;
@@ -13540,13 +13581,16 @@ async function crawlSite(input, dependencies) {
 				return;
 			}
 			if (queue.size + visited.size + inFlight.size >= config.maxPages * 5) {
-				recordSkip("queue-safety-limit");
-				queueSafetySkippedUrls += 1;
-				emitStatus("url_skipped", {
-					url: normalized,
-					depth,
-					reason: "queue_safety_limit"
-				});
+				if (!queueSafetyDeferredUrls.has(normalized)) if (queueSafetyDeferredUrls.size < maxTrackedQueueSafetyUrls) {
+					queueSafetyDeferredUrls.add(normalized);
+					queueSafetySkippedUrls = queueSafetyDeferredUrls.size;
+					recordSkip("queue-safety-limit");
+					emitStatus("url_skipped", {
+						url: normalized,
+						depth,
+						reason: "queue_safety_limit"
+					});
+				} else queueSafetyInventoryCapped = true;
 				return;
 			}
 			discovered.add(normalized);
@@ -13919,6 +13963,7 @@ async function crawlSite(input, dependencies) {
 				pageLimitReached,
 				maxPages: config.maxPages,
 				queueSafetySkippedUrls,
+				queueSafetyInventoryCapped,
 				originBackpressureSkippedUrls,
 				memoryPressureSkippedUrls
 			}),
@@ -15401,6 +15446,7 @@ const crawlPageSnapshotSchema = z.object({
 		"robots-blocked",
 		"canonical-conflict",
 		"canonical-hint-other",
+		"meta-refresh",
 		"not-html",
 		"unknown"
 	]).optional(),
@@ -15447,6 +15493,10 @@ const crawlPageSnapshotSchema = z.object({
 		indicators: z.array(z.string()),
 		formActionPath: z.string().optional()
 	}).optional(),
+	metaRefresh: z.object({
+		targetUrl: z.string().url(),
+		delaySeconds: z.number().nonnegative()
+	}).optional(),
 	tabbedContent: z.object({
 		groups: z.number().int().positive(),
 		panels: z.number().int().positive(),