seo 0.2.9 → 0.2.10

package/dist/mcp.js

@@ -1,8 +1,8 @@
 import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
-import { n as load, t as version } from "./package-C51oI-6o.js";
+import { n as load, t as version } from "./package-92W1Zc-o.js";
 import { Headers, fetch as fetch$1 } from "undici";
 import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
-import { arch, homedir, platform } from "node:os";
+import { arch, homedir, platform, totalmem } from "node:os";
 import { dirname, join } from "node:path";
 import { z } from "zod";
 import lockfile from "proper-lockfile";
@@ -769,14 +769,14 @@ async function createGoogleAccessTokenClient() {
 }
 //#endregion
 //#region packages/core/dist/storage/cache-maintenance.js
-const MEBIBYTE$1 = 1024 * 1024;
+const MEBIBYTE$2 = 1024 * 1024;
 const CACHE_MAX_AGE_MS = 720 * 60 * 60 * 1e3;
 const CACHE_LIMITS = {
-	gsc_cache: 48 * MEBIBYTE$1,
-	ga4_cache: 24 * MEBIBYTE$1,
-	semrush_cache: 16 * MEBIBYTE$1,
-	http_cache: 160 * MEBIBYTE$1,
-	performance_reports: 8 * MEBIBYTE$1
+	gsc_cache: 48 * MEBIBYTE$2,
+	ga4_cache: 24 * MEBIBYTE$2,
+	semrush_cache: 16 * MEBIBYTE$2,
+	http_cache: 160 * MEBIBYTE$2,
+	performance_reports: 8 * MEBIBYTE$2
 };
 const CACHE_POLICIES = [
 	{
@@ -2571,7 +2571,7 @@ function isValidStructuredDate(value) {
 	const parsed = new Date(trimmed);
 	return Number.isFinite(parsed.getTime()) && parsed.toISOString().slice(0, 10) === datePart;
 }
-function parseJsonLd(blocks, baseUrl) {
+function parseJsonLd(blocks, baseUrl, retainJsonLd) {
 	const jsonLd = [];
 	const invalidJsonLdSamples = [];
 	const unrecognizedJsonLdTypes = [];
@@ -2643,7 +2643,7 @@ function parseJsonLd(blocks, baseUrl) {
 	blocks.forEach((block, index) => {
 		try {
 			const parsed = JSON.parse(block);
-			jsonLd.push(parsed);
+			if (retainJsonLd) jsonLd.push(parsed);
 			visit(parsed, { prefixes: /* @__PURE__ */ new Map() }, index, "$");
 		} catch (error) {
 			invalidJsonLdSamples.push({
@@ -2704,8 +2704,8 @@ function usableElementValue($, selector) {
 		return ($(element).attr("content") ?? $(element).attr("datetime") ?? $(element).text()).trim().length > 0;
 	});
 }
-function extractStructuredData($, baseUrl) {
-	const parsed = parseJsonLd($("script[type=\"application/ld+json\"]").toArray().map((element) => $(element).html() ?? ""), baseUrl);
+function extractStructuredData($, baseUrl, options = {}) {
+	const parsed = parseJsonLd($("script[type=\"application/ld+json\"]").toArray().map((element) => $(element).html() ?? ""), baseUrl, options.retainJsonLd ?? true);
 	const microdata = microdataTypes($);
 	const rdfa = rdfaTypes($);
 	const schemaTypes = [.../* @__PURE__ */ new Set([
@@ -2739,6 +2739,16 @@ function extractStructuredData($, baseUrl) {
 }
 //#endregion
 //#region packages/core/dist/extract/page-extractor.js
+const SOCIAL_PROFILE_HOSTS = [
+	"facebook.com",
+	"instagram.com",
+	"linkedin.com",
+	"pinterest.com",
+	"tiktok.com",
+	"twitter.com",
+	"x.com",
+	"youtube.com"
+];
 function safeText$1(value) {
 	const trimmed = value?.replace(/\s+/g, " ").trim();
 	return trimmed ? trimmed : void 0;
@@ -2811,6 +2821,28 @@ function imageDimensionsFromUrl(src) {
 		height
 	};
 }
+function addAnchorSample(samples, seen, href, text) {
+	if (samples.length >= 25 || !text) return;
+	const sampleText = text.length > 120 ? text.slice(0, 120).trimEnd() : text;
+	const key = `${href}\n${sampleText}`;
+	if (seen.has(key)) return;
+	seen.add(key);
+	samples.push({
+		href,
+		text: sampleText
+	});
+}
+function addSocialProfile(profiles, href) {
+	if (profiles.size >= 50) return;
+	try {
+		const url = new URL(href);
+		const host = url.hostname.replace(/^www\./, "").toLowerCase();
+		if (SOCIAL_PROFILE_HOSTS.some((value) => host === value || host.endsWith(`.${value}`))) {
+			url.hash = "";
+			profiles.add(url.toString());
+		}
+	} catch {}
+}
 async function extractPage(fetchResult, extractor = "defuddle", dependencies = {}) {
 	const $ = extractor === "crawler" && /<html(?:\s|>)/i.test(fetchResult.html) ? load(fetchResult.html, { xml: {
 		xmlMode: false,
@@ -2818,32 +2850,69 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
 	} }) : load(fetchResult.html);
 	const url = new URL(fetchResult.finalUrl);
 	const canonicalEvidence = extractCanonicalEvidence($, fetchResult.headers, fetchResult.finalUrl);
-	const headings = $("h1, h2, h3, h4, h5, h6").toArray().map((element) => ({
-		level: Number.parseInt(element.tagName.slice(1), 10),
-		text: $(element).text().replace(/\s+/g, " ").trim()
-	})).filter((heading) => heading.text.length > 0);
+	const headings = [];
+	let crawlerH1;
+	let crawlerH1Count = 0;
+	let crawlerH2Count = 0;
+	let crawlerH3Count = 0;
+	let questionHeadings = 0;
+	$("h1, h2, h3, h4, h5, h6").each((_index, element) => {
+		const level = Number.parseInt(element.tagName.slice(1), 10);
+		const text = $(element).text().replace(/\s+/g, " ").trim();
+		if (!text) return;
+		if (text.trimEnd().endsWith("?")) questionHeadings += 1;
+		if (level === 1) {
+			crawlerH1 ??= text;
+			crawlerH1Count += 1;
+		} else if (level === 2) crawlerH2Count += 1;
+		else if (level === 3) crawlerH3Count += 1;
+		if (extractor !== "crawler") headings.push({
+			level,
+			text
+		});
+	});
 	const links = [];
+	const crawlerInternalLinks = /* @__PURE__ */ new Set();
+	const crawlerExternalLinks = /* @__PURE__ */ new Set();
+	const internalAnchorSamples = [];
+	const externalAnchorSamples = [];
+	const internalAnchorKeys = /* @__PURE__ */ new Set();
+	const externalAnchorKeys = /* @__PURE__ */ new Set();
+	const socialProfileLinks = /* @__PURE__ */ new Set();
 	let invalidLinkCount = 0;
 	let unsupportedLinkCount = 0;
-	for (const element of $("a[href]").toArray()) {
+	$("a[href]").each((_index, element) => {
 		const href = $(element).attr("href") ?? "";
 		try {
 			const absolute = new URL(href, fetchResult.finalUrl);
 			if (!["http:", "https:"].includes(absolute.protocol)) {
 				unsupportedLinkCount += 1;
-				continue;
+				return;
 			}
-			links.push({
-				href: absolute.toString(),
-				text: $(element).text().replace(/\s+/g, " ").trim(),
+			const absoluteHref = absolute.toString();
+			const text = $(element).text().replace(/\s+/g, " ").trim();
+			const internal = absolute.host === url.host;
+			if (extractor === "crawler") {
+				if (absolute.origin === url.origin) {
+					absolute.hash = "";
+					crawlerInternalLinks.add(absolute.toString());
+				}
+				if (!internal) {
+					crawlerExternalLinks.add(absoluteHref);
+					addSocialProfile(socialProfileLinks, absoluteHref);
+				}
+				addAnchorSample(internal ? internalAnchorSamples : externalAnchorSamples, internal ? internalAnchorKeys : externalAnchorKeys, absoluteHref, text);
+			} else links.push({
+				href: absoluteHref,
+				text,
 				rel: ($(element).attr("rel") ?? "").split(/\s+/).filter(Boolean),
-				internal: absolute.host === url.host,
+				internal,
 				location: $(element).closest("nav, header").length ? "navigation" : $(element).closest("footer").length ? "footer" : $(element).closest("main, article").length ? "main-content" : "other"
 			});
 		} catch {
 			invalidLinkCount += 1;
 		}
-	}
+	});
 	const hreflang = [];
 	let invalidHreflangCount = 0;
 	for (const element of $("link[rel~=\"alternate\"][hreflang][href]").toArray()) {
@@ -2858,25 +2927,29 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
 	const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
 	const openGraph = Object.fromEntries($("meta[property^=\"og:\"]").toArray().map((element) => [$(element).attr("property") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
 	const twitter = Object.fromEntries($("meta[name^=\"twitter:\"]").toArray().map((element) => [$(element).attr("name") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
-	const structuredData = extractStructuredData($, fetchResult.finalUrl);
+	const structuredData = extractStructuredData($, fetchResult.finalUrl, { retainJsonLd: extractor !== "crawler" });
 	const hasDate = structuredData.hasDate || isValidStructuredDate($("meta[property=\"article:published_time\"]").attr("content")) || isValidStructuredDate($("meta[property=\"article:modified_time\"]").attr("content")) || $("article time[datetime], main time[datetime]").toArray().some((element) => isValidStructuredDate($(element).attr("datetime")));
 	const author = safeText$1($("meta[name=\"author\"]").attr("content"));
 	const hasAuthor = Boolean(author) || structuredData.hasAuthor || $("[rel~=author], .author, .byline").toArray().some((element) => Boolean(safeText$1($(element).text())));
 	const semanticHtml = $("main, article").length > 0;
-	const questionHeadings = headings.filter((heading) => heading.text.trimEnd().endsWith("?")).length;
 	const listCount = $("ul, ol").length;
 	const tableCount = $("table").length;
 	const structuredBlocks = listCount + tableCount;
 	const answerable = $("main p, article p, p").toArray().slice(0, 3).some((element) => $(element).text().replace(/\s+/g, " ").trim().split(/\s+/).filter(Boolean).length >= 25);
-	const imageElements = $("img").toArray();
-	const oversizedImageCandidates = imageElements.map((element) => {
+	let imagesTotal = 0;
+	let imagesMissingAlt = 0;
+	const oversizedImageCandidates = [];
+	$("img").each((_index, element) => {
+		imagesTotal += 1;
+		if ($(element).attr("alt") === void 0) imagesMissingAlt += 1;
+		if (oversizedImageCandidates.length >= 25) return;
 		const src = absoluteUrl$2($(element).attr("src"), fetchResult.finalUrl);
-		if (!src) return void 0;
+		if (!src) return;
 		const width = numericAttribute($(element).attr("width"));
 		const height = numericAttribute($(element).attr("height"));
 		const srcsetWidth = largestSrcsetWidth($(element).attr("srcset"));
 		const filenameDimensions = imageDimensionsFromUrl(src);
-		if (Math.max(width ?? 0, height ?? 0, srcsetWidth ?? 0, filenameDimensions?.width ?? 0, filenameDimensions?.height ?? 0) < 2e3) return void 0;
+		if (Math.max(width ?? 0, height ?? 0, srcsetWidth ?? 0, filenameDimensions?.width ?? 0, filenameDimensions?.height ?? 0) < 2e3) return;
 		const detectedFrom = [
 			width && width >= 2e3 ? "width" : void 0,
 			height && height >= 2e3 ? "height" : void 0,
@@ -2885,13 +2958,13 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
 		].filter(Boolean).join(",");
 		const candidateWidth = Math.max(width ?? 0, srcsetWidth ?? 0, filenameDimensions?.width ?? 0);
 		const candidateHeight = Math.max(height ?? 0, filenameDimensions?.height ?? 0);
-		return {
+		oversizedImageCandidates.push({
 			src,
 			detectedFrom,
 			...candidateWidth > 0 ? { width: candidateWidth } : {},
 			...candidateHeight > 0 ? { height: candidateHeight } : {}
-		};
-	}).filter((candidate) => Boolean(candidate));
+		});
+	});
 	const mixedContentUrls = url.protocol === "https:" ? [...new Set($("img[src], script[src], iframe[src], source[src], video[src], audio[src], link[href]").toArray().map((element) => absoluteUrl$2($(element).attr("src") ?? $(element).attr("href"), fetchResult.finalUrl)).filter((value) => typeof value === "string" && new URL(value).protocol === "http:"))] : [];
 	const metaRobots = combineRobotsValues($("meta[name]").toArray().filter((element) => {
 		const name = $(element).attr("name")?.trim().toLowerCase();
@@ -2915,6 +2988,18 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
 		hasViewport: Boolean($("meta[name=\"viewport\"]").attr("content")),
 		headings,
 		links,
+		...extractor === "crawler" ? { crawlerEvidence: {
+			h1: crawlerH1,
+			h1Count: crawlerH1Count,
+			h2Count: crawlerH2Count,
+			h3Count: crawlerH3Count,
+			questionHeadings,
+			internalLinks: [...crawlerInternalLinks],
+			externalLinks: [...crawlerExternalLinks],
+			internalAnchorSamples,
+			externalAnchorSamples,
+			socialProfileLinks: [...socialProfileLinks]
+		} } : {},
 		markdownAlternates,
 		hreflang,
 		jsonLd: structuredData.jsonLd,
@@ -2923,19 +3008,17 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
 		unrecognizedJsonLdTypes: structuredData.unrecognizedJsonLdTypes.slice(0, 25),
 		structuredDataFormats: structuredData.formats,
 		googleRichResults: structuredData.googleRichResults,
-		schemaSameAsEvidence: structuredData.sameAs,
-		invalidSchemaSameAs: structuredData.invalidSameAs,
+		schemaSameAsEvidence: extractor === "crawler" ? structuredData.sameAs.slice(0, 50) : structuredData.sameAs,
+		invalidSchemaSameAs: extractor === "crawler" ? structuredData.invalidSameAs.slice(0, 25) : structuredData.invalidSameAs,
 		schemaTypes: structuredData.schemaTypes,
 		openGraph,
 		twitter,
 		author,
 		hasAuthor,
 		hasDate,
-		imagesTotal: imageElements.length,
-		imagesMissingAlt: imageElements.filter((element) => {
-			return $(element).attr("alt") === void 0;
-		}).length,
-		oversizedImageCandidates: oversizedImageCandidates.slice(0, 25),
+		imagesTotal,
+		imagesMissingAlt,
+		oversizedImageCandidates,
 		mixedContentUrls,
 		semanticHtml,
 		questionHeadings,
@@ -2943,7 +3026,7 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
 		tableCount,
 		structuredBlocks,
 		answerable,
-		contentText: text.replace(/\s+/g, " ").trim(),
+		contentText: extractor === "crawler" ? text : text.replace(/\s+/g, " ").trim(),
 		excerpt,
 		wordCount: content.wordCount,
 		contentExtraction: content.diagnostics,
@@ -3473,7 +3556,7 @@ function decodePageFetchCacheEvidence(value) {
 		return;
 	}
 }
-async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, respectRobots = false) {
+async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, respectRobots = false, writeCache = true) {
 	const startedAt = Date.now();
 	const db = getDb();
 	const key = hashKey(["page", url]);
@@ -3585,7 +3668,7 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
 		warnings: [...robots.allowed === null ? [`robots.txt availability is unknown${robots.status ? ` after HTTP ${robots.status}` : ""}; this fetch does not prove crawler access is allowed.`] : [], ...accessBlock ? [`${accessBlock.guidance.summary} Identify the crawler as ${accessBlock.crawler.userAgent}.`] : []],
 		robotsTxt: pageRobotsEvidence(robots)
 	};
-	if (!accessBlock) {
+	if (!accessBlock && writeCache) {
 		db.prepare(`INSERT OR REPLACE INTO http_cache
       (url_hash, url, status, headers_json, body_blob, metadata_json, etag, fetched_at, expires_at)
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(key, url, response.response.status, JSON.stringify(headerMap), Buffer.from(html), encodePageFetchCacheEvidence(result), response.response.headers.get("etag"), Date.now(), Date.now() + 36e5);
@@ -4153,7 +4236,7 @@ async function fetchPage(url, opts = {}, dependencies = {}) {
 	const makeRenderer = dependencies.createPageRenderer ?? createPageRenderer;
 	const rate = normalizeRateControls(opts.rate);
 	return await queueForHost(new URL(url).host, rate).add(async () => {
-		const first = await plainFetch(url, opts.refresh, opts.timeoutMs, rate, opts.signal, opts.respectRobots);
+		const first = await plainFetch(url, opts.refresh, opts.timeoutMs, rate, opts.signal, opts.respectRobots, opts.writeCache);
 		if (!(!first.diagnostics.accessBlock && (renderingMode === "on" || renderingMode === "auto" && looksLikeSpa(first.html))) || opts.signal?.aborted) return withRenderingState(first, renderingMode, opts.signal?.aborted ? "skipped" : renderingMode === "off" ? "not-requested" : "not-needed");
 		const ownsRenderer = !opts.renderer;
 		const renderer = opts.renderer ?? makeRenderer();
@@ -10431,6 +10514,7 @@ const CRAWL_SKIP_REASONS = [
 	"configured-exclusion",
 	"duplicate-normalization",
 	"invalid-url",
+	"memory-pressure",
 	"off-origin",
 	"origin-backpressure",
 	"queue-safety-limit",
@@ -10443,6 +10527,7 @@ const SKIP_IMPACT = {
 	"configured-exclusion": "non-impacting",
 	"duplicate-normalization": "non-impacting",
 	"invalid-url": "non-impacting",
+	"memory-pressure": "coverage-affecting",
 	"off-origin": "non-impacting",
 	"origin-backpressure": "coverage-affecting",
 	"queue-safety-limit": "coverage-affecting",
@@ -10620,17 +10705,44 @@ function sanitizeTenantString(value) {
 	return value.replace(/https?:\/\/[^\s"'<>`]+/g, (match) => sanitizeUrlString(match)).replace(/\b(access_?token|api_?key|auth|client_?secret|jwt|password|refresh_?token|secret|session|signature|token)(\s*[=:]\s*)[^\s,;&]+/gi, (_match, key, separator) => `${key}${separator}[redacted]`).replace(/\b[A-Za-z]:\\[^\s"'<>`]+/g, "[local-path]").replace(/\/(?:Users|home|tmp|var\/folders)\/[^\s"'<>`]+/g, "[local-path]");
 }
 function sanitizeTenantValue(value, key) {
-	if (key && isSensitiveFieldName(key)) return "[redacted]";
+	if (key && isSensitiveFieldName(key)) return value === "[redacted]" ? value : "[redacted]";
 	if (typeof value === "string") return sanitizeTenantString(value);
-	if (Array.isArray(value)) return value.map((item) => sanitizeTenantValue(item));
+	if (Array.isArray(value)) {
+		let sanitized;
+		for (let index = 0; index < value.length; index += 1) {
+			const item = sanitizeTenantValue(value[index]);
+			if (sanitized) sanitized.push(item);
+			else if (item !== value[index]) sanitized = [...value.slice(0, index), item];
+		}
+		return sanitized ?? value;
+	}
 	if (!value || typeof value !== "object") return value;
-	return Object.fromEntries(Object.entries(value).map(([entryKey, entry]) => [entryKey, sanitizeTenantValue(entry, entryKey)]));
+	const record = value;
+	let sanitized;
+	for (const [entryKey, entry] of Object.entries(record)) {
+		const safeEntry = sanitizeTenantValue(entry, entryKey);
+		if (safeEntry === entry) continue;
+		sanitized ??= { ...record };
+		sanitized[entryKey] = safeEntry;
+	}
+	return sanitized ?? value;
 }
 function sanitizeCrawlConfig(config) {
 	return sanitizeTenantValue(config);
 }
 function sanitizePages(pages) {
-	return sanitizeTenantValue(pages).map(({ seoScore: _seoScore, geoScore: _geoScore, ...page }) => page);
+	const safePages = sanitizeTenantValue(pages);
+	let normalized;
+	for (let index = 0; index < safePages.length; index += 1) {
+		const page = safePages[index];
+		if (!page) continue;
+		if ("seoScore" in page || "geoScore" in page) {
+			normalized ??= safePages.slice(0, index);
+			const { seoScore: _seoScore, geoScore: _geoScore, ...current } = page;
+			normalized.push(current);
+		} else if (normalized) normalized.push(page);
+	}
+	return normalized ?? safePages;
 }
 function sanitizeIssues(issues) {
 	return sanitizeTenantValue(issues);
@@ -10659,18 +10771,28 @@ function summarizeCrawlReport(input) {
 	const byCategory = {};
 	let responseMs = 0;
 	let responseCount = 0;
+	let statusOnlyPages = 0;
+	let indexablePages = 0;
+	let nonIndexablePages = 0;
+	let statusErrors = 0;
 	for (const page of input.pages) {
 		byStatus[String(page.status)] = (byStatus[String(page.status)] ?? 0) + 1;
 		if (typeof page.responseTimeMs === "number") {
 			responseMs += page.responseTimeMs;
 			responseCount += 1;
 		}
+		if (page.auditScope === "status") statusOnlyPages += 1;
+		else if (page.indexable) indexablePages += 1;
+		else nonIndexablePages += 1;
+		if (page.status === 0 || page.status >= 400) statusErrors += 1;
 	}
 	const requests = input.requests ?? [];
-	const failedRequests = requests.filter((request) => request.outcome === "failure" && request.failureKind !== "aborted");
-	const abortedRequests = requests.filter((request) => request.outcome === "failure" && request.failureKind === "aborted");
 	let requestMs = 0;
 	let requestMsCount = 0;
+	let responseRequests = 0;
+	let failedRequests = 0;
+	let abortedRequests = 0;
+	let extractionFailures = 0;
 	for (const request of requests) {
 		const status = request.outcome === "response" ? String(request.status) : request.outcome === "skipped" ? request.reason : request.failureKind === "aborted" ? "aborted" : "no-response";
 		requestByStatus[status] = (requestByStatus[status] ?? 0) + 1;
@@ -10678,27 +10800,39 @@ function summarizeCrawlReport(input) {
 			requestMs += request.durationMs;
 			requestMsCount += 1;
 		}
+		if (request.outcome === "response") {
+			responseRequests += 1;
+			if (request.extraction === "failed") extractionFailures += 1;
+		} else if (request.outcome === "failure") if (request.failureKind === "aborted") abortedRequests += 1;
+		else failedRequests += 1;
+	}
+	let highIssues = 0;
+	let mediumIssues = 0;
+	let lowIssues = 0;
+	for (const issue of input.issues) {
+		byCategory[issue.category] = (byCategory[issue.category] ?? 0) + 1;
+		if (issue.severity === "high") highIssues += 1;
+		else if (issue.severity === "medium") mediumIssues += 1;
+		else lowIssues += 1;
 	}
-	for (const issue of input.issues) byCategory[issue.category] = (byCategory[issue.category] ?? 0) + 1;
 	const runStats = crawlRunStats(input.pages, input.stats);
-	const indexabilityPages = input.pages.filter((page) => page.auditScope !== "status");
 	return {
 		totalPages: input.pages.length,
-		statusOnlyPages: input.pages.filter((page) => page.auditScope === "status").length,
-		indexablePages: indexabilityPages.filter((page) => page.indexable).length,
-		nonIndexablePages: indexabilityPages.filter((page) => !page.indexable).length,
-		statusErrors: input.pages.filter((page) => page.status === 0 || page.status >= 400).length,
+		statusOnlyPages,
+		indexablePages,
+		nonIndexablePages,
+		statusErrors,
 		...runStats,
 		attemptedRequests: input.requestEvidenceStatus === "unavailable" ? 0 : requests.length,
-		responseRequests: requests.filter((request) => request.outcome === "response").length,
-		failedRequests: failedRequests.length,
-		abortedRequests: abortedRequests.length,
-		extractionFailures: requests.filter((request) => request.outcome === "response" && request.extraction === "failed").length,
+		responseRequests,
+		failedRequests,
+		abortedRequests,
+		extractionFailures,
 		requestByStatus,
 		avgRequestMs: requestMsCount ? Math.round(requestMs / requestMsCount) : void 0,
-		highIssues: input.issues.filter((issue) => issue.severity === "high").length,
-		mediumIssues: input.issues.filter((issue) => issue.severity === "medium").length,
-		lowIssues: input.issues.filter((issue) => issue.severity === "low").length,
+		highIssues,
+		mediumIssues,
+		lowIssues,
 		avgResponseMs: responseCount ? Math.round(responseMs / responseCount) : void 0,
 		byStatus,
 		byCategory
@@ -10706,42 +10840,55 @@ function summarizeCrawlReport(input) {
 }
 const ACCESS_BLOCK_SAMPLE_LIMIT = 10;
 function crawlAccessSummary(requests) {
-	const blocked = requests.flatMap((request) => request.outcome === "response" && request.accessBlock ? [{
-		url: request.requestedUrl,
-		evidence: request.accessBlock
-	}] : []);
 	const providers = {};
-	for (const item of blocked) providers[item.evidence.provider] = (providers[item.evidence.provider] ?? 0) + 1;
+	const samples = [];
+	let blockedRequests = 0;
+	for (const request of requests) {
+		if (request.outcome !== "response" || !request.accessBlock) continue;
+		blockedRequests += 1;
+		providers[request.accessBlock.provider] = (providers[request.accessBlock.provider] ?? 0) + 1;
+		if (samples.length < ACCESS_BLOCK_SAMPLE_LIMIT) samples.push({
+			url: request.requestedUrl,
+			evidence: request.accessBlock
+		});
+	}
 	return {
 		crawler: SEO_CRAWLER_IDENTITY,
-		blockedRequests: blocked.length,
+		blockedRequests,
 		providers,
-		samples: blocked.slice(0, ACCESS_BLOCK_SAMPLE_LIMIT),
+		samples,
 		sampleLimit: ACCESS_BLOCK_SAMPLE_LIMIT,
-		truncated: blocked.length > ACCESS_BLOCK_SAMPLE_LIMIT
+		truncated: blockedRequests > ACCESS_BLOCK_SAMPLE_LIMIT
 	};
 }
 function crawlRunStats(pages, stats = {}) {
-	const discovered = /* @__PURE__ */ new Set();
+	const needsDiscoveredUrls = stats.discoveredUrls === void 0;
+	const needsObservedInternalLinks = stats.observedInternalLinks === void 0;
+	const needsFailedUrls = stats.failedUrls === void 0;
+	const discovered = needsDiscoveredUrls ? /* @__PURE__ */ new Set() : void 0;
 	let observedInternalLinks = 0;
-	for (const page of pages) {
-		discovered.add(page.url);
-		for (const url of page.sampleInternalLinks ?? []) discovered.add(url);
-		for (const url of page.sampleExternalLinks ?? []) discovered.add(url);
-		observedInternalLinks += page.outgoingInternalCount;
+	let failedUrls = 0;
+	if (needsDiscoveredUrls || needsObservedInternalLinks || needsFailedUrls) for (const page of pages) {
+		if (discovered) {
+			discovered.add(page.url);
+			for (const url of page.sampleInternalLinks ?? []) discovered.add(url);
+			for (const url of page.sampleExternalLinks ?? []) discovered.add(url);
+		}
+		if (needsObservedInternalLinks) observedInternalLinks += page.outgoingInternalCount;
+		if (needsFailedUrls && (page.status === 0 || page.status >= 400)) failedUrls += 1;
 	}
 	const skips = normalizeCrawlSkipReasonCounts({
 		skippedUrls: stats.skippedUrls,
 		skipReasons: stats.skipReasons ?? crawlSkipReasonCountsFromRecord(stats.skipReasonCounts)
 	});
 	return {
-		discoveredUrls: stats.discoveredUrls ?? discovered.size,
+		discoveredUrls: stats.discoveredUrls ?? discovered?.size ?? 0,
 		queuedUrls: stats.queuedUrls ?? pages.length,
 		crawledUrls: stats.crawledUrls ?? pages.length,
 		skippedUrls: skips.skippedUrls,
 		skipReasons: skips.skipReasons,
 		skippedUrlsByImpact: skips.skippedUrlsByImpact,
-		failedUrls: stats.failedUrls ?? pages.filter((page) => page.status === 0 || page.status >= 400).length,
+		failedUrls: stats.failedUrls ?? failedUrls,
 		observedInternalLinks: stats.observedInternalLinks ?? observedInternalLinks,
 		pageLimitReached: stats.pageLimitReached ?? false
 	};
@@ -10758,11 +10905,14 @@ function normalizeLinkUrl(value) {
 function deriveInternalLinkAuthority(pages, linkGraph) {
 	const hasLinkGraph = linkGraph && Object.keys(linkGraph).length > 0;
 	const hasStoredScores = pages.some((page) => page.internalInlinkCount !== void 0 || page.internalLinkAuthorityScore !== void 0);
-	if (!hasLinkGraph && hasStoredScores) return pages.map((page) => ({
-		...page,
-		internalInlinkCount: page.internalInlinkCount ?? 0,
-		internalLinkAuthorityScore: page.internalLinkAuthorityScore ?? 0
-	}));
+	if (!hasLinkGraph && hasStoredScores) {
+		if (pages.every((page) => page.internalInlinkCount !== void 0 && page.internalLinkAuthorityScore !== void 0)) return pages;
+		return pages.map((page) => page.internalInlinkCount !== void 0 && page.internalLinkAuthorityScore !== void 0 ? page : {
+			...page,
+			internalInlinkCount: page.internalInlinkCount ?? 0,
+			internalLinkAuthorityScore: page.internalLinkAuthorityScore ?? 0
+		});
+	}
 	const pageByUrl = /* @__PURE__ */ new Map();
 	for (const page of pages) for (const value of [page.url, page.finalUrl]) {
 		const normalized = normalizeLinkUrl(value);
@@ -10781,7 +10931,8 @@ function deriveInternalLinkAuthority(pages, linkGraph) {
 		}
 		for (const linkedPage of linkedPages) inlinks.set(linkedPage, (inlinks.get(linkedPage) ?? 0) + 1);
 	}
-	const maxInlinks = Math.max(0, ...inlinks.values());
+	let maxInlinks = 0;
+	for (const count of inlinks.values()) maxInlinks = Math.max(maxInlinks, count);
 	return pages.map((page) => {
 		const internalInlinkCount = inlinks.get(page.url) ?? 0;
 		return {
@@ -11030,6 +11181,7 @@ function crawlCaveats(input) {
 	const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
 	if (input.queueSafetySkippedUrls > 0) caveats.push(`Left ${input.queueSafetySkippedUrls} eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
 	if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
+	if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
 	return caveats;
 }
 //#endregion
@@ -11056,6 +11208,26 @@ function cancellationRace(signal) {
 	});
 }
 //#endregion
+//#region packages/core/dist/analyze/crawler/crawl-memory.js
+const MEBIBYTE$1 = 1024 * 1024;
+const MIN_CRAWL_MEMORY_LIMIT = 384 * MEBIBYTE$1;
+const MAX_CRAWL_MEMORY_LIMIT = 640 * MEBIBYTE$1;
+const MIN_CRAWL_RSS_LIMIT = 512 * MEBIBYTE$1;
+const MAX_CRAWL_RSS_LIMIT = 896 * MEBIBYTE$1;
+const SYSTEM_MEMORY_FRACTION = .1;
+const SYSTEM_RSS_FRACTION = .2;
+function crawlMemoryLimitBytes(totalMemoryBytes) {
+	if (!Number.isFinite(totalMemoryBytes) || totalMemoryBytes <= 0) return MAX_CRAWL_MEMORY_LIMIT;
+	return Math.min(MAX_CRAWL_MEMORY_LIMIT, Math.max(MIN_CRAWL_MEMORY_LIMIT, Math.floor(totalMemoryBytes * SYSTEM_MEMORY_FRACTION)));
+}
+function crawlRssLimitBytes(totalMemoryBytes) {
+	if (!Number.isFinite(totalMemoryBytes) || totalMemoryBytes <= 0) return MAX_CRAWL_RSS_LIMIT;
+	return Math.min(MAX_CRAWL_RSS_LIMIT, Math.max(MIN_CRAWL_RSS_LIMIT, Math.floor(totalMemoryBytes * SYSTEM_RSS_FRACTION)));
+}
+function crawlMemoryPressure(input) {
+	return input.memoryUsage.heapUsed + input.memoryUsage.external >= crawlMemoryLimitBytes(input.totalMemoryBytes) || input.memoryUsage.rss >= crawlRssLimitBytes(input.totalMemoryBytes);
+}
+//#endregion
 //#region packages/core/dist/analyze/crawler/crawl-url-queue.js
 function before(left, right) {
 	return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
@@ -11773,6 +11945,7 @@ function crawlProviderLimits(input) {
 }
 async function crawlDataSources(input) {
 	const window = crawlMetricsWindow(input.now());
+	const skippedForMemory = input.skipReason === "memory-pressure";
 	const dataSources = {
 		searchConsole: {
 			status: "skipped",
@@ -11783,7 +11956,7 @@ async function crawlDataSources(input) {
 			pageLimit: input.searchMetricsLimit,
 			pageLimitReached: false,
 			retainedRowLimitReached: false,
-			warning: input.cancelled ? "Search Console join skipped because the crawl was cancelled." : "Search Console join skipped because no property was selected."
+			warning: skippedForMemory ? "Search Console join skipped after the local memory safety limit was reached." : input.skipReason === "cancelled" ? "Search Console join skipped because the crawl was cancelled." : "Search Console join skipped because no property was selected."
 		},
 		analytics: {
 			status: "skipped",
@@ -11792,10 +11965,10 @@ async function crawlDataSources(input) {
 			joinedPages: 0,
 			retainedRowLimit: input.analyticsLimit,
 			retainedRowLimitReached: false,
-			warning: input.cancelled ? "Google Analytics join skipped because the crawl was cancelled." : "Google Analytics join skipped because no property was selected."
+			warning: skippedForMemory ? "Google Analytics join skipped after the local memory safety limit was reached." : input.skipReason === "cancelled" ? "Google Analytics join skipped because the crawl was cancelled." : "Google Analytics join skipped because no property was selected."
 		}
 	};
-	if (!input.cancelled && input.site) dataSources.searchConsole = await joinSearchMetrics({
+	if (!input.skipReason && input.site) dataSources.searchConsole = await joinSearchMetrics({
 		site: input.site,
 		pages: input.pages,
 		warnings: input.warnings,
@@ -11808,7 +11981,7 @@ async function crawlDataSources(input) {
 		queryPagesMetricsBatch: input.queryPagesMetricsBatch,
 		queryPagesTopQueriesBatch: input.queryPagesTopQueriesBatch
 	});
-	if (!input.cancelled && input.googleAnalyticsPropertyId) dataSources.analytics = await joinAnalytics({
+	if (!input.skipReason && input.googleAnalyticsPropertyId) dataSources.analytics = await joinAnalytics({
 		propertyId: input.googleAnalyticsPropertyId,
 		pages: input.pages,
 		warnings: input.warnings,
@@ -11821,17 +11994,6 @@ async function crawlDataSources(input) {
 }
 //#endregion
 //#region packages/core/dist/analyze/monitoring/crawl-page.js
-function sameOriginUrl(href, base) {
-	try {
-		const url = new URL(href, base);
-		url.hash = "";
-		if (url.origin !== base.origin) return void 0;
-		if (!["http:", "https:"].includes(url.protocol)) return void 0;
-		return url.toString();
-	} catch {
-		return;
-	}
-}
 function resolvedUrl(href, base) {
 	if (!href) return void 0;
 	try {
@@ -11861,49 +12023,6 @@ function safeResponseHeaders(headers) {
 function truncate$1(value, max) {
 	return value.length > max ? value.slice(0, max).trimEnd() : value;
 }
-function anchorSamples(links, internal) {
-	const seen = /* @__PURE__ */ new Set();
-	const samples = [];
-	for (const link of links) {
-		if (link.internal !== internal) continue;
-		const text = truncate$1(link.text.replace(/\s+/g, " ").trim(), 120);
-		if (!text) continue;
-		const key = `${link.href}\n${text}`;
-		if (seen.has(key)) continue;
-		seen.add(key);
-		samples.push({
-			href: link.href,
-			text
-		});
-		if (samples.length >= 25) break;
-	}
-	return samples;
-}
-function socialProfileLinks(links) {
-	const hosts = [
-		"facebook.com",
-		"instagram.com",
-		"linkedin.com",
-		"pinterest.com",
-		"tiktok.com",
-		"twitter.com",
-		"x.com",
-		"youtube.com"
-	];
-	const profiles = /* @__PURE__ */ new Set();
-	for (const link of links) {
-		if (link.internal) continue;
-		try {
-			const url = new URL(link.href);
-			const host = url.hostname.replace(/^www\./, "").toLowerCase();
-			if (hosts.some((value) => host === value || host.endsWith(`.${value}`))) {
-				url.hash = "";
-				profiles.add(url.toString());
-			}
-		} catch {}
-	}
-	return [...profiles].slice(0, 50);
-}
 function sameDocumentUrl(left, right) {
 	if (!left || !right) return false;
 	try {
@@ -12003,17 +12122,9 @@ function statusOnlyPage(fetched) {
 		finalUrl: fetched.finalUrl,
 		status: fetched.status,
 		contentType,
-		responseHeaders: safeResponseHeaders(fetched.headers),
 		responseTimeMs: fetched.diagnostics.durationMs,
-		usedJs: false,
-		fetchSource: "network",
-		cacheState: "bypass",
-		fetchDiagnostics: fetched.diagnostics,
 		auditScope: "status",
-		accessBlock: fetched.diagnostics.accessBlock,
-		blocked: fetched.diagnostics.blocked,
 		contentAuditAllowed: false,
-		robotsTxt: fetched.robotsTxt,
 		indexable: false,
 		indexability: "Not evaluated by status-only health probe",
 		declaredIndexability: "unknown",
@@ -12021,8 +12132,7 @@ function statusOnlyPage(fetched) {
 		wordCount: 0,
 		contentHash: "",
 		outgoingInternalCount: 0,
-		outgoingExternalCount: 0,
-		isHttps: new URL(fetched.finalUrl).protocol === "https:"
+		outgoingExternalCount: 0
 	};
 }
 function indexabilityReason(page) {
@@ -12098,12 +12208,11 @@ async function crawlOne(url, opts = {}) {
 	};
 	try {
 		const extracted = await extractPage(fetched, "crawler");
-		const base = new URL(extracted.finalUrl);
-		const internalLinks = extracted.links.map((link) => sameOriginUrl(link.href, base)).filter((value) => Boolean(value));
-		const externalLinks = extracted.links.filter((link) => !link.internal).map((link) => link.href).filter((href) => /^https?:\/\//.test(href));
-		const h1 = extracted.headings.find((heading) => heading.level === 1)?.text;
-		const uniqueInternalLinks = [...new Set(internalLinks)];
-		const uniqueExternalLinks = [...new Set(externalLinks)];
+		const crawlerEvidence = extracted.crawlerEvidence;
+		if (!crawlerEvidence) throw new Error("Crawler extraction did not return crawl evidence.");
+		const h1 = crawlerEvidence.h1;
+		const uniqueInternalLinks = crawlerEvidence.internalLinks;
+		const uniqueExternalLinks = crawlerEvidence.externalLinks;
 		const canonical = extracted.canonicalEvidence?.selectedUrl ?? resolvedUrl(extracted.canonical, extracted.finalUrl);
 		const richResults = selectGoogleRichResultAssessments(extracted.googleRichResults ?? []);
 		const page = {
@@ -12129,9 +12238,9 @@ async function crawlOne(url, opts = {}) {
 			metaRobots: extracted.metaRobots,
 			xRobotsTag: extracted.xRobotsTag,
 			h1,
-			h1Count: extracted.headings.filter((heading) => heading.level === 1).length,
-			h2Count: extracted.headings.filter((heading) => heading.level === 2).length,
-			h3Count: extracted.headings.filter((heading) => heading.level === 3).length,
+			h1Count: crawlerEvidence.h1Count,
+			h2Count: crawlerEvidence.h2Count,
+			h3Count: crawlerEvidence.h3Count,
 			indexable: false,
 			extractionStatus: "complete",
 			wordCount: extracted.wordCount,
@@ -12167,8 +12276,8 @@ async function crawlOne(url, opts = {}) {
 			outgoingExternalCount: uniqueExternalLinks.length,
 			sampleInternalLinks: uniqueInternalLinks.slice(0, 25),
 			sampleExternalLinks: uniqueExternalLinks.slice(0, 25),
-			internalAnchorSamples: anchorSamples(extracted.links, true),
-			externalAnchorSamples: anchorSamples(extracted.links, false),
+			internalAnchorSamples: crawlerEvidence.internalAnchorSamples,
+			externalAnchorSamples: crawlerEvidence.externalAnchorSamples,
 			markdownAlternates: extracted.markdownAlternates,
 			schemaTypes: extracted.schemaTypes,
 			structuredDataFormats: extracted.structuredDataFormats ?? [],
@@ -12177,7 +12286,7 @@ async function crawlOne(url, opts = {}) {
 			schemaSameAs: [...new Set((extracted.schemaSameAsEvidence ?? []).map((evidence) => evidence.url))].slice(0, 50),
 			schemaSameAsEvidence: (extracted.schemaSameAsEvidence ?? []).slice(0, 50),
 			invalidSchemaSameAs: (extracted.invalidSchemaSameAs ?? []).slice(0, 25),
-			socialProfileLinks: socialProfileLinks(extracted.links),
+			socialProfileLinks: crawlerEvidence.socialProfileLinks,
 			invalidJsonLdCount: extracted.invalidJsonLdCount,
 			invalidJsonLdSamples: extracted.invalidJsonLdSamples,
 			unrecognizedJsonLdTypes: extracted.unrecognizedJsonLdTypes ?? [],
@@ -12830,7 +12939,9 @@ function resolveCrawlSiteDependencies(dependencies = {}) {
 		fetchLandingPageValues: dependencies.fetchLandingPageValues ?? fetchLandingPageValues,
 		landingValueForUrl: dependencies.landingValueForUrl ?? landingValueForUrl,
 		collectAgentDiscovery: dependencies.collectAgentDiscovery ?? collectAgentDiscovery,
-		now: dependencies.now ?? (() => /* @__PURE__ */ new Date())
+		now: dependencies.now ?? (() => /* @__PURE__ */ new Date()),
+		memoryUsage: dependencies.memoryUsage ?? (() => process.memoryUsage()),
+		totalMemory: dependencies.totalMemory ?? totalmem
 	};
 }
 //#endregion
@@ -12986,15 +13097,26 @@ async function crawlSite(input, dependencies) {
 	let queuedUrls = 0;
 	let skippedUrls = 0;
 	const skipReasonCounts = {};
-	const recordSkip = (reason) => {
-		skippedUrls += 1;
-		skipReasonCounts[reason] = (skipReasonCounts[reason] ?? 0) + 1;
+	const recordSkips = (reason, count = 1) => {
+		if (count <= 0) return;
+		skippedUrls += count;
+		skipReasonCounts[reason] = (skipReasonCounts[reason] ?? 0) + count;
 	};
+	const recordSkip = (reason) => recordSkips(reason);
 	let robotsDeferredUrls = 0;
 	let originBackpressureSkippedUrls = 0;
 	let queueSafetySkippedUrls = 0;
 	let failedUrls = 0;
 	let observedInternalLinks = 0;
+	let memoryLimited = false;
+	const memoryDeferredUrls = /* @__PURE__ */ new Set();
+	const checkMemoryPressure = () => {
+		if (memoryLimited) return;
+		memoryLimited = crawlMemoryPressure({
+			memoryUsage: deps.memoryUsage(),
+			totalMemoryBytes: deps.totalMemory()
+		});
+	};
 	let sitemapDiscovery;
 	let externalLinkVerification;
 	let statusEventChain = Promise.resolve();
@@ -13124,6 +13246,7 @@ async function crawlSite(input, dependencies) {
 			for (const url of sitemap.urls) enqueue(url, 0);
 			sitemapDiscovery = sitemap.discovery;
 		}
+		checkMemoryPressure();
 		const nextQueueItem = () => {
 			while (queue.size) {
 				const item = queue.take();
@@ -13149,6 +13272,7 @@ async function crawlSite(input, dependencies) {
 					timeoutMs: config.timeoutMs,
 					rate: config.fetchRate,
 					signal,
+					writeCache: false,
 					respectRobots: config.respectRobots,
 					robotsResolver,
 					renderer
@@ -13176,8 +13300,8 @@ async function crawlSite(input, dependencies) {
 				depth: item.depth
 			});
 		};
-		while (!isCancelled() && (queue.size || inFlight.size) && pages.length < config.maxPages) {
-			while (!isCancelled() && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
+		while (!isCancelled() && (queue.size || inFlight.size) && (!memoryLimited || inFlight.size > 0) && pages.length < config.maxPages) {
+			while (!isCancelled() && !memoryLimited && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
 				const item = nextQueueItem();
 				if (!item) break;
 				startTask(item);
@@ -13205,6 +13329,7 @@ async function crawlSite(input, dependencies) {
 				healthyProbeStreak = 0;
 				activeConcurrency = 1;
 			}
+			checkMemoryPressure();
 			if (request.outcome === "skipped") {
 				recordSkip(request.reason === "robots-disallowed" ? "robots-disallowed" : request.reason === "robots-deferred" ? "robots-uncertain" : "origin-backpressure");
 				if (request.reason === "robots-deferred") {
@@ -13335,10 +13460,24 @@ async function crawlSite(input, dependencies) {
 				depth: task.depth,
 				statusCode: page.status
 			});
-			if (followLinks && task.depth < config.maxDepth) for (const url of result.urls) enqueue(url, task.depth + 1);
+			if (followLinks && task.depth < config.maxDepth) if (memoryLimited) for (const value of result.urls) {
+				const normalized = normalizeUrl$1(value, finalUrl);
+				if (normalized && sameOrigin(normalized, origin) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
+					discovered.add(normalized);
+					memoryDeferredUrls.add(normalized);
+				}
+			}
+			else for (const url of result.urls) enqueue(url, task.depth + 1);
+		}
+		const memoryPressureSkippedUrls = memoryLimited ? queue.size + memoryDeferredUrls.size : 0;
+		const stoppedForMemory = memoryPressureSkippedUrls > 0;
+		if (stoppedForMemory) {
+			recordSkips("memory-pressure", memoryPressureSkippedUrls);
+			warnings.push("Crawl stopped before local memory use could become unsafe. The retained report is partial.");
 		}
 		const resolvedInlinkCounts = resolveLinkCountAliases(incomingLinkCounts, fetchedAliases);
-		const maxInlinks = Math.max(0, ...pages.map((page) => resolvedInlinkCounts.get(page.finalUrl) ?? 0));
+		let maxInlinks = 0;
+		for (const page of pages) maxInlinks = Math.max(maxInlinks, resolvedInlinkCounts.get(page.finalUrl) ?? 0);
 		for (const page of pages) {
 			const internalInlinkCount = resolvedInlinkCounts.get(page.finalUrl) ?? 0;
 			page.internalInlinkCount = internalInlinkCount;
@@ -13358,11 +13497,11 @@ async function crawlSite(input, dependencies) {
 		const sitemapEvidencePartial = sitemapDiscovery?.dataStatus === "partial";
 		const accessEvidencePartial = requests.some((request) => request.outcome === "response" && request.accessBlock);
 		const requiredSitemapUnavailable = config.mode === "sitemap" && sitemapDiscovery?.dataStatus === "unavailable";
-		const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
+		const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
 		const failed = !cancelled && pages.length === 0 && (requiredSitemapUnavailable || requests.length > 0 && requests.every((request) => request.outcome === "failure"));
 		const requestEvidenceStatus = cancelled && inFlight.size > 0 ? "partial" : "available";
 		const dataSources = await crawlDataSources({
-			cancelled,
+			skipReason: cancelled ? "cancelled" : stoppedForMemory ? "memory-pressure" : void 0,
 			site: healthStrategy ? void 0 : site,
 			googleAnalyticsPropertyId: healthStrategy ? void 0 : input.googleAnalyticsPropertyId,
 			pages,
@@ -13381,7 +13520,7 @@ async function crawlSite(input, dependencies) {
 		});
 		const sourceEvidencePartial = [dataSources.searchConsole.status, dataSources.analytics.status].some((status) => status === "partial" || status === "unavailable");
 		const reportStatus = failed ? "failed" : partial || sourceEvidencePartial ? "partial" : "completed";
-		if (!cancelled && config.checkExternal) {
+		if (!cancelled && !stoppedForMemory && config.checkExternal) {
 			emitStatus("external_links_started", {
 				url: config.url,
 				message: "Started external link checks."
@@ -13398,7 +13537,7 @@ async function crawlSite(input, dependencies) {
 				message: "Finished external link checks."
 			});
 		}
-		const agentDiscovery = !cancelled && config.checkAgentDiscovery ? await deps.collectAgentDiscovery({
+		const agentDiscovery = !cancelled && !stoppedForMemory && config.checkAgentDiscovery ? await deps.collectAgentDiscovery({
 			startUrl: config.url,
 			pages,
 			timeoutMs: config.timeoutMs,
@@ -13439,7 +13578,8 @@ async function crawlSite(input, dependencies) {
 				pageLimitReached,
 				maxPages: config.maxPages,
 				queueSafetySkippedUrls,
-				originBackpressureSkippedUrls
+				originBackpressureSkippedUrls,
+				memoryPressureSkippedUrls
 			}),
 			stats: {
 				discoveredUrls: discovered.size,