seo 0.2.7 → 0.2.9

package/dist/index.js

@@ -1,9 +1,9 @@
 import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
-import { n as load, t as version } from "./package-BI-bLeV4.js";
+import { n as load, t as version } from "./package-C51oI-6o.js";
 import { Headers, fetch as fetch$1 } from "undici";
-import { chmodSync, createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { chmodSync, createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { arch, homedir, platform } from "node:os";
-import { dirname, join } from "node:path";
+import { basename, dirname, join } from "node:path";
 import { z } from "zod";
 import lockfile from "proper-lockfile";
 import crypto, { createHash, randomUUID, sign } from "node:crypto";
@@ -14,7 +14,6 @@ import { Readability } from "@mozilla/readability";
 import Defuddle from "defuddle";
 import { parseHTML } from "linkedom";
 import pRetry, { AbortError } from "p-retry";
-import UserAgent from "user-agents";
 import PQueue from "p-queue";
 import robotsParserModule from "robots-parser";
 import { chromium } from "playwright-core";
@@ -512,10 +511,16 @@ function getPrivacySnapshot() {
 //#region packages/core/src/storage/lock.ts
 async function withFileLock(path, task) {
 	ensureParentDir(path);
+	const fastLocks = process.env.SEO_LOCK_FAST === "1";
 	const release = await lockfile.lock(path, {
-		stale: 6e4,
-		update: 5e3,
-		retries: {
+		stale: fastLocks ? 2e3 : 6e4,
+		update: fastLocks ? 1e3 : 5e3,
+		retries: fastLocks ? {
+			retries: 5,
+			factor: 1.2,
+			minTimeout: 50,
+			maxTimeout: 200
+		} : {
 			retries: 60,
 			factor: 1.2,
 			minTimeout: 200,
@@ -1076,6 +1081,165 @@ async function exchangeCode(input) {
 	return tokenResponse.json();
 }
 //#endregion
+//#region packages/core/src/storage/cache-maintenance.ts
+const MEBIBYTE$1 = 1024 * 1024;
+const CACHE_MAX_BYTES = 256 * MEBIBYTE$1;
+const CACHE_MAX_AGE_MS = 720 * 60 * 60 * 1e3;
+const CACHE_LIMITS = {
+	gsc_cache: 48 * MEBIBYTE$1,
+	ga4_cache: 24 * MEBIBYTE$1,
+	semrush_cache: 16 * MEBIBYTE$1,
+	http_cache: 160 * MEBIBYTE$1,
+	performance_reports: 8 * MEBIBYTE$1
+};
+const CACHE_POLICIES = [
+	{
+		table: "gsc_cache",
+		keys: ["site_url", "query_hash"],
+		timestamp: "fetched_at",
+		sizeColumns: [
+			"site_url",
+			"query_hash",
+			"request_json",
+			"response_json"
+		]
+	},
+	{
+		table: "ga4_cache",
+		keys: ["property_id", "query_hash"],
+		timestamp: "fetched_at",
+		sizeColumns: [
+			"property_id",
+			"query_hash",
+			"request_json",
+			"response_json"
+		]
+	},
+	{
+		table: "semrush_cache",
+		keys: ["endpoint", "query_hash"],
+		timestamp: "fetched_at",
+		sizeColumns: [
+			"endpoint",
+			"query_hash",
+			"request_json",
+			"response_json"
+		]
+	},
+	{
+		table: "http_cache",
+		keys: ["url_hash"],
+		timestamp: "fetched_at",
+		sizeColumns: [
+			"url_hash",
+			"url",
+			"headers_json",
+			"body_blob",
+			"metadata_json",
+			"etag"
+		]
+	},
+	{
+		table: "performance_reports",
+		keys: ["id"],
+		timestamp: "created_at",
+		sizeColumns: [
+			"id",
+			"url",
+			"strategy",
+			"report_json"
+		]
+	}
+];
+function sizeExpression(columns) {
+	return `${columns.map((column) => `COALESCE(LENGTH(CAST(${column} AS BLOB)), 0)`).join(" + ")} + 256`;
+}
+function policySize(database, policy) {
+	const row = database.prepare(`SELECT COALESCE(SUM(${sizeExpression(policy.sizeColumns)}), 0) AS size_bytes
+      FROM ${policy.table}`).get();
+	return Number(row?.size_bytes ?? 0);
+}
+function cacheLogicalSizes(database) {
+	return Object.fromEntries(CACHE_POLICIES.map((policy) => [policy.table, policySize(database, policy)]));
+}
+function deleteExpired(database, policy, now) {
+	const cutoff = now - CACHE_MAX_AGE_MS;
+	return database.prepare(`DELETE FROM ${policy.table}
+      WHERE (expires_at IS NOT NULL AND expires_at <= ?)
+         OR (${policy.timestamp} IS NOT NULL AND ${policy.timestamp} < ?)`).run(now, cutoff).changes;
+}
+function pruneToLimit(database, policy, limit) {
+	if (policySize(database, policy) <= limit) return 0;
+	const order = [`${policy.timestamp} DESC`, ...policy.keys.map((key) => `${key} ASC`)].join(", ");
+	const rows = database.prepare(`SELECT ${policy.keys.join(", ")},
+        ${policy.timestamp} AS fetched_at,
+        ${sizeExpression(policy.sizeColumns)} AS size_bytes
+      FROM ${policy.table}
+      ORDER BY ${order}`).all();
+	const remove = [];
+	let retainedBytes = 0;
+	for (const row of rows) {
+		const rowBytes = Number(row.size_bytes);
+		if (retainedBytes + rowBytes <= limit) retainedBytes += rowBytes;
+		else remove.push(row);
+	}
+	if (!remove.length) return 0;
+	const statement = database.prepare(`DELETE FROM ${policy.table} WHERE ${policy.keys.map((key) => `${key} = ?`).join(" AND ")}`);
+	return database.transaction(() => {
+		let removed = 0;
+		for (const row of remove) removed += statement.run(...policy.keys.map((key) => row[key])).changes;
+		return removed;
+	}).immediate();
+}
+function pragmaNumber(database, pragma, field) {
+	const result = database.pragma(pragma);
+	const row = Array.isArray(result) ? result[0] : result;
+	if (!row || typeof row !== "object") return 0;
+	return Number(row[field] ?? 0);
+}
+function compactCacheDatabase(database, options = {}) {
+	const autoVacuum = pragmaNumber(database, "auto_vacuum", "auto_vacuum");
+	try {
+		if (autoVacuum === 2) {
+			database.pragma("incremental_vacuum");
+			database.pragma("wal_checkpoint(PASSIVE)");
+			return "incremental";
+		}
+		if (!options.allowFullVacuum) return "skipped";
+		database.pragma("busy_timeout = 100");
+		database.pragma("auto_vacuum = INCREMENTAL");
+		database.exec("VACUUM");
+		database.pragma("wal_checkpoint(PASSIVE)");
+		return "full";
+	} catch {
+		return "skipped";
+	} finally {
+		database.pragma("busy_timeout = 5000");
+	}
+}
+function runCacheMaintenance(database, options = {}) {
+	const now = options.now ?? Date.now();
+	const limits = {
+		...CACHE_LIMITS,
+		...options.limits
+	};
+	const removedByTable = Object.fromEntries(CACHE_POLICIES.map((policy) => [policy.table, 0]));
+	database.transaction(() => {
+		for (const policy of CACHE_POLICIES) removedByTable[policy.table] += deleteExpired(database, policy, now);
+	}).immediate();
+	for (const policy of CACHE_POLICIES) removedByTable[policy.table] += pruneToLimit(database, policy, limits[policy.table]);
+	const removed = Object.values(removedByTable).reduce((total, count) => total + count, 0);
+	const compaction = options.compact ? compactCacheDatabase(database, { allowFullVacuum: options.allowFullVacuum }) : "skipped";
+	const sizes = cacheLogicalSizes(database);
+	return {
+		removed,
+		removedByTable,
+		logicalSizeBytes: Object.values(sizes).reduce((total, size) => total + size, 0),
+		sizes,
+		compaction
+	};
+}
+//#endregion
 //#region packages/core/src/storage/sqlite.ts
 function withoutDriverMetadata(row) {
 	if (!row || typeof row !== "object" || Array.isArray(row)) return row;
@@ -1166,6 +1330,7 @@ CREATE TABLE IF NOT EXISTS semrush_cache (
   expires_at INTEGER,
   PRIMARY KEY(endpoint, query_hash)
 ) WITHOUT ROWID;
+CREATE INDEX IF NOT EXISTS idx_semrush_expires ON semrush_cache(expires_at);
 
 CREATE TABLE IF NOT EXISTS http_cache (
   url_hash TEXT PRIMARY KEY,
@@ -1178,6 +1343,8 @@ CREATE TABLE IF NOT EXISTS http_cache (
   fetched_at INTEGER,
   expires_at INTEGER
 );
+CREATE INDEX IF NOT EXISTS idx_http_expires ON http_cache(expires_at);
+CREATE INDEX IF NOT EXISTS idx_http_fetched ON http_cache(fetched_at);
 
 CREATE TABLE IF NOT EXISTS content_groups (
   id TEXT PRIMARY KEY,
@@ -1221,6 +1388,7 @@ CREATE TABLE IF NOT EXISTS crawl_reports (
   total_pages INTEGER NOT NULL,
   issue_count INTEGER NOT NULL,
   created_at INTEGER NOT NULL,
+  retention_class TEXT NOT NULL DEFAULT 'saved',
   report_json TEXT NOT NULL
 );
 CREATE INDEX IF NOT EXISTS idx_crawl_reports_latest ON crawl_reports(created_at DESC);
@@ -1364,23 +1532,28 @@ CREATE TABLE IF NOT EXISTS performance_reports (
   expires_at INTEGER NOT NULL
 );
 CREATE INDEX IF NOT EXISTS idx_performance_reports_url ON performance_reports(url, strategy, created_at);
+CREATE INDEX IF NOT EXISTS idx_performance_reports_expires ON performance_reports(expires_at);
 `;
 let db;
+let cacheWritesSinceMaintenance = 0;
+let cacheBytesSinceMaintenance = 0;
 function ensureColumn(database, table, column, definition) {
 	if (database.prepare(`PRAGMA table_info(${table})`).all().some((item) => item.name === column)) return;
 	database.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
 }
-function initDb(database) {
+function initDb(database, isNewDatabase) {
 	database.pragma("journal_mode = WAL");
 	database.pragma("synchronous = NORMAL");
 	database.pragma("foreign_keys = ON");
 	database.pragma("temp_store = MEMORY");
 	database.pragma("mmap_size = 268435456");
 	database.pragma("busy_timeout = 5000");
+	if (isNewDatabase) database.pragma("auto_vacuum = INCREMENTAL");
 	database.transaction(() => {
 		database.exec(CREATE_SQL);
 		ensureColumn(database, "crawl_pages", "snapshot_json", "TEXT");
 		ensureColumn(database, "http_cache", "metadata_json", "TEXT");
+		ensureColumn(database, "crawl_reports", "retention_class", "TEXT NOT NULL DEFAULT 'saved'");
 		ensureColumn(database, "index_watch_snapshots", "root_site_url", "TEXT");
 		ensureColumn(database, "index_watch_snapshots", "property_site_url", "TEXT");
 		ensureColumn(database, "index_watch_snapshots", "error_code", "TEXT");
@@ -1395,20 +1568,35 @@ function initDb(database) {
         ON index_watch_snapshots(root_site_url, url, inspected_at DESC, id DESC);
       CREATE INDEX IF NOT EXISTS idx_index_watch_property_url
         ON index_watch_snapshots(property_site_url, url, inspected_at DESC, id DESC);
+      CREATE INDEX IF NOT EXISTS idx_crawl_reports_retention
+        ON crawl_reports(retention_class, site_url, created_at DESC, id DESC);
     `);
 	}).immediate();
+	runCacheMaintenance(database, {
+		compact: true,
+		allowFullVacuum: true
+	});
 }
 function getDb() {
 	if (db) return db;
 	const path = getSeoCliPaths().cacheDbFile;
+	const isNewDatabase = !existsSync(path);
 	mkdirSync(dirname(path), {
 		recursive: true,
 		mode: 448
 	});
 	db = new SqliteDatabase(path);
-	initDb(db);
+	initDb(db, isNewDatabase);
 	return db;
 }
+function noteCacheWrite(sizeBytes = 0) {
+	cacheWritesSinceMaintenance += 1;
+	cacheBytesSinceMaintenance += Math.max(0, sizeBytes);
+	if (cacheWritesSinceMaintenance < 50 && cacheBytesSinceMaintenance < 16777216) return;
+	runCacheMaintenance(getDb());
+	cacheWritesSinceMaintenance = 0;
+	cacheBytesSinceMaintenance = 0;
+}
 function checkDatabaseReadiness() {
 	if (getDb().prepare("SELECT 1 AS ready").get()?.ready !== 1) throw new Error("The local SQLite database did not return a readiness row.");
 	return { dbPath: getSeoCliPaths().cacheDbFile };
@@ -1420,6 +1608,7 @@ function hashKey(parts) {
 function getCacheStats() {
 	const database = getDb();
 	const dbPath = getSeoCliPaths().cacheDbFile;
+	const logicalSizes = cacheLogicalSizes(database);
 	const counts = {
 		sites: database.prepare("SELECT COUNT(*) AS count FROM sites").get(),
 		gsc_cache: database.prepare("SELECT COUNT(*) AS count FROM gsc_cache").get(),
@@ -1429,10 +1618,19 @@ function getCacheStats() {
 	};
 	return {
 		dbPath,
-		sizeBytes: existsSync(dbPath) ? fileSize(dbPath) : 0,
+		sizeBytes: databaseFootprint(dbPath),
+		logicalSizeBytes: Object.values(logicalSizes).reduce((total, size) => total + size, 0),
+		maxSizeBytes: CACHE_MAX_BYTES,
 		counts: Object.fromEntries(Object.entries(counts).map(([key, value]) => [key, value.count]))
 	};
 }
+function databaseFootprint(path) {
+	return [
+		path,
+		`${path}-wal`,
+		`${path}-shm`
+	].reduce((total, file) => total + fileSize(file), 0);
+}
 function clearCache(provider, olderThanMs) {
 	const database = getDb();
 	const cutoff = olderThanMs ? Date.now() - olderThanMs : void 0;
@@ -1448,6 +1646,7 @@ function clearCache(provider, olderThanMs) {
 		const info = cutoff ? database.prepare(sql).run(cutoff) : database.prepare(sql).run();
 		removed += info.changes;
 	}
+	compactCacheDatabase(database, { allowFullVacuum: true });
 	return removed;
 }
 //#endregion
@@ -1484,9 +1683,14 @@ async function runGa4Report(propertyId, body, opts = {}) {
 		throw new Error(`Google Analytics report failed with ${response.status}.`);
 	}
 	const result = await response.json();
-	if (cacheable) db.prepare(`INSERT OR REPLACE INTO ga4_cache
+	if (cacheable) {
+		const requestJson = JSON.stringify(body);
+		const responseJson = JSON.stringify(result);
+		db.prepare(`INSERT OR REPLACE INTO ga4_cache
     (property_id, query_hash, request_json, response_json, row_count, fetched_at, expires_at)
-    VALUES (?, ?, ?, ?, ?, ?, ?)`).run(propertyId, queryHash, JSON.stringify(body), JSON.stringify(result), result.rowCount ?? result.rows?.length ?? 0, Date.now(), Date.now() + 864e5);
+    VALUES (?, ?, ?, ?, ?, ?, ?)`).run(propertyId, queryHash, requestJson, responseJson, result.rowCount ?? result.rows?.length ?? 0, Date.now(), Date.now() + 864e5);
+		noteCacheWrite(Buffer.byteLength(requestJson) + Buffer.byteLength(responseJson));
+	}
 	return result;
 }
 async function listGa4AccountSummaries() {
@@ -2178,7 +2382,7 @@ function effectiveSnippetControl(input) {
 }
 //#endregion
 //#region packages/core/src/extract/canonical.ts
-function headerValue$4(headers, name) {
+function headerValue$5(headers, name) {
 	const lower = name.toLowerCase();
 	return Object.entries(headers).find(([key]) => key.toLowerCase() === lower)?.[1];
 }
@@ -2262,7 +2466,7 @@ function parameter(value, name) {
 	return match?.[1] ?? match?.[2] ?? match?.[3];
 }
 function headerCandidates(headers, baseUrl) {
-	const link = headerValue$4(headers, "link");
+	const link = headerValue$5(headers, "link");
 	if (!link) return [];
 	const candidates = [];
 	for (const item of splitLinkHeader(link)) {
@@ -2279,14 +2483,14 @@ function headerCandidates(headers, baseUrl) {
 	}
 	return candidates;
 }
-function normalizedUrl$2(value) {
+function normalizedUrl$1(value) {
 	const url = new URL(value);
 	url.hash = "";
 	return url.toString();
 }
 function evidenceFromCandidates(candidates) {
 	const eligible = candidates.filter((candidate) => candidate.resolved && !candidate.ignoredReason);
-	const distinct = [...new Set(eligible.map((candidate) => normalizedUrl$2(candidate.resolved ?? "")))];
+	const distinct = [...new Set(eligible.map((candidate) => normalizedUrl$1(candidate.resolved ?? "")))];
 	if (distinct.length > 1) return {
 		status: "conflicting",
 		candidates
@@ -2931,7 +3135,7 @@ function safeText$1(value) {
 	const trimmed = value?.replace(/\s+/g, " ").trim();
 	return trimmed ? trimmed : void 0;
 }
-function headerValue$3(headers, name) {
+function headerValue$4(headers, name) {
 	const target = name.toLowerCase();
 	return Object.entries(headers).find(([key]) => key.toLowerCase() === target)?.[1];
 }
@@ -2949,8 +3153,7 @@ function httpUrl(value, base) {
 	const protocol = new URL(resolved).protocol;
 	return protocol === "http:" || protocol === "https:" ? resolved : void 0;
 }
-function sanitizedContentHtml(html, base) {
-	const $ = load(html);
+function sanitizedContentHtml($, html, base) {
 	let changed = false;
 	for (const element of $("[href], [src]").toArray()) for (const attribute of ["href", "src"]) {
 		const value = $(element).attr(attribute);
@@ -2961,6 +3164,22 @@ function sanitizedContentHtml(html, base) {
 	}
 	return changed ? $.html() : html;
 }
+function crawlerMainContent($, baseUrl) {
+	const text = ($("main").first().length ? $("main").first() : $("article").first().length ? $("article").first() : $("body").first()).text().replace(/\s+/g, " ").trim();
+	return {
+		text,
+		excerpt: safeText$1($("meta[name=\"description\"]").attr("content")),
+		wordCount: countCjkAwareWords(text),
+		diagnostics: {
+			requested: "crawler",
+			used: "crawler",
+			fallback: false,
+			wordCountSource: "local_cjk_aware",
+			baseUrl
+		},
+		warnings: []
+	};
+}
 function numericAttribute(value) {
 	if (!value) return void 0;
 	const match = value.trim().match(/^\d+/);
@@ -2985,12 +3204,10 @@ function imageDimensionsFromUrl(src) {
 	};
 }
 async function extractPage(fetchResult, extractor = "defuddle", dependencies = {}) {
-	const $ = load(fetchResult.html);
-	const content = extractMainContent({
-		...fetchResult,
-		html: sanitizedContentHtml(fetchResult.html, fetchResult.finalUrl)
-	}, extractor, dependencies);
-	const { text, excerpt } = content;
+	const $ = extractor === "crawler" && /<html(?:\s|>)/i.test(fetchResult.html) ? load(fetchResult.html, { xml: {
+		xmlMode: false,
+		decodeEntities: true
+	} }) : load(fetchResult.html);
 	const url = new URL(fetchResult.finalUrl);
 	const canonicalEvidence = extractCanonicalEvidence($, fetchResult.headers, fetchResult.finalUrl);
 	const headings = $("h1, h2, h3, h4, h5, h6").toArray().map((element) => ({
@@ -3072,13 +3289,18 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
 		const name = $(element).attr("name")?.trim().toLowerCase();
 		return name === "robots" || name === "googlebot";
 	}).map((element) => $(element).attr("content")));
+	const content = extractor === "crawler" ? crawlerMainContent($, fetchResult.finalUrl) : extractMainContent({
+		...fetchResult,
+		html: sanitizedContentHtml($, fetchResult.html, fetchResult.finalUrl)
+	}, extractor, dependencies);
+	const { text, excerpt } = content;
 	return {
 		url: fetchResult.url,
 		finalUrl: fetchResult.finalUrl,
 		title: safeText$1($("title").first().text()),
 		metaDescription: safeText$1($("meta[name=\"description\"]").attr("content")),
 		metaRobots,
-		xRobotsTag: safeText$1(headerValue$3(fetchResult.headers, "x-robots-tag")),
+		xRobotsTag: safeText$1(headerValue$4(fetchResult.headers, "x-robots-tag")),
 		canonical: canonicalEvidence.selectedRaw,
 		canonicalEvidence,
 		lang: safeText$1($("html").attr("lang")),
@@ -3129,14 +3351,108 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
 	};
 }
 //#endregion
+//#region packages/core/src/version.ts
+const SEO_VERSION = version;
+//#endregion
+//#region packages/core/src/fetch/crawler-identity.ts
+const SEO_CRAWLER_TOKEN = "SEO-Skill";
+const SEO_CRAWLER_USER_AGENT = `${SEO_CRAWLER_TOKEN}/${SEO_VERSION} (+https://seoskill.dev)`;
+const SEO_CRAWLER_IDENTITY = {
+	name: "SEO Skill",
+	robotsToken: SEO_CRAWLER_TOKEN,
+	version: SEO_VERSION,
+	userAgent: SEO_CRAWLER_USER_AGENT,
+	documentationUrl: "https://seoskill.dev/docs/crawler/"
+};
+//#endregion
+//#region packages/core/src/fetch/access-block.ts
+const CLOUDFLARE_SKIP_DOCS = "https://developers.cloudflare.com/waf/custom-rules/skip/";
+function isHeadersLike(headers) {
+	return "get" in headers && typeof headers.get === "function";
+}
+function headerValue$3(headers, name) {
+	if (isHeadersLike(headers)) return headers.get(name) ?? void 0;
+	const lower = name.toLowerCase();
+	return headers[name] ?? headers[lower] ?? Object.entries(headers).find(([key]) => key.toLowerCase() === lower)?.[1];
+}
+function cloudflareEvidence(headers) {
+	const mitigated = headerValue$3(headers, "cf-mitigated");
+	const rayId = headerValue$3(headers, "cf-ray");
+	const server = headerValue$3(headers, "server");
+	const challenge = mitigated?.toLowerCase() === "challenge";
+	const indicators = [
+		...challenge ? ["cf-mitigated: challenge"] : [],
+		...rayId ? [`cf-ray: ${rayId}`] : [],
+		...server?.toLowerCase().includes("cloudflare") ? [`server: ${server}`] : []
+	];
+	return {
+		detected: challenge || Boolean(rayId) || Boolean(indicators.length),
+		challenge,
+		rayId,
+		indicators
+	};
+}
+function blockKind(status, cloudflareChallenge) {
+	if (cloudflareChallenge) return "challenge";
+	if (status === 429) return "rate-limit";
+	if (status === 401) return "authentication";
+	return "firewall";
+}
+function detectAccessBlock(input) {
+	const cloudflare = cloudflareEvidence(input.headers);
+	if (![
+		401,
+		403,
+		429
+	].includes(input.status) && !cloudflare.challenge) return void 0;
+	const provider = cloudflare.detected ? "cloudflare" : "unknown";
+	const kind = blockKind(input.status, cloudflare.challenge);
+	const indicators = [`HTTP ${input.status}`, ...cloudflare.indicators];
+	if (provider === "cloudflare") return {
+		provider,
+		kind,
+		status: input.status,
+		crawler: SEO_CRAWLER_IDENTITY,
+		indicators,
+		...cloudflare.rayId ? { requestId: cloudflare.rayId } : {},
+		guidance: {
+			summary: kind === "challenge" ? "Cloudflare returned a Challenge Page to the crawler." : "The crawler received a denied or rate-limited response through Cloudflare.",
+			recommendedAction: "Look for this request in Cloudflare Security Events. If Cloudflare blocked it, create the narrowest custom Skip rule that matches the audit machine source IP, hostname or required paths, and this exact User-Agent. Skip only the product or rule that caused the block. If there is no matching event, inspect the origin security logs.",
+			securityNote: "A User-Agent can be spoofed. Do not trust it by itself or create a broad site-wide bypass. Remove temporary access after the audit. Bot Fight Mode cannot be skipped by a custom rule.",
+			documentationUrl: CLOUDFLARE_SKIP_DOCS
+		}
+	};
+	return {
+		provider,
+		kind,
+		status: input.status,
+		crawler: SEO_CRAWLER_IDENTITY,
+		indicators,
+		guidance: {
+			summary: kind === "rate-limit" ? "The site rate-limited the crawler request." : "The site denied the crawler request.",
+			recommendedAction: "Inspect the site, CDN, and origin security logs for this request. If the audit should have access, allow the audit machine source IP and this exact User-Agent only on the public paths being checked, or lower the crawl rate.",
+			securityNote: "A User-Agent can be spoofed. Do not trust it by itself or create a broad site-wide bypass. Remove temporary access after the audit."
+		}
+	};
+}
+//#endregion
 //#region packages/core/src/fetch/http-client.ts
-const BROWSER_USER_AGENT = new UserAgent({ deviceCategory: "desktop" }).toString();
+var ResponseSizeLimitError = class extends Error {
+	maxBytes;
+	constructor(maxBytes, label = "Response") {
+		super(`${label} exceeds the ${maxBytes}-byte response limit.`);
+		this.maxBytes = maxBytes;
+		this.name = "ResponseSizeLimitError";
+	}
+};
+/** @deprecated Use SEO_CRAWLER_USER_AGENT. */
+const BROWSER_USER_AGENT = SEO_CRAWLER_USER_AGENT;
 function requestHeaders(profile) {
 	if (profile === "bot") return new Headers({
 		accept: "text/plain,*/*;q=0.8",
 		"accept-language": "en-US,en;q=0.9",
 		connection: "close",
-		"user-agent": BROWSER_USER_AGENT
+		"user-agent": SEO_CRAWLER_USER_AGENT
 	});
 	return new Headers({
 		accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
@@ -3145,7 +3461,7 @@ function requestHeaders(profile) {
 		"sec-ch-ua-mobile": "?0",
 		"sec-ch-ua-platform": "\"macOS\"",
 		"upgrade-insecure-requests": "1",
-		"user-agent": BROWSER_USER_AGENT
+		"user-agent": SEO_CRAWLER_USER_AGENT
 	});
 }
 function publicHttpFetch(url, input = {}) {
@@ -3161,6 +3477,32 @@ function publicHttpFetch(url, input = {}) {
 		headers: base
 	});
 }
+async function readBoundedResponseText(response, maxBytes, label = "Response") {
+	const declaredLength = Number(response.headers.get("content-length"));
+	if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
+		await response.body?.cancel().catch(() => void 0);
+		throw new ResponseSizeLimitError(maxBytes, label);
+	}
+	const reader = response.body?.getReader();
+	if (!reader) return "";
+	const chunks = [];
+	let total = 0;
+	try {
+		while (true) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			total += value.byteLength;
+			if (total > maxBytes) {
+				await reader.cancel().catch(() => void 0);
+				throw new ResponseSizeLimitError(maxBytes, label);
+			}
+			chunks.push(value);
+		}
+	} finally {
+		reader.releaseLock();
+	}
+	return Buffer.concat(chunks, total).toString("utf8");
+}
 //#endregion
 //#region packages/core/src/fetch/page-fetcher/rate-controls.ts
 const HOST_QUEUES = /* @__PURE__ */ new Map();
@@ -3217,8 +3559,10 @@ function snapshotForHost(host, status) {
 	};
 }
 function normalizeRateControls(rate) {
+	const concurrency = rate?.concurrency ?? numberEnv("SEO_FETCH_CONCURRENCY", 4);
+	if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 16) throw new RangeError(`Fetch concurrency must be an integer from 1 to 16.`);
 	return {
-		concurrency: rate?.concurrency ?? numberEnv("SEO_FETCH_CONCURRENCY", 4),
+		concurrency,
 		intervalCap: rate?.intervalCap ?? numberEnv("SEO_FETCH_INTERVAL_CAP", 4),
 		intervalMs: rate?.intervalMs ?? numberEnv("SEO_FETCH_INTERVAL_MS", 1e3),
 		backpressure: {
@@ -3306,7 +3650,34 @@ function recordHostFetch(input) {
 	return snapshotForHost(input.host);
 }
 //#endregion
+//#region packages/core/src/fetch/page-fetcher/redirects.ts
+async function fetchWithRedirectChain(url, signal) {
+	const redirectChain = [];
+	let currentUrl = url;
+	for (let redirectCount = 0; redirectCount <= 10; redirectCount += 1) {
+		const response = await publicHttpFetch(currentUrl, {
+			redirect: "manual",
+			signal
+		});
+		const location = response.headers.get("location");
+		if (response.status < 300 || response.status >= 400 || !location) return {
+			response,
+			redirectChain
+		};
+		const nextUrl = new URL(location, currentUrl).toString();
+		redirectChain.push({
+			url: currentUrl,
+			status: response.status,
+			location: nextUrl
+		});
+		await response.body?.cancel().catch(() => void 0);
+		currentUrl = nextUrl;
+	}
+	throw new Error(`Too many redirects for ${url}`);
+}
+//#endregion
 //#region packages/core/src/fetch/page-fetcher/robots.ts
+const MAX_ROBOTS_RESPONSE_BYTES = 1024 * 1024;
 var RobotsAccessError = class extends Error {
 	reason;
 	evidence;
@@ -3326,10 +3697,10 @@ function cacheableStatus(status) {
 function resultFromResponse(input) {
 	if (input.status >= 200 && input.status < 300) {
 		const parsed = parseRobots$1(input.robotsUrl, input.text);
-		const lineNumber = parsed.getMatchingLineNumber?.(input.targetUrl, BROWSER_USER_AGENT);
+		const lineNumber = parsed.getMatchingLineNumber?.(input.targetUrl, SEO_CRAWLER_USER_AGENT);
 		const matchedLine = lineNumber ? input.text.split(/\r?\n/)[lineNumber - 1]?.trim() : void 0;
 		return {
-			allowed: parsed.isAllowed(input.targetUrl, BROWSER_USER_AGENT) ?? true,
+			allowed: parsed.isAllowed(input.targetUrl, SEO_CRAWLER_USER_AGENT) ?? true,
 			availability: "available",
 			status: input.status,
 			cache: input.cache,
@@ -3353,44 +3724,105 @@ function resultFromResponse(input) {
 		url: input.robotsUrl
 	};
 }
-async function fetchRobots(origin, targetUrl = origin, refresh = false) {
-	const db = getDb();
-	const robotsUrl = new URL("/robots.txt", origin).toString();
-	const key = hashKey(["robots", robotsUrl]);
-	const cached = db.prepare("SELECT status, body_blob, expires_at FROM http_cache WHERE url_hash = ?").get(key);
-	if (!refresh && cached?.body_blob && cached.status !== void 0 && cacheableStatus(cached.status) && cached.expires_at && cached.expires_at > Date.now()) return resultFromResponse({
-		robotsUrl,
+function resultFromSource(source, targetUrl) {
+	if (source.status !== void 0 && source.text !== void 0) return resultFromResponse({
+		robotsUrl: source.robotsUrl,
 		targetUrl,
-		status: cached.status,
-		text: cached.body_blob.toString("utf8"),
-		cache: "hit"
+		status: source.status,
+		text: source.text,
+		cache: source.cache
 	});
+	return {
+		allowed: null,
+		availability: "unreachable",
+		error: source.error ?? "robots.txt could not be fetched.",
+		cache: source.cache,
+		url: source.robotsUrl
+	};
+}
+async function fetchRobotsSource(origin, refresh, options) {
+	const robotsUrl = new URL("/robots.txt", origin).toString();
+	const key = hashKey(["robots", robotsUrl]);
+	if (!refresh) {
+		const cached = getDb().prepare("SELECT status, body_blob, expires_at FROM http_cache WHERE url_hash = ?").get(key);
+		if (cached?.body_blob && cached.status !== void 0 && cacheableStatus(cached.status) && cached.expires_at && cached.expires_at > Date.now()) return {
+			robotsUrl,
+			status: cached.status,
+			text: cached.body_blob.toString("utf8"),
+			cache: "hit"
+		};
+	}
+	const controller = new AbortController();
+	const abort = () => controller.abort();
+	const timer = setTimeout(abort, options.timeoutMs ?? 2e4);
+	if (options.signal?.aborted) controller.abort();
+	else options.signal?.addEventListener("abort", abort, { once: true });
 	try {
-		const response = await publicHttpFetch(robotsUrl, { profile: "bot" });
-		const text = await response.text();
-		const result = resultFromResponse({
+		const response = await publicHttpFetch(robotsUrl, {
+			profile: "bot",
+			signal: controller.signal
+		});
+		const text = await readBoundedResponseText(response, MAX_ROBOTS_RESPONSE_BYTES, "robots.txt response");
+		if (options.writeCache !== false && cacheableStatus(response.status)) {
+			getDb().prepare(`INSERT OR REPLACE INTO http_cache
+          (url_hash, url, status, headers_json, body_blob, etag, fetched_at, expires_at)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(key, robotsUrl, response.status, "{}", Buffer.from(text), null, Date.now(), Date.now() + 864e5);
+			noteCacheWrite(Buffer.byteLength(text));
+		}
+		return {
 			robotsUrl,
-			targetUrl,
 			status: response.status,
 			text,
 			cache: refresh ? "bypass" : "miss"
-		});
-		if (cacheableStatus(response.status)) db.prepare(`INSERT OR REPLACE INTO http_cache
-        (url_hash, url, status, headers_json, body_blob, etag, fetched_at, expires_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(key, robotsUrl, response.status, "{}", Buffer.from(text), null, Date.now(), Date.now() + 864e5);
-		return result;
+		};
 	} catch (error) {
 		return {
-			allowed: null,
-			availability: "unreachable",
+			robotsUrl,
 			error: error instanceof Error ? error.message : String(error),
-			cache: refresh ? "bypass" : "miss",
-			url: robotsUrl
+			cache: refresh ? "bypass" : "miss"
 		};
-	}
+	} finally {
+		clearTimeout(timer);
+		options.signal?.removeEventListener("abort", abort);
+	}
+}
+async function fetchRobots(origin, targetUrl = origin, refresh = false, options = {}) {
+	return resultFromSource(await fetchRobotsSource(origin, refresh, options), targetUrl);
+}
+function createRobotsSession(input = {}) {
+	const sources = /* @__PURE__ */ new Map();
+	const sourceFor = (origin) => {
+		const normalizedOrigin = new URL(origin).origin;
+		let source = sources.get(normalizedOrigin);
+		if (!source) {
+			source = fetchRobotsSource(normalizedOrigin, input.refresh ?? false, {
+				writeCache: input.writeCache,
+				timeoutMs: input.timeoutMs,
+				signal: input.signal
+			});
+			sources.set(normalizedOrigin, source);
+		}
+		return source;
+	};
+	return {
+		resolve: async (origin, targetUrl) => resultFromSource(await sourceFor(origin), targetUrl),
+		sitemapUrls: async (origin) => {
+			const source = await sourceFor(origin);
+			if (source.status === void 0 || source.status >= 300 || !source.text) return [];
+			const urls = /* @__PURE__ */ new Set();
+			for (const match of source.text.matchAll(/^\s*sitemap\s*:\s*(\S+)\s*$/gim)) {
+				const value = match[1]?.trim();
+				if (!value) continue;
+				try {
+					urls.add(new URL(value).toString());
+				} catch {}
+			}
+			return [...urls].sort();
+		}
+	};
 }
 //#endregion
-//#region packages/core/src/fetch/page-fetcher/plain.ts
+//#region packages/core/src/fetch/page-fetcher/robots-evidence.ts
 function pageRobotsEvidence(robots) {
 	return {
 		url: robots.url,
@@ -3411,10 +3843,14 @@ function diagnosticRobotsEvidence(robots) {
 		error: robots.error
 	};
 }
+//#endregion
+//#region packages/core/src/fetch/page-fetcher/plain.ts
+const MAX_PAGE_RESPONSE_BYTES = 5 * 1024 * 1024;
 function encodePageFetchCacheEvidence(result) {
 	return JSON.stringify({
 		finalUrl: result.finalUrl,
 		blocked: result.diagnostics.blocked,
+		accessBlock: result.diagnostics.accessBlock,
 		robotsTxt: result.robotsTxt,
 		diagnosticsRobotsTxt: result.diagnostics.robotsTxt,
 		redirectChain: result.diagnostics.redirectChain,
@@ -3431,36 +3867,15 @@ function decodePageFetchCacheEvidence(value) {
 		return;
 	}
 }
-async function fetchWithRedirectChain(url, signal) {
-	const redirectChain = [];
-	let currentUrl = url;
-	for (let redirectCount = 0; redirectCount <= 10; redirectCount += 1) {
-		const response = await publicHttpFetch(currentUrl, {
-			redirect: "manual",
-			signal
-		});
-		const location = response.headers.get("location");
-		if (response.status < 300 || response.status >= 400 || !location) return {
-			response,
-			redirectChain
-		};
-		const nextUrl = new URL(location, currentUrl).toString();
-		redirectChain.push({
-			url: currentUrl,
-			status: response.status,
-			location: nextUrl
-		});
-		await response.body?.cancel().catch(() => void 0);
-		currentUrl = nextUrl;
-	}
-	throw new Error(`Too many redirects for ${url}`);
-}
 async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, respectRobots = false) {
 	const startedAt = Date.now();
 	const db = getDb();
 	const key = hashKey(["page", url]);
 	const host = new URL(url).host;
-	let robots = respectRobots ? await fetchRobots(new URL(url).origin, url, refresh) : void 0;
+	let robots = respectRobots ? await fetchRobots(new URL(url).origin, url, refresh, {
+		timeoutMs,
+		signal
+	}) : void 0;
 	if (robots && robots.allowed !== true) throw new RobotsAccessError(robots.allowed === false ? "robots-disallowed" : "robots-deferred", robots);
 	const cached = db.prepare("SELECT status, headers_json, body_blob, metadata_json, expires_at FROM http_cache WHERE url_hash = ? AND expires_at > ?").get(key, Date.now());
 	if (!refresh && cached) {
@@ -3486,6 +3901,7 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
 				retries: 0,
 				rateLimit: { ...rateLimitDiagnostics(host, rate) },
 				backpressure: hostBackpressureSnapshot(host),
+				accessBlock: evidence?.accessBlock,
 				robotsTxt: robots ? diagnosticRobotsEvidence(robots) : evidence?.diagnosticsRobotsTxt ? {
 					...evidence.diagnosticsRobotsTxt,
 					cache: "hit"
@@ -3496,7 +3912,10 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
 			robotsTxt: robots ? pageRobotsEvidence(robots) : evidence?.robotsTxt
 		};
 	}
-	robots ??= await fetchRobots(new URL(url).origin, url, refresh);
+	robots ??= await fetchRobots(new URL(url).origin, url, refresh, {
+		timeoutMs,
+		signal
+	});
 	const beforeFetch = await waitForHostBackpressure(host, rate);
 	let attempts = 0;
 	const response = await pRetry(async () => {
@@ -3516,8 +3935,14 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
 			signal?.removeEventListener("abort", abort);
 		}
 	}, { retries: 2 });
-	const html = await response.response.text();
 	const headerMap = Object.fromEntries(response.response.headers.entries());
+	const accessBlock = detectAccessBlock({
+		status: response.response.status,
+		headers: response.response.headers
+	});
+	let html = "";
+	if (accessBlock) await response.response.body?.cancel().catch(() => void 0);
+	else html = await readBoundedResponseText(response.response, MAX_PAGE_RESPONSE_BYTES, "Page response");
 	const durationMs = Date.now() - startedAt;
 	const backpressure = recordHostFetch({
 		host,
@@ -3538,7 +3963,7 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
 			cache: refresh ? "bypass" : "miss",
 			fetched: true,
 			rendered: false,
-			blocked: robots.allowed === false || [
+			blocked: robots.allowed === false || Boolean(accessBlock) || [
 				401,
 				403,
 				429
@@ -3547,15 +3972,19 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
 			retries: Math.max(0, attempts - 1),
 			rateLimit: { ...rateLimitDiagnostics(host, rate) },
 			backpressure: backpressure.status === "ok" && beforeFetch.status !== "ok" ? beforeFetch : backpressure,
+			accessBlock,
 			robotsTxt: diagnosticRobotsEvidence(robots),
 			redirectChain: response.redirectChain
 		},
-		warnings: robots.allowed === null ? [`robots.txt availability is unknown${robots.status ? ` after HTTP ${robots.status}` : ""}; this fetch does not prove crawler access is allowed.`] : [],
+		warnings: [...robots.allowed === null ? [`robots.txt availability is unknown${robots.status ? ` after HTTP ${robots.status}` : ""}; this fetch does not prove crawler access is allowed.`] : [], ...accessBlock ? [`${accessBlock.guidance.summary} Identify the crawler as ${accessBlock.crawler.userAgent}.`] : []],
 		robotsTxt: pageRobotsEvidence(robots)
 	};
-	db.prepare(`INSERT OR REPLACE INTO http_cache
-    (url_hash, url, status, headers_json, body_blob, metadata_json, etag, fetched_at, expires_at)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(key, url, response.response.status, JSON.stringify(headerMap), Buffer.from(html), encodePageFetchCacheEvidence(result), response.response.headers.get("etag"), Date.now(), Date.now() + 36e5);
+	if (!accessBlock) {
+		db.prepare(`INSERT OR REPLACE INTO http_cache
+      (url_hash, url, status, headers_json, body_blob, metadata_json, etag, fetched_at, expires_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(key, url, response.response.status, JSON.stringify(headerMap), Buffer.from(html), encodePageFetchCacheEvidence(result), response.response.headers.get("etag"), Date.now(), Date.now() + 36e5);
+		noteCacheWrite(Buffer.byteLength(html));
+	}
 	return result;
 }
 //#endregion
@@ -3768,7 +4197,7 @@ async function renderPage(input) {
 	const beforeFetch = await waitForHostBackpressure(host, input.rate);
 	const timeoutMs = input.timeoutMs ?? DEFAULT_RENDER_TIMEOUT_MS;
 	const context = await input.browser.browser.newContext({
-		userAgent: BROWSER_USER_AGENT,
+		userAgent: SEO_CRAWLER_USER_AGENT,
 		viewport: {
 			width: 1440,
 			height: 900
@@ -3797,6 +4226,11 @@ async function renderPage(input) {
 		});
 		const html = await page.content();
 		const status = response?.status() ?? 200;
+		const headers = response?.headers() ?? {};
+		const accessBlock = detectAccessBlock({
+			status,
+			headers
+		});
 		const durationMs = Date.now() - startedAt;
 		const backpressure = recordHostFetch({
 			host,
@@ -3809,7 +4243,7 @@ async function renderPage(input) {
 			url: input.url,
 			finalUrl: page.url(),
 			status,
-			headers: response?.headers() ?? {},
+			headers,
 			html,
 			usedJs: true,
 			diagnostics: {
@@ -3817,7 +4251,7 @@ async function renderPage(input) {
 				cache: "bypass",
 				fetched: true,
 				rendered: true,
-				blocked: [
+				blocked: Boolean(accessBlock) || [
 					401,
 					403,
 					429
@@ -3826,6 +4260,7 @@ async function renderPage(input) {
 				retries: 0,
 				rateLimit: rateLimitDiagnostics(host, input.rate),
 				backpressure: backpressure.status === "ok" && beforeFetch.status !== "ok" ? beforeFetch : backpressure,
+				accessBlock,
 				rendering: {
 					mode: "on",
 					status: "rendered",
@@ -3842,7 +4277,7 @@ async function renderPage(input) {
 					...renderDiagnostics
 				}
 			},
-			warnings: networkIdleReached ? [] : [`Rendered page did not reach network idle within ${Math.min(NETWORK_IDLE_TIMEOUT_MS, timeoutMs)}ms; captured DOM after DOMContentLoaded.`]
+			warnings: [...networkIdleReached ? [] : [`Rendered page did not reach network idle within ${Math.min(NETWORK_IDLE_TIMEOUT_MS, timeoutMs)}ms; captured DOM after DOMContentLoaded.`], ...accessBlock ? [`${accessBlock.guidance.summary} Identify the crawler as ${accessBlock.crawler.userAgent}.`] : []]
 		};
 	} catch (error) {
 		if (error instanceof JavaScriptRenderingError) throw error;
@@ -3989,6 +4424,85 @@ function renderingDocumentDifference(raw, rendered) {
 	};
 }
 //#endregion
+//#region packages/core/src/fetch/page-fetcher/status-probe.ts
+async function statusProbe(url, opts) {
+	const startedAt = Date.now();
+	const host = new URL(url).host;
+	const rate = normalizeRateControls(opts.rate);
+	const origin = new URL(url).origin;
+	const robots = opts.robotsResolver ? await opts.robotsResolver(origin, url) : await fetchRobots(origin, url, opts.refresh ?? false, {
+		writeCache: false,
+		timeoutMs: opts.timeoutMs,
+		signal: opts.signal
+	});
+	if (opts.respectRobots && robots.allowed !== true) throw new RobotsAccessError(robots.allowed === false ? "robots-disallowed" : "robots-deferred", robots);
+	const beforeFetch = await waitForHostBackpressure(host, rate);
+	let attempts = 0;
+	const result = await pRetry(async () => {
+		attempts += 1;
+		const controller = new AbortController();
+		const abort = () => controller.abort();
+		const timer = setTimeout(abort, opts.timeoutMs ?? 2e4);
+		if (opts.signal?.aborted) controller.abort();
+		else opts.signal?.addEventListener("abort", abort, { once: true });
+		try {
+			return await fetchWithRedirectChain(url, controller.signal);
+		} catch (error) {
+			if (opts.signal?.aborted) throw new AbortError(error instanceof Error ? error : new Error(String(error)));
+			throw error;
+		} finally {
+			clearTimeout(timer);
+			opts.signal?.removeEventListener("abort", abort);
+		}
+	}, { retries: 1 });
+	const headers = Object.fromEntries(result.response.headers.entries());
+	const accessBlock = detectAccessBlock({
+		status: result.response.status,
+		headers: result.response.headers
+	});
+	await result.response.body?.cancel().catch(() => void 0);
+	const durationMs = Date.now() - startedAt;
+	const backpressure = recordHostFetch({
+		host,
+		status: result.response.status,
+		durationMs,
+		retryAfterMs: retryAfterMs(result.response.headers.get("retry-after")),
+		rate
+	});
+	return {
+		url,
+		finalUrl: result.response.url || url,
+		status: result.response.status,
+		headers,
+		html: "",
+		usedJs: false,
+		diagnostics: {
+			source: "network",
+			cache: "bypass",
+			fetched: true,
+			rendered: false,
+			blocked: robots.allowed === false || Boolean(accessBlock) || [
+				401,
+				403,
+				429
+			].includes(result.response.status),
+			durationMs,
+			retries: Math.max(0, attempts - 1),
+			rateLimit: rateLimitDiagnostics(host, rate),
+			backpressure: backpressure.status === "ok" && beforeFetch.status !== "ok" ? beforeFetch : backpressure,
+			accessBlock,
+			robotsTxt: diagnosticRobotsEvidence(robots),
+			redirectChain: result.redirectChain
+		},
+		warnings: ["Status-only health probe. The response body was not downloaded, parsed, rendered, or cached.", ...accessBlock ? [`${accessBlock.guidance.summary} Identify the crawler as ${accessBlock.crawler.userAgent}.`] : []],
+		robotsTxt: pageRobotsEvidence(robots)
+	};
+}
+async function fetchPageStatus(url, opts = {}) {
+	const rate = normalizeRateControls(opts.rate);
+	return await queueForHost(new URL(url).host, rate).add(() => statusProbe(url, opts));
+}
+//#endregion
 //#region packages/core/src/fetch/page-fetcher.ts
 function normalizeJavaScriptRenderingMode(input) {
 	if (input === true) return "on";
@@ -4034,7 +4548,7 @@ async function fetchPage(url, opts = {}, dependencies = {}) {
 	const rate = normalizeRateControls(opts.rate);
 	return await queueForHost(new URL(url).host, rate).add(async () => {
 		const first = await plainFetch(url, opts.refresh, opts.timeoutMs, rate, opts.signal, opts.respectRobots);
-		if (!(renderingMode === "on" || renderingMode === "auto" && looksLikeSpa(first.html)) || opts.signal?.aborted) return withRenderingState(first, renderingMode, opts.signal?.aborted ? "skipped" : renderingMode === "off" ? "not-requested" : "not-needed");
+		if (!(!first.diagnostics.accessBlock && (renderingMode === "on" || renderingMode === "auto" && looksLikeSpa(first.html))) || opts.signal?.aborted) return withRenderingState(first, renderingMode, opts.signal?.aborted ? "skipped" : renderingMode === "off" ? "not-requested" : "not-needed");
 		const ownsRenderer = !opts.renderer;
 		const renderer = opts.renderer ?? makeRenderer();
 		try {
@@ -4048,6 +4562,7 @@ async function fetchPage(url, opts = {}, dependencies = {}) {
 					...rendered.diagnostics,
 					robotsTxt: first.diagnostics.robotsTxt,
 					redirectChain: first.diagnostics.redirectChain,
+					accessBlock: rendered.diagnostics.accessBlock ?? first.diagnostics.accessBlock,
 					rendering: {
 						...rendered.diagnostics.rendering,
 						mode: renderingMode,
@@ -4423,9 +4938,12 @@ async function querySearchAnalytics(site, body, opts = {}) {
 		if (rows.length < rowLimit || typeof maxRows === "number" && allRows.length >= maxRows) break;
 		startRow += rowLimit;
 	}
+	const requestJson = JSON.stringify(body);
+	const responseJson = JSON.stringify(allRows);
 	db.prepare(`INSERT OR REPLACE INTO gsc_cache
     (site_url, query_hash, request_json, response_json, row_count, fetched_at, expires_at)
-    VALUES (?, ?, ?, ?, ?, ?, ?)`).run(site, queryHash, JSON.stringify(body), JSON.stringify(allRows), allRows.length, Date.now(), Date.now() + 864e5);
+    VALUES (?, ?, ?, ?, ?, ?, ?)`).run(site, queryHash, requestJson, responseJson, allRows.length, Date.now(), Date.now() + 864e5);
+	noteCacheWrite(Buffer.byteLength(requestJson) + Buffer.byteLength(responseJson));
 	return {
 		rows: allRows,
 		calls,
@@ -4570,7 +5088,7 @@ async function listSites(_refresh = false) {
 }
 //#endregion
 //#region packages/core/src/analyze/page-technical-signals.ts
-function normalizedUrl$1(value) {
+function normalizePageUrl(value) {
 	try {
 		const parsed = new URL(value);
 		parsed.hash = "";
@@ -4581,7 +5099,7 @@ function normalizedUrl$1(value) {
 	}
 }
 function samePageUrl(left, right) {
-	return normalizedUrl$1(left) === normalizedUrl$1(right);
+	return normalizePageUrl(left) === normalizePageUrl(right);
 }
 function absoluteUrl$1(value, base) {
 	try {
@@ -5036,12 +5554,13 @@ const CTR_BASELINE = {
 	9: .015,
 	10: .012
 };
+const WORD_SEGMENTER = typeof Intl !== "undefined" && "Segmenter" in Intl ? new Intl.Segmenter("und", { granularity: "word" }) : void 0;
 function normalizeText$1(value) {
 	return value.normalize("NFKD").replace(/\p{Diacritic}/gu, "").replace(/[’']/g, "'").toLowerCase().replace(/\s+/g, " ").trim();
 }
 function unicodeTokens(value) {
 	const normalized = normalizeText$1(value);
-	if (typeof Intl !== "undefined" && "Segmenter" in Intl) return [...new Intl.Segmenter("und", { granularity: "word" }).segment(normalized)].filter((part) => part.isWordLike).map((part) => part.segment.trim()).filter((token) => token.length > 1);
+	if (WORD_SEGMENTER) return [...WORD_SEGMENTER.segment(normalized)].filter((part) => part.isWordLike).map((part) => part.segment.trim()).filter((token) => token.length > 1);
 	return (normalized.match(/[\p{L}\p{N}]+/gu) ?? []).filter((token) => token.length > 1);
 }
 function tokenize(value) {
@@ -5688,6 +6207,7 @@ const MIN_BENCHMARK_URL_SAMPLES = 5;
 const MIN_POSITIVE_URL_SAMPLES = 3;
 const SITE_BENCHMARK_FLOOR_MULTIPLIER = .5;
 const SITE_BENCHMARK_CAP_MULTIPLIER = 1.5;
+const URL_BENCHMARK_CACHE_LIMIT = 1e4;
 function roundedPosition(position) {
 	return Math.max(1, Math.min(10, Math.round(position)));
 }
@@ -5704,7 +6224,8 @@ function bucketBenchmark(input) {
 	const impressions = peerRows.reduce((sum, row) => sum + row.impressions, 0);
 	const byUrl = /* @__PURE__ */ new Map();
 	for (const row of peerRows) {
-		const key = row.keys?.[1] || `${row.keys?.[0] ?? "query"}:${row.position}`;
+		const rawKey = row.keys?.[1] || `${row.keys?.[0] ?? "query"}:${row.position}`;
+		const key = row.keys?.[1] ? normalizePageUrl(rawKey) : rawKey;
 		const current = byUrl.get(key) ?? {
 			clicks: 0,
 			impressions: 0
@@ -5738,10 +6259,81 @@ function bucketBenchmark(input) {
 		positiveUrlSamples: positiveUrlCtrSamples.length
 	};
 }
-function smoothMonotonicBenchmarks(input) {
-	const byPosition = {};
-	let previousCtr = Number.POSITIVE_INFINITY;
-	for (const position of Object.keys(CTR_BASELINE).map(Number)) {
+function indexedPositionBucket(rows) {
+	const byUrl = /* @__PURE__ */ new Map();
+	let impressions = 0;
+	for (const row of rows) {
+		impressions += row.impressions;
+		const rawKey = row.keys?.[1] || `${row.keys?.[0] ?? "query"}:${row.position}`;
+		const key = row.keys?.[1] ? normalizePageUrl(rawKey) : rawKey;
+		const current = byUrl.get(key) ?? {
+			key,
+			clicks: 0,
+			impressions: 0,
+			rows: 0,
+			ctr: 0
+		};
+		current.clicks += row.clicks;
+		current.impressions += row.impressions;
+		current.rows += 1;
+		current.ctr = current.impressions ? current.clicks / current.impressions : 0;
+		byUrl.set(key, current);
+	}
+	const compareSample = (left, right) => left.ctr - right.ctr || (left.key < right.key ? -1 : left.key > right.key ? 1 : 0);
+	const qualified = [...byUrl.values()].filter((sample) => sample.impressions >= MIN_BENCHMARK_URL_IMPRESSIONS).sort(compareSample);
+	const positive = qualified.filter((sample) => sample.ctr > 0).sort(compareSample);
+	return {
+		rows: rows.length,
+		impressions,
+		byUrl,
+		qualified,
+		qualifiedIndexes: new Map(qualified.map((sample, index) => [sample.key, index])),
+		positive,
+		positiveIndexes: new Map(positive.map((sample, index) => [sample.key, index])),
+		qualifiedImpressions: qualified.reduce((sum, sample) => sum + sample.impressions, 0)
+	};
+}
+function indexedPercentile(samples, indexes, excludedKey, percentileValue) {
+	const excludedIndex = indexes.get(excludedKey);
+	const length = samples.length - (excludedIndex === void 0 ? 0 : 1);
+	if (length === 0) return void 0;
+	const rank = Math.min(length - 1, Math.max(0, Math.ceil(percentileValue / 100 * length) - 1));
+	return samples[excludedIndex !== void 0 && rank >= excludedIndex ? rank + 1 : rank]?.ctr;
+}
+function indexedBucketBenchmark(input) {
+	const excluded = input.bucket.byUrl.get(input.excludedKey);
+	const excludesQualified = excluded && excluded.impressions >= MIN_BENCHMARK_URL_IMPRESSIONS ? excluded : void 0;
+	const rows = input.bucket.rows - (excluded?.rows ?? 0);
+	const impressions = input.bucket.impressions - (excluded?.impressions ?? 0);
+	const qualifiedImpressions = input.bucket.qualifiedImpressions - (excludesQualified?.impressions ?? 0);
+	const urlSamples = input.bucket.qualified.length - (excludesQualified ? 1 : 0);
+	const excludesPositive = Boolean(excludesQualified && excludesQualified.ctr > 0);
+	const positiveUrlSamples = input.bucket.positive.length - (excludesPositive ? 1 : 0);
+	const hasEnoughSiteData = qualifiedImpressions >= MIN_BENCHMARK_IMPRESSIONS && urlSamples >= MIN_BENCHMARK_URL_SAMPLES && positiveUrlSamples >= MIN_POSITIVE_URL_SAMPLES;
+	const samplePopulation = input.samplePopulation ?? "positive_url_samples";
+	const rawSiteCtr = hasEnoughSiteData ? samplePopulation === "all_qualified_url_samples" ? indexedPercentile(input.bucket.qualified, input.bucket.qualifiedIndexes, input.excludedKey, 75) : indexedPercentile(input.bucket.positive, input.bucket.positiveIndexes, input.excludedKey, 75) : void 0;
+	const fallbackCtr = expectedCtrForPosition(input.position);
+	const floorCtr = fallbackCtr * SITE_BENCHMARK_FLOOR_MULTIPLIER;
+	const capCtr = fallbackCtr * SITE_BENCHMARK_CAP_MULTIPLIER;
+	const hasSiteCtr = rawSiteCtr !== void 0;
+	const ctr = hasSiteCtr ? Math.max(floorCtr, Math.min(rawSiteCtr, capCtr)) : fallbackCtr;
+	const adjustment = hasSiteCtr && rawSiteCtr < floorCtr ? "_floored" : hasSiteCtr && rawSiteCtr > capCtr ? "_capped" : "";
+	const source = hasSiteCtr ? `site_gsc_position_bucket_robust_p75${samplePopulation === "all_qualified_url_samples" ? "_all_samples" : ""}${input.sourceSuffix}${adjustment}` : input.fallbackSource ?? "default_position_curve";
+	return {
+		ctr: Number(ctr.toFixed(4)),
+		source,
+		samplePopulation,
+		rows,
+		impressions,
+		qualifiedImpressions,
+		urlSamples,
+		positiveUrlSamples
+	};
+}
+function smoothMonotonicBenchmarks(input) {
+	const byPosition = {};
+	let previousCtr = Number.POSITIVE_INFINITY;
+	for (const position of Object.keys(CTR_BASELINE).map(Number)) {
 		const benchmark = input[position] ?? bucketBenchmark({
 			rows: [],
 			position
@@ -5759,9 +6351,25 @@ function smoothMonotonicBenchmarks(input) {
 }
 function createCtrBenchmarkContext(rows, options = {}) {
 	const buckets = /* @__PURE__ */ new Map();
+	const diagnostics = {
+		indexRowVisits: 0,
+		urlLookups: 0,
+		fallbackRowScans: 0
+	};
 	for (const row of rows) {
 		const position = roundedPosition(row.position);
-		buckets.set(position, [...buckets.get(position) ?? [], row]);
+		const bucket = buckets.get(position) ?? [];
+		bucket.push(row);
+		buckets.set(position, bucket);
+		diagnostics.indexRowVisits += 1;
+	}
+	const indexedBuckets = new Map([...buckets].map(([position, bucketRows]) => [position, indexedPositionBucket(bucketRows)]));
+	const emptyIndexedBucket = indexedPositionBucket([]);
+	const urlBenchmarkCache = /* @__PURE__ */ new Map();
+	const urlRowCounts = /* @__PURE__ */ new Map();
+	for (const row of rows) {
+		const key = normalizePageUrl(row.keys?.[1] ?? "");
+		urlRowCounts.set(key, (urlRowCounts.get(key) ?? 0) + 1);
 	}
 	const raw = {};
 	for (const position of Object.keys(CTR_BASELINE).map(Number)) raw[position] = bucketBenchmark({
@@ -5772,6 +6380,7 @@ function createCtrBenchmarkContext(rows, options = {}) {
 	});
 	const byPosition = smoothMonotonicBenchmarks(raw);
 	function benchmarkForRow(row, excludedRows, sourceSuffix) {
+		diagnostics.fallbackRowScans += rows.length;
 		const position = roundedPosition(row.position);
 		const excludedSet = new Set(excludedRows);
 		const rowRaw = {};
@@ -5788,18 +6397,49 @@ function createCtrBenchmarkContext(rows, options = {}) {
 			position
 		});
 	}
+	function benchmarkForUrl(row) {
+		diagnostics.urlLookups += 1;
+		const position = roundedPosition(row.position);
+		const excludedKey = normalizePageUrl(row.keys?.[1] ?? "");
+		const cached = excludedKey ? urlBenchmarkCache.get(excludedKey)?.get(position) : void 0;
+		if (cached) return cached;
+		const rowRaw = {};
+		for (const bucketPosition of Object.keys(CTR_BASELINE).map(Number)) rowRaw[bucketPosition] = indexedBucketBenchmark({
+			bucket: indexedBuckets.get(bucketPosition) ?? emptyIndexedBucket,
+			position: bucketPosition,
+			excludedKey,
+			sourceSuffix: "_leave_target_url_out",
+			samplePopulation: options.samplePopulation,
+			fallbackSource: options.fallbackSource
+		});
+		const benchmark = smoothMonotonicBenchmarks(rowRaw)[String(position)] ?? byPosition[String(position)] ?? bucketBenchmark({
+			rows: [],
+			position
+		});
+		let byPositionForUrl = excludedKey ? urlBenchmarkCache.get(excludedKey) : void 0;
+		if (excludedKey && !byPositionForUrl && urlBenchmarkCache.size < URL_BENCHMARK_CACHE_LIMIT) {
+			byPositionForUrl = /* @__PURE__ */ new Map();
+			urlBenchmarkCache.set(excludedKey, byPositionForUrl);
+		}
+		byPositionForUrl?.set(position, benchmark);
+		return benchmark;
+	}
 	return {
 		byPosition,
 		forRow(row) {
 			return benchmarkForRow(row, [row], "_leave_one_out");
 		},
 		forUrl(row) {
-			const url = row.keys?.[1];
-			return benchmarkForRow(row, url ? rows.filter((candidate) => samePageUrl(candidate.keys?.[1] ?? "", url)) : [row], "_leave_target_url_out");
+			return benchmarkForUrl(row);
+		},
+		urlRowCount(row) {
+			const key = normalizePageUrl(row.keys?.[1] ?? "");
+			return urlRowCounts.get(key) ?? 0;
 		},
 		forAggregate(row, excludedRows = []) {
 			return benchmarkForRow(row, excludedRows, excludedRows.length ? "_leave_group_out" : void 0);
-		}
+		},
+		diagnostics
 	};
 }
 function queryFieldCoverage(query, input) {
@@ -6604,6 +7244,210 @@ function matchesFilters(issue, filters) {
 	return true;
 }
 //#endregion
+//#region packages/core/src/analyze/crawler/agent-endpoints.ts
+const MAX_LINK_HEADER_ENTRIES = 25;
+const REGISTERED_AGENT_LINK_RELS = [
+	"api-catalog",
+	"describedby",
+	"service-desc",
+	"service-doc"
+];
+const EMERGING_AGENT_LINK_RELS = ["agent-skills", "llms-txt"];
+const AGENT_ENDPOINT_PROBES = [
+	{
+		id: "mcp-server-card",
+		path: "/.well-known/mcp/server-card.json",
+		accept: "application/json",
+		expects: "json",
+		interestFields: [
+			"serverInfo",
+			"transport",
+			"capabilities"
+		],
+		requiredFields: ["serverInfo"]
+	},
+	{
+		id: "mcp-server-cards",
+		path: "/.well-known/mcp/server-cards.json",
+		accept: "application/json",
+		expects: "json",
+		interestFields: ["serverCards", "servers"]
+	},
+	{
+		id: "a2a-agent-card",
+		path: "/.well-known/agent-card.json",
+		accept: "application/json",
+		expects: "json",
+		interestFields: [
+			"name",
+			"description",
+			"url",
+			"capabilities",
+			"skills"
+		],
+		requiredFields: ["name"]
+	},
+	{
+		id: "openid-configuration",
+		path: "/.well-known/openid-configuration",
+		accept: "application/json",
+		expects: "json",
+		interestFields: [
+			"issuer",
+			"authorization_endpoint",
+			"token_endpoint",
+			"jwks_uri",
+			"response_types_supported"
+		],
+		requiredFields: ["issuer"]
+	},
+	{
+		id: "oauth-authorization-server",
+		path: "/.well-known/oauth-authorization-server",
+		accept: "application/json",
+		expects: "json",
+		interestFields: [
+			"issuer",
+			"authorization_endpoint",
+			"token_endpoint",
+			"jwks_uri",
+			"grant_types_supported"
+		],
+		requiredFields: ["issuer"]
+	},
+	{
+		id: "oauth-protected-resource",
+		path: "/.well-known/oauth-protected-resource",
+		accept: "application/json",
+		expects: "json",
+		interestFields: [
+			"resource",
+			"authorization_servers",
+			"scopes_supported"
+		],
+		requiredFields: ["resource"]
+	},
+	{
+		id: "api-catalog",
+		path: "/.well-known/api-catalog",
+		accept: "application/linkset+json,application/json;q=0.9",
+		expects: "json",
+		interestFields: ["linkset"],
+		requiredFields: ["linkset"]
+	},
+	{
+		id: "web-bot-auth-directory",
+		path: "/.well-known/http-message-signatures-directory",
+		accept: "application/http-message-signatures-directory+json,application/json;q=0.9",
+		expects: "json",
+		interestFields: ["keys"],
+		requiredFields: ["keys"]
+	},
+	{
+		id: "auth-md",
+		path: "/auth.md",
+		accept: "text/markdown,text/plain;q=0.9",
+		expects: "text"
+	},
+	{
+		id: "llms-full-txt",
+		path: "/llms-full.txt",
+		accept: "text/plain,text/markdown;q=0.9",
+		expects: "text"
+	}
+];
+async function inspectAgentEndpoint(input) {
+	const url = new URL(input.probe.path, input.origin).toString();
+	try {
+		const { response, body } = await fetchText({
+			url,
+			timeoutMs: input.timeoutMs,
+			fetch: input.fetch,
+			signal: input.signal,
+			accept: input.probe.accept
+		});
+		const contentType = response.headers.get("content-type") ?? void 0;
+		const htmlFallback = /\btext\/html\b/iu.test(contentType ?? "");
+		const exists = response.status >= 200 && response.status < 300 && !htmlFallback;
+		const observation = {
+			id: input.probe.id,
+			url: response.url || url,
+			status: response.status,
+			exists,
+			...contentType === void 0 ? {} : { contentType }
+		};
+		if (exists && input.probe.expects === "json") {
+			let parsed;
+			try {
+				const value = JSON.parse(body);
+				observation.validJson = true;
+				if (value && typeof value === "object" && !Array.isArray(value)) parsed = value;
+			} catch {
+				observation.validJson = false;
+			}
+			if (parsed) {
+				observation.presentFields = [.../* @__PURE__ */ new Set([...input.probe.interestFields ?? [], ...input.probe.requiredFields ?? []])].sort().filter((field) => parsed[field] !== void 0);
+				observation.missingFields = (input.probe.requiredFields ?? []).filter((field) => parsed[field] === void 0).sort();
+			} else if (observation.validJson) {
+				observation.presentFields = [];
+				observation.missingFields = [...input.probe.requiredFields ?? []].sort();
+			}
+		}
+		return observation;
+	} catch (error) {
+		return {
+			id: input.probe.id,
+			url,
+			exists: false,
+			error: safeError(error)
+		};
+	}
+}
+async function inspectAgentEndpoints(input) {
+	const [linkHeader, ...endpoints] = await Promise.all([(async () => {
+		try {
+			const { response } = await fetchText({
+				url: input.startUrl,
+				timeoutMs: input.timeoutMs,
+				fetch: input.fetch,
+				signal: input.signal,
+				accept: "text/html,application/xhtml+xml;q=0.9"
+			});
+			const entries = linkEntries(response.headers.get("link") ?? void 0, input.startUrl).slice(0, MAX_LINK_HEADER_ENTRIES).map((entry) => ({
+				url: entry.url,
+				rel: entry.rel.filter(Boolean),
+				...entry.type ? { type: entry.type } : {}
+			}));
+			const observedRels = new Set(entries.flatMap((entry) => entry.rel));
+			return {
+				url: response.url || input.startUrl,
+				status: response.status,
+				entries,
+				registeredRels: REGISTERED_AGENT_LINK_RELS.filter((rel) => observedRels.has(rel)),
+				emergingRels: EMERGING_AGENT_LINK_RELS.filter((rel) => observedRels.has(rel))
+			};
+		} catch (error) {
+			return {
+				url: input.startUrl,
+				entries: [],
+				registeredRels: [],
+				emergingRels: [],
+				error: safeError(error)
+			};
+		}
+	})(), ...AGENT_ENDPOINT_PROBES.map((probe) => inspectAgentEndpoint({
+		origin: input.origin,
+		probe,
+		timeoutMs: input.timeoutMs,
+		fetch: input.fetch,
+		signal: input.signal
+	}))]);
+	return {
+		linkHeader,
+		endpoints
+	};
+}
+//#endregion
 //#region packages/core/src/analyze/crawler/agent-discovery.ts
 const MAX_BODY_BYTES = 2e6;
 const MAX_LLMS_LINKS = 100;
@@ -6797,16 +7641,15 @@ async function inspectMarkdownPage(input) {
 		repeatedHashStable: null,
 		markdownCanonicalMatchesHtml: null
 	};
-	if (advertisedUrls.length !== 1) return observation;
-	const explicitUrl = advertisedUrls[0];
+	const explicitUrl = advertisedUrls.length === 1 ? advertisedUrls[0] : void 0;
 	const [explicit, negotiated, repeated] = await Promise.all([
-		fetchRepresentation({
+		explicitUrl ? fetchRepresentation({
 			url: explicitUrl,
 			accept: "text/markdown",
 			timeoutMs: input.timeoutMs,
 			fetch: input.fetch,
 			signal: input.signal
-		}),
+		}) : Promise.resolve(void 0),
 		fetchRepresentation({
 			url: input.page.finalUrl,
 			accept: "text/markdown",
@@ -6815,20 +7658,21 @@ async function inspectMarkdownPage(input) {
 			signal: input.signal
 		}),
 		fetchRepresentation({
-			url: explicitUrl,
+			url: explicitUrl ?? input.page.finalUrl,
 			accept: "text/markdown",
 			timeoutMs: input.timeoutMs,
 			fetch: input.fetch,
 			signal: input.signal
 		})
 	]);
-	observation.explicit = explicit.observation;
+	if (explicit) observation.explicit = explicit.observation;
 	observation.negotiated = negotiated.observation;
 	observation.repeated = repeated.observation;
-	observation.explicitMatchesNegotiated = explicit.observation.sha256 && negotiated.observation.sha256 ? explicit.observation.sha256 === negotiated.observation.sha256 : null;
-	observation.repeatedHashStable = explicit.observation.sha256 && repeated.observation.sha256 ? explicit.observation.sha256 === repeated.observation.sha256 : null;
-	observation.markdownCanonicalMatchesHtml = sameDocument(explicit.observation.canonicalUrl, input.page.finalUrl);
-	if (explicit.body) observation.quality = markdownQuality(explicit.body, input.page);
+	observation.explicitMatchesNegotiated = explicit?.observation.sha256 && negotiated.observation.sha256 ? explicit.observation.sha256 === negotiated.observation.sha256 : null;
+	const primary = explicit ?? negotiated;
+	observation.repeatedHashStable = primary.observation.sha256 && repeated.observation.sha256 ? primary.observation.sha256 === repeated.observation.sha256 : null;
+	observation.markdownCanonicalMatchesHtml = sameDocument(primary.observation.canonicalUrl, input.page.finalUrl);
+	if (primary.body) observation.quality = markdownQuality(primary.body, input.page);
 	return observation;
 }
 async function fetchText(input) {
@@ -7124,7 +7968,7 @@ async function collectAgentDiscovery(input) {
 		fetch: input.fetch,
 		signal: input.signal
 	}))));
-	const [agentSkills, llmsTxt, routeManifest] = await Promise.all([
+	const [agentSkills, llmsTxt, routeManifest, endpointDiscovery] = await Promise.all([
 		inspectAgentSkills({
 			...input,
 			origin
@@ -7139,6 +7983,10 @@ async function collectAgentDiscovery(input) {
 			origin,
 			pages,
 			observations
+		}),
+		inspectAgentEndpoints({
+			...input,
+			origin
 		})
 	]);
 	const qZero = await fetchRepresentation({
@@ -7164,10 +8012,13 @@ async function collectAgentDiscovery(input) {
 	const wwwLocation = www.location ? new URL(www.location, wwwUrl).toString() : void 0;
 	const qZeroContentType = qZero.observation.contentType;
 	const advertisedPages = observations.filter((observation) => observation.htmlAlternateUnique).length;
-	const evaluatedPages = observations.filter((observation) => observation.explicit?.status !== void 0).length;
+	const evaluatedPages = observations.filter((observation) => {
+		const representation = observation.explicit ?? observation.negotiated;
+		return representation?.status !== void 0 && representation.status >= 200 && representation.status < 300 && /^\s*text\/markdown\b/iu.test(representation.contentType ?? "");
+	}).length;
 	const warnings = [...pages.length !== input.pages.length ? [`Agent representations were evaluated for ${pages.length} successful HTML pages from ${input.pages.length} retained documents.`] : [], ...routeManifest.valid ? [] : ["No valid public agent route manifest was available for orphan checks."]].sort();
 	const htmlContentSignals = pages.map((page) => headerValue$1(page.responseHeaders, "content-signal")).filter((value) => Boolean(value));
-	const markdownContentSignals = observations.map((observation) => observation.explicit?.contentSignal).filter((value) => Boolean(value));
+	const markdownContentSignals = observations.map((observation) => observation.explicit?.contentSignal ?? observation.negotiated?.contentSignal).filter((value) => Boolean(value));
 	const contentSignalValues = [.../* @__PURE__ */ new Set([...htmlContentSignals, ...markdownContentSignals])].sort();
 	return {
 		profile: "content",
@@ -7207,6 +8058,7 @@ async function collectAgentDiscovery(input) {
 		routeManifest,
 		agentSkills,
 		llmsTxt,
+		endpointDiscovery,
 		contentSignals: {
 			htmlValues: [...new Set(htmlContentSignals)].sort(),
 			markdownValues: [...new Set(markdownContentSignals)].sort(),
@@ -7250,7 +8102,11 @@ function section$1(id, title, checks) {
 	};
 }
 function failedMarkdownUrls(discovery) {
-	return discovery.markdownAlternates.pages.filter((page) => !page.htmlAlternateUnique || page.httpAlternateUrls.length !== 1 || page.explicit?.status !== 200 || !/^\s*text\/markdown\b/iu.test(page.explicit.contentType ?? "") || page.explicitMatchesNegotiated !== true || page.repeatedHashStable !== true || page.markdownCanonicalMatchesHtml !== true || page.explicit?.varyAccept !== true || page.negotiated?.varyAccept !== true).map((page) => page.htmlUrl);
+	return discovery.markdownAlternates.pages.filter((page) => {
+		const negotiatedOk = page.negotiated?.status === 200 && /^\s*text\/markdown\b/iu.test(page.negotiated.contentType ?? "") && page.negotiated.varyAccept;
+		const explicitOk = !page.htmlAlternateUnique ? true : page.httpAlternateUrls.length === 1 && page.explicit?.status === 200 && /^\s*text\/markdown\b/iu.test(page.explicit.contentType ?? "") && page.explicitMatchesNegotiated === true && page.markdownCanonicalMatchesHtml === true;
+		return !negotiatedOk || !explicitOk || page.repeatedHashStable !== true;
+	}).map((page) => page.htmlUrl);
 }
 function qualityFailureUrls(discovery) {
 	return discovery.markdownAlternates.pages.filter((page) => {
@@ -7292,20 +8148,25 @@ function markdownChecks(discovery) {
 	const markdown = discovery.markdownAlternates;
 	const failedUrls = failedMarkdownUrls(discovery);
 	const qualityUrls = qualityFailureUrls(discovery);
-	const coverageComplete = markdown.eligibleHtmlPages > 0 && markdown.advertisedPages === markdown.eligibleHtmlPages && markdown.evaluatedPages === markdown.eligibleHtmlPages;
-	const bytesMatch = markdown.eligibleHtmlPages > 0 && markdown.exactByteMatches === markdown.eligibleHtmlPages;
+	const negotiatedPages = markdown.pages.filter((page) => page.negotiated?.status === 200 && /^\s*text\/markdown\b/iu.test(page.negotiated.contentType ?? "")).length;
+	const explicitPages = markdown.pages.filter((page) => page.htmlAlternateUnique).length;
+	const coverageComplete = markdown.eligibleHtmlPages > 0 && markdown.evaluatedPages === markdown.eligibleHtmlPages;
+	const negotiationComplete = markdown.eligibleHtmlPages > 0 && negotiatedPages === markdown.eligibleHtmlPages && markdown.pages.every((page) => page.negotiated?.varyAccept) && markdown.pages.every((page) => !page.htmlAlternateUnique || page.explicitMatchesNegotiated === true);
 	const stable = markdown.eligibleHtmlPages > 0 && markdown.stableResponses === markdown.eligibleHtmlPages;
-	const tokenHeaders = markdown.pages.filter((page) => page.explicit?.bytes !== void 0 && page.explicit?.markdownTokens !== void 0 && page.explicit.markdownTokens === Math.ceil(page.explicit.bytes / 4) && page.explicit.markdownTokens === page.negotiated?.markdownTokens).length;
+	const tokenHeaders = markdown.pages.filter((page) => {
+		return (page.explicit ?? page.negotiated)?.markdownTokens !== void 0 && (!page.explicit || !page.negotiated || page.explicit.markdownTokens === page.negotiated.markdownTokens);
+	}).length;
 	return [
 		check$1("representations", {
 			id: "markdown-coverage",
 			status: markdown.eligibleHtmlPages === 0 ? "unknown" : coverageComplete ? "pass" : "fail",
-			title: coverageComplete ? "Every successful HTML page advertises one Markdown alternative" : "Markdown alternative coverage is incomplete",
-			plainEnglish: `${markdown.advertisedPages} of ${markdown.eligibleHtmlPages} successful HTML pages advertised one Markdown alternative, and ${markdown.evaluatedPages} were fetched.`,
-			action: coverageComplete ? "Keep route generation tied to the same deterministic page manifest." : "Add one text/markdown alternate for each public HTML page and make every advertised URL return successfully.",
+			title: coverageComplete ? "Every successful HTML page has a Markdown representation" : "Markdown representation coverage is incomplete",
+			plainEnglish: `${markdown.evaluatedPages} of ${markdown.eligibleHtmlPages} successful HTML pages returned Markdown. ${explicitPages} advertised an explicit alternative and ${negotiatedPages} supported content negotiation.`,
+			action: coverageComplete ? "Keep the negotiated or explicit Markdown response stable as routes change." : "Return text/markdown through content negotiation or advertise one working Markdown alternative for each public HTML page.",
 			evidence: {
 				eligibleHtmlPages: markdown.eligibleHtmlPages,
 				advertisedPages: markdown.advertisedPages,
+				negotiatedPages,
 				evaluatedPages: markdown.evaluatedPages
 			},
 			urls: failedUrls.slice(0, 25)
@@ -7313,18 +8174,22 @@ function markdownChecks(discovery) {
 		check$1("representations", {
 			id: "markdown-token-estimates",
 			status: markdown.evaluatedPages === 0 ? "unknown" : tokenHeaders === markdown.evaluatedPages ? "pass" : "warning",
-			title: tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes the same token estimate" : "Some Markdown responses are missing a stable token estimate",
-			plainEnglish: `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned matching X-Markdown-Tokens values on the explicit and negotiated responses. The value is a quick size estimate, not an exact model-specific token count.`,
-			action: "Generate the estimate from the stored Markdown bytes and return the same value for explicit and negotiated responses.",
-			urls: markdown.pages.filter((page) => page.explicit?.markdownTokens === void 0 || page.explicit.markdownTokens !== page.negotiated?.markdownTokens).map((page) => page.htmlUrl).slice(0, 25)
+			title: tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes a token estimate" : "Some Markdown responses are missing a stable token estimate",
+			plainEnglish: `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned X-Markdown-Tokens. Paired explicit and negotiated responses agreed where both existed. The value is an estimate, not an exact model-specific token count.`,
+			action: "Return a stable token estimate and keep it consistent when both explicit and negotiated responses exist.",
+			urls: markdown.pages.filter((page) => {
+				return (page.explicit ?? page.negotiated)?.markdownTokens === void 0 || page.explicit !== void 0 && page.negotiated !== void 0 && page.explicit.markdownTokens !== page.negotiated.markdownTokens;
+			}).map((page) => page.htmlUrl).slice(0, 25)
 		}),
 		check$1("representations", {
 			id: "markdown-negotiation",
-			status: markdown.eligibleHtmlPages === 0 ? "unknown" : bytesMatch && discovery.contentNegotiation.qZeroHonoured ? "pass" : "fail",
-			title: bytesMatch && discovery.contentNegotiation.qZeroHonoured ? "Explicit and negotiated Markdown use the same bytes" : "Content negotiation does not match the explicit Markdown contract",
-			plainEnglish: `${markdown.exactByteMatches} of ${markdown.eligibleHtmlPages} pages returned identical explicit and negotiated Markdown. A request that refuses Markdown ${discovery.contentNegotiation.qZeroHonoured ? "received HTML" : "did not produce confirmed HTML evidence"}.`,
-			action: "Use one stored Markdown artifact for both routes, honour Accept q-values, and send Vary: Accept on negotiated responses.",
+			status: markdown.eligibleHtmlPages === 0 ? "unknown" : negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "pass" : "fail",
+			title: negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "Markdown content negotiation works across the site" : "Markdown content negotiation needs attention",
+			plainEnglish: `${negotiatedPages} of ${markdown.eligibleHtmlPages} pages returned negotiated Markdown. ${markdown.exactByteMatches} of ${explicitPages} paired explicit responses matched byte for byte. A request that refuses Markdown ${discovery.contentNegotiation.qZeroHonoured ? "received HTML" : "did not produce confirmed HTML evidence"}.`,
+			action: "Honour Accept q-values, send Vary: Accept, and keep paired explicit and negotiated responses identical when both exist.",
 			evidence: {
+				negotiatedPages,
+				explicitPages,
 				exactByteMatches: markdown.exactByteMatches,
 				qZero: discovery.contentNegotiation
 			},
@@ -7334,8 +8199,8 @@ function markdownChecks(discovery) {
 			id: "markdown-determinism",
 			status: markdown.eligibleHtmlPages === 0 ? "unknown" : stable ? "pass" : "fail",
 			title: stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
-			plainEnglish: `${markdown.stableResponses} of ${markdown.eligibleHtmlPages} repeated explicit responses kept the same SHA-256 digest.`,
-			action: "Generate Markdown during the build and remove runtime rewriting, timestamps, or random output.",
+			plainEnglish: `${markdown.stableResponses} of ${markdown.eligibleHtmlPages} repeated Markdown responses kept the same SHA-256 digest.`,
+			action: "Remove timestamps, random output, or other runtime rewriting that changes the same Markdown response between requests.",
 			evidence: { stableResponses: markdown.stableResponses },
 			urls: failedUrls.slice(0, 25)
 		}),
@@ -7349,7 +8214,59 @@ function markdownChecks(discovery) {
 		})
 	];
 }
-function discoveryChecks(discovery) {
+function contentSignalsCheck(discovery, robotsContentSignals) {
+	const signals = discovery.contentSignals;
+	const robotsDeclared = robotsContentSignals ?? [];
+	const robotsCollected = robotsContentSignals !== void 0;
+	const headersObserved = signals.htmlValues.length > 0 || signals.markdownValues.length > 0;
+	const headersConsistent = signals.consistent === true;
+	const evidence = {
+		robotsDirectives: robotsDeclared,
+		robotsDirectivesCollected: robotsCollected,
+		...signals
+	};
+	if (headersConsistent) return check$1("discovery", {
+		id: "content-signals",
+		status: "pass",
+		title: "Every response publishes one explicit Content Signals policy",
+		plainEnglish: `The same ${signals.htmlValues[0]} policy was observed on every evaluated HTML and Markdown response${robotsDeclared.length ? ", and robots.txt declares a Content-Signal directive" : ""}.`,
+		action: "Keep the same Content-Signal value on robots.txt, HTML, and Markdown responses when the policy changes.",
+		evidence
+	});
+	if (robotsDeclared.length > 0 && !headersObserved) return check$1("discovery", {
+		id: "content-signals",
+		status: "pass",
+		title: "robots.txt declares a Content Signals policy",
+		plainEnglish: `robots.txt declares ${robotsDeclared.length === 1 ? "one Content-Signal directive" : `${robotsDeclared.length} Content-Signal directives`}. No response-header variant was observed, which is fine because robots.txt is the primary published location.`,
+		action: "Keep the robots.txt directive as the single policy, or mirror the same value in response headers if you add them.",
+		evidence
+	});
+	if (robotsDeclared.length > 0 || headersObserved) return check$1("discovery", {
+		id: "content-signals",
+		status: "warning",
+		title: "Content Signals are published inconsistently",
+		plainEnglish: `${signals.missingHtmlPages} HTML responses and ${signals.missingMarkdownPages} Markdown responses were missing the header, the observed values differed, or the header policy does not line up with robots.txt.`,
+		action: "Choose the site policy deliberately and publish the same Content-Signal value everywhere it appears.",
+		evidence
+	});
+	if (signals.consistent === null && !robotsCollected) return check$1("discovery", {
+		id: "content-signals",
+		status: "unknown",
+		title: "Content Signals evidence was not collected",
+		plainEnglish: "No pages were evaluated for the header and robots.txt directive evidence is unavailable in this crawl.",
+		action: "Re-run the report with a URL so robots.txt and response headers can be checked.",
+		evidence
+	});
+	return check$1("discovery", {
+		id: "content-signals",
+		status: "info",
+		title: "No Content Signals policy is published",
+		plainEnglish: "Neither robots.txt nor the evaluated responses declare a Content-Signal policy. Publishing one is optional.",
+		action: "Add a Content-Signal directive to robots.txt only when you want to state an explicit search, ai-input, or ai-train preference.",
+		evidence
+	});
+}
+function discoveryChecks(discovery, robotsContentSignals) {
 	const skills = discovery.agentSkills;
 	const skillsValid = skills.validIndex && skills.skills.length > 0 && skills.skills.every((skill) => skill.sameOrigin && skill.status === 200 && /^\s*text\/markdown\b/iu.test(skill.contentType ?? "") && Boolean(skill.cors) && skill.digestMatches === true && skill.frontmatterValid === true) && Boolean(skills.cors);
 	const llms = discovery.llmsTxt;
@@ -7393,14 +8310,7 @@ function discoveryChecks(discovery) {
 				...llms.missingCrawlRoutes
 			].slice(0, 25)
 		}),
-		check$1("discovery", {
-			id: "content-signals",
-			status: discovery.contentSignals.consistent === null ? "unknown" : discovery.contentSignals.consistent ? "pass" : "warning",
-			title: discovery.contentSignals.consistent ? "HTML and Markdown publish one explicit Content Signals policy" : "Content Signals are missing or inconsistent across representations",
-			plainEnglish: discovery.contentSignals.consistent ? `The same ${discovery.contentSignals.htmlValues[0]} policy was observed on every evaluated HTML and Markdown response.` : `${discovery.contentSignals.missingHtmlPages} HTML responses and ${discovery.contentSignals.missingMarkdownPages} Markdown responses were missing the header, or the observed values differed.`,
-			action: "Choose the site policy deliberately and return the same Content-Signal value on HTML and Markdown responses.",
-			evidence: discovery.contentSignals
-		}),
+		contentSignalsCheck(discovery, robotsContentSignals),
 		check$1("discovery", {
 			id: "route-manifest",
 			status: !discovery.routeManifest.valid ? "info" : discovery.routeManifest.missingHtmlRoutes.length === 0 && discovery.routeManifest.missingMarkdownRoutes.length === 0 && discovery.routeManifest.orphanMarkdownRoutes.length === 0 ? "pass" : "warning",
@@ -7411,6 +8321,126 @@ function discoveryChecks(discovery) {
 		})
 	];
 }
+function optionalEndpointCheck(endpoints, copy) {
+	const found = endpoints.filter((endpoint) => endpoint.exists);
+	const evidence = { endpoints };
+	if (found.length === 0) return check$1("endpoints", {
+		id: copy.id,
+		status: "info",
+		title: `${copy.subject} is not published`,
+		plainEnglish: copy.absent,
+		action: copy.absentAction,
+		evidence
+	});
+	const valid = found.filter((endpoint) => endpoint.validJson !== false && (endpoint.missingFields ?? []).length === 0);
+	if (valid.length > 0) return check$1("endpoints", {
+		id: copy.id,
+		status: "pass",
+		title: `${copy.subject} is published and parses`,
+		plainEnglish: `${valid[0]?.url} returned ${valid[0]?.contentType ?? "a response"} with the expected structure. ${copy.requiredNote}`,
+		action: copy.presentAction,
+		evidence,
+		urls: valid.map((endpoint) => endpoint.url)
+	});
+	const broken = found[0];
+	return check$1("endpoints", {
+		id: copy.id,
+		status: "warning",
+		title: `${copy.subject} is published but needs review`,
+		plainEnglish: `${broken?.url} responded, but ${broken?.validJson === false ? "the body is not valid JSON" : `required fields are missing: ${(broken?.missingFields ?? []).join(", ")}`}. ${copy.requiredNote}`,
+		action: copy.presentAction,
+		evidence,
+		urls: found.map((endpoint) => endpoint.url)
+	});
+}
+function endpointChecks(discovery) {
+	const endpointDiscovery = discovery.endpointDiscovery;
+	if (!endpointDiscovery) return [check$1("endpoints", {
+		id: "agent-endpoints-evidence",
+		status: "unknown",
+		title: "Agent endpoint evidence was not collected",
+		plainEnglish: "This saved crawl predates the Link header and well-known endpoint probes, so that evidence is unavailable.",
+		action: "Run the agent-readiness report with a URL to collect endpoint evidence."
+	})];
+	const byId = new Map(endpointDiscovery.endpoints.map((endpoint) => [endpoint.id, endpoint]));
+	const pick = (...ids) => ids.map((id) => byId.get(id)).filter((endpoint) => Boolean(endpoint));
+	const linkHeader = endpointDiscovery.linkHeader;
+	const advertisedRels = [...linkHeader.registeredRels, ...linkHeader.emergingRels];
+	return [
+		check$1("endpoints", {
+			id: "link-headers",
+			status: linkHeader.error ? "unknown" : advertisedRels.length > 0 ? "pass" : "info",
+			title: linkHeader.error ? "Link header evidence was not collected" : advertisedRels.length > 0 ? "The start page Link header advertises agent resources" : linkHeader.entries.length > 0 ? "Link headers exist but advertise no agent resources" : "No Link header advertises agent resources",
+			plainEnglish: linkHeader.error ? `The start page request failed: ${linkHeader.error}` : advertisedRels.length > 0 ? `${linkHeader.entries.length} Link header ${linkHeader.entries.length === 1 ? "entry was" : "entries were"} observed. Registered relation types: ${linkHeader.registeredRels.join(", ") || "none"}. Emerging relation types: ${linkHeader.emergingRels.join(", ") || "none"}.` : linkHeader.entries.length > 0 ? `${linkHeader.entries.length} Link header ${linkHeader.entries.length === 1 ? "entry was" : "entries were"} observed, but none used a recognised agent relation type such as api-catalog, service-desc, service-doc, describedby, llms-txt, or agent-skills.` : "The start page response includes no Link header. Advertising machine-readable resources this way is optional.",
+			action: "Advertise real machine-readable resources with Link response headers. Use registered relation types such as api-catalog and service-desc, and emerging types such as llms-txt only when the target resolves.",
+			evidence: linkHeader
+		}),
+		optionalEndpointCheck(pick("mcp-server-card", "mcp-server-cards"), {
+			id: "mcp-server-card",
+			subject: "An MCP server card",
+			absent: "Neither /.well-known/mcp/server-card.json nor server-cards.json returned a machine-readable document. The server card format is a draft MCP proposal, and a content site does not need one.",
+			absentAction: "Publish a server card only when the site exposes a real public MCP endpoint.",
+			presentAction: "Keep serverInfo, the transport endpoint, and capabilities accurate for the deployed MCP server.",
+			requiredNote: "The server card format is a draft MCP proposal, so this is a structural observation rather than a compliance verdict."
+		}),
+		optionalEndpointCheck(pick("a2a-agent-card"), {
+			id: "a2a-agent-card",
+			subject: "An A2A agent card",
+			absent: "/.well-known/agent-card.json returned no machine-readable document. Agent cards matter only for sites that expose an agent-to-agent endpoint.",
+			absentAction: "Publish an agent card only when the site hosts a real A2A endpoint.",
+			presentAction: "Keep the agent card name, url, and capabilities aligned with the deployed endpoint.",
+			requiredNote: "A2A is an emerging protocol, so this is a structural observation rather than a compliance verdict."
+		}),
+		optionalEndpointCheck(pick("openid-configuration", "oauth-authorization-server"), {
+			id: "oauth-discovery",
+			subject: "OAuth or OpenID Connect discovery metadata",
+			absent: "No authorization-server metadata was found at /.well-known/openid-configuration or /.well-known/oauth-authorization-server. That only matters when the site exposes protected APIs that agents authenticate with.",
+			absentAction: "Publish RFC 8414 or OpenID Connect discovery metadata only when a real authorization server issues tokens for this origin.",
+			presentAction: "Keep the issuer and endpoint URLs accurate so clients can discover how to authenticate.",
+			requiredNote: "Structural presence of issuer metadata does not prove the authorization flow works."
+		}),
+		optionalEndpointCheck(pick("oauth-protected-resource"), {
+			id: "oauth-protected-resource",
+			subject: "OAuth protected resource metadata",
+			absent: "/.well-known/oauth-protected-resource returned no metadata. RFC 9728 metadata matters only for origins that serve OAuth-protected APIs.",
+			absentAction: "Publish protected resource metadata only when this origin actually serves OAuth-protected APIs.",
+			presentAction: "Keep the resource identifier and authorization_servers list accurate.",
+			requiredNote: "Structural presence does not prove tokens are issued or accepted."
+		}),
+		optionalEndpointCheck(pick("api-catalog"), {
+			id: "api-catalog",
+			subject: "An API catalog",
+			absent: "/.well-known/api-catalog returned no linkset document. An RFC 9727 catalog matters only for sites that publish APIs.",
+			absentAction: "Publish an API catalog only when the site has real APIs.",
+			presentAction: "Keep the linkset entries pointing at live service-desc and service-doc resources.",
+			requiredNote: "The catalog was checked for structure only, not for whether each linked API works."
+		}),
+		optionalEndpointCheck(pick("web-bot-auth-directory"), {
+			id: "web-bot-auth",
+			subject: "A Web Bot Auth key directory",
+			absent: "/.well-known/http-message-signatures-directory returned no key set. Publishing one matters only for operators that sign their own outbound bot requests.",
+			absentAction: "Publish a signature directory only when this origin operates bots that sign requests with Web Bot Auth.",
+			presentAction: "Keep the published keys current and rotate them safely.",
+			requiredNote: "Web Bot Auth is an emerging draft, so this is a structural observation rather than a compliance verdict."
+		}),
+		optionalEndpointCheck(pick("auth-md"), {
+			id: "auth-md",
+			subject: "An auth.md registration guide",
+			absent: "/auth.md returned no document. auth.md is an emerging proposal for telling agents how to register and authenticate, and most content sites do not need one.",
+			absentAction: "Add auth.md only when agents can genuinely register for and authenticate with this site.",
+			presentAction: "Keep the registration steps accurate and pair the file with real OAuth discovery metadata.",
+			requiredNote: "auth.md is an emerging proposal, so this is a presence observation only."
+		}),
+		optionalEndpointCheck(pick("llms-full-txt"), {
+			id: "llms-full-txt",
+			subject: "llms-full.txt",
+			absent: "/llms-full.txt returned no document. The expanded companion to llms.txt is optional, and its absence is not a search ranking problem.",
+			absentAction: "Add llms-full.txt only when an intended consumer wants the full expanded content in one file.",
+			presentAction: "Keep the file deterministic and consistent with the curated llms.txt entry points.",
+			requiredNote: "Only presence and content type were checked, not the quality of the expanded content."
+		})
+	];
+}
 function accessChecks(report, discovery) {
 	const bots = report.ai?.robotsTxt?.botAccess ?? [];
 	const blocked = bots.filter((bot) => bot.allowed === false);
@@ -7449,10 +8479,10 @@ function identityChecks(report) {
 	const officialProfiles = [...new Set(report.pages.flatMap((page) => [...page.schemaSameAs ?? [], ...page.socialProfileLinks ?? []]))];
 	return [check$1("identity", {
 		id: "identity-graph",
-		status: hasSoftware && hasWebsite && hasCreator && pageTypes.length > 0 ? "pass" : "warning",
-		title: hasSoftware && hasWebsite && hasCreator && pageTypes.length > 0 ? "Structured identity covers the software, website, creator, and pages" : "The structured identity graph is missing a useful entity layer",
-		plainEnglish: `Software identity: ${hasSoftware ? "present" : "missing"}. Website identity: ${hasWebsite ? "present" : "missing"}. Creator or publisher identity: ${hasCreator ? "present" : "missing"}. Page-level types: ${pageTypes.length ? pageTypes.join(", ") : "missing"}.`,
-		action: "Connect truthful SoftwareApplication, WebSite, creator or publisher, and page nodes. Use sameAs only for official profiles that identify the same entity.",
+		status: hasWebsite && hasCreator && pageTypes.length > 0 ? "pass" : "warning",
+		title: hasWebsite && hasCreator && pageTypes.length > 0 ? "Structured identity covers the website, creator, and pages" : "The structured identity graph is missing a useful entity layer",
+		plainEnglish: `Website identity: ${hasWebsite ? "present" : "missing"}. Creator or publisher identity: ${hasCreator ? "present" : "missing"}. Page-level types: ${pageTypes.length ? pageTypes.join(", ") : "missing"}. Software identity: ${hasSoftware ? "present" : "not observed and not required for a content site"}.`,
+		action: "Connect truthful WebSite, creator or publisher, and page nodes. Add SoftwareApplication only when the site actually represents software. Use sameAs only for official profiles that identify the same entity.",
 		evidence: {
 			schemaTypes: [...types].sort(),
 			officialProfiles: officialProfiles.sort()
@@ -7512,7 +8542,8 @@ function agentReadiness(report) {
 		section$1("profile", "Profile scope", profileChecks(discovery)),
 		section$1("access", "Crawler and protocol access", accessChecks(report, discovery)),
 		section$1("representations", "HTML and Markdown representations", markdownChecks(discovery)),
-		section$1("discovery", "Agent discovery files", discoveryChecks(discovery)),
+		section$1("discovery", "Agent discovery files", discoveryChecks(discovery, report.ai?.robotsTxt?.contentSignals)),
+		section$1("endpoints", "Agent endpoints and auth discovery", endpointChecks(discovery)),
 		section$1("identity", "Identity evidence", identityChecks(report))
 	];
 	const checks = sections.flatMap((item) => item.checks);
@@ -7549,6 +8580,7 @@ function agentReadiness(report) {
 			...discovery.warnings,
 			"A clean content profile does not prove indexing, rankings, AI mentions, citations, or selection.",
 			"A crawler token being allowed does not prove that a service fetched or used the page.",
+			"MCP server cards, A2A agent cards, Web Bot Auth directories, and auth.md are emerging conventions. Their absence is an observation, not a defect.",
 			"API, application, and commerce checks were not applicable to this content-site run."
 		]
 	};
@@ -7940,7 +8972,7 @@ function aiReadiness(report) {
 				score: 0,
 				evaluated: Boolean(agentDiscovery),
 				title: agentDiscovery ? "Markdown alternatives were checked" : "Markdown alternatives were not evaluated",
-				plainEnglish: agentDiscovery ? `${agentDiscovery.markdownAlternates.advertisedPages} of ${agentDiscovery.markdownAlternates.eligibleHtmlPages} successful HTML pages advertised one Markdown alternative. ${agentDiscovery.markdownAlternates.exactByteMatches} explicit responses matched content negotiation byte for byte.` : "This crawl does not include the focused representation checks, so it cannot say whether HTML pages publish stable Markdown alternatives.",
+				plainEnglish: agentDiscovery ? `${agentDiscovery.markdownAlternates.evaluatedPages} of ${agentDiscovery.markdownAlternates.eligibleHtmlPages} successful HTML pages returned Markdown through an explicit alternative or content negotiation. ${agentDiscovery.markdownAlternates.exactByteMatches} paired explicit responses matched content negotiation byte for byte.` : "This crawl does not include the focused representation checks, so it cannot say whether HTML pages publish stable Markdown alternatives.",
 				action: "Use the focused agent-readiness report when you need route coverage, content negotiation, byte stability, or extraction-quality evidence.",
 				evidence: agentDiscovery ? {
 					eligibleHtmlPages: agentDiscovery.markdownAlternates.eligibleHtmlPages,
@@ -8153,6 +9185,15 @@ function originHost(url) {
 function cleanTitle(page) {
 	return page.title ?? page.h1 ?? (new URL(page.finalUrl).pathname || page.finalUrl);
 }
+function truncateAtWord(value, maxLength) {
+	const normalized = value.replace(/\s+/g, " ").trim();
+	if (normalized.length <= maxLength) return normalized;
+	const room = Math.max(1, maxLength - 3);
+	const prefix = normalized.slice(0, room);
+	const boundary = prefix.lastIndexOf(" ");
+	if (boundary < 0) return "...";
+	return `${prefix.slice(0, boundary).trimEnd()}...`;
+}
 function sectionForPage(page) {
 	const path = new URL(page.finalUrl).pathname.toLowerCase();
 	if (/\/(docs|documentation|guide|guides|help|kb|learn)\b/.test(path)) return "Docs and guides";
@@ -8285,7 +9326,7 @@ function generateLlmsTxt(report, options = {}) {
 		let sectionCount = 0;
 		for (const page of sectionPages) {
 			const title = cleanTitle(page).replace(/\]/g, "\\]");
-			const note = page.metaDescription ? ` - ${page.metaDescription.replace(/\s+/g, " ").slice(0, 160)}` : "";
+			const note = page.metaDescription ? ` - ${truncateAtWord(page.metaDescription, 160)}` : "";
 			const line = `- [${title}](${page.finalUrl})${note}`;
 			if (estimatedTokens([
 				...lines,
@@ -8303,14 +9344,29 @@ function generateLlmsTxt(report, options = {}) {
 		counts[sectionName] = sectionCount;
 		if (sectionCount < sectionPages.length) break;
 	}
-	lines.push("## Notes", "", "- Optional metadata for agents and services that explicitly support llms.txt.", "- [Google Search guidance](https://developers.google.com/search/updates#clarifying-guidance-on-llms-txt-files) says llms.txt does not affect visibility or rankings.", `- Generated from crawl report ${report.id}.`, `- Crawl generated at ${report.generatedAt}.`);
+	lines.push("## Notes", "", "- Optional metadata for agents and services that explicitly support llms.txt.", "- [Google Search guidance](https://developers.google.com/search/updates#clarifying-guidance-on-llms-txt-files) says llms.txt does not affect visibility or rankings.", `- Generated from crawl report ${report.id}.`, `- Crawl generated at ${report.generatedAt}.`, `- Crawl coverage: ${report.status}; retained ${report.summary.crawledUrls} of ${report.summary.discoveredUrls} discovered URLs.`);
+	if (report.sitemapDiscovery) lines.push(`- Sitemap discovery: ${report.sitemapDiscovery.dataStatus}; returned ${report.sitemapDiscovery.urlsReturned} URLs.`);
 	if (report.caveats.length) lines.push(...report.caveats.map((caveat) => `- Caveat: ${caveat}`));
+	if (report.warnings.length) {
+		lines.push(...report.warnings.slice(0, 10).map((warning) => `- Source warning: ${warning}`));
+		if (report.warnings.length > 10) lines.push(`- Source warnings: ${report.warnings.length - 10} more warnings remain in crawl report ${report.id}.`);
+	}
 	const content = `${lines.join("\n")}\n`;
 	return {
 		content,
 		includedUrls,
 		estimatedTokens: estimatedTokens(content),
-		sections: counts
+		sections: counts,
+		source: {
+			reportId: report.id,
+			status: report.status,
+			crawledUrls: report.summary.crawledUrls,
+			discoveredUrls: report.summary.discoveredUrls,
+			sitemapUrlsReturned: report.sitemapDiscovery?.urlsReturned,
+			warnings: report.warnings.slice(0, 10),
+			warningCount: report.warnings.length,
+			warningsTruncated: report.warnings.length > 10
+		}
 	};
 }
 //#endregion
@@ -8923,6 +9979,24 @@ const RULE_DEFINITIONS = [
 		impactIfIgnored: "The page cannot be crawled, ranked, or cited while the connection failure remains.",
 		howToVerify: "Re-run the crawl and confirm the URL returns a normal HTTP status instead of status 0."
 	},
+	{
+		id: "crawler_access_blocked",
+		title: "Crawler access blocked",
+		category: "response",
+		defaultSeverity: "medium",
+		whyItMatters: "The audit could not observe the intended page because an authentication layer, rate limit, firewall, or challenge intercepted this crawler. This limits the audit evidence but does not prove that search engines or users are blocked.",
+		howToFix: "First confirm the block is unintended. If the audit should have access, use the provider evidence in the report to grant temporary, narrow access for the audit machine source IP, required hostname or paths, and the exact crawler User-Agent. Do not trust a User-Agent by itself.",
+		impactIfIgnored: "The affected URL remains unverified in this crawl, so content and indexability checks cannot support a pass or an all-clear.",
+		howToVerify: "Re-run the same health check and confirm access.blockedRequests is zero and the URL returns its intended status without challenge evidence.",
+		agentHints: {
+			evidenceFields: [
+				"request.accessBlock",
+				"report.access",
+				"issue.evidence.accessBlock"
+			],
+			suggestedCommands: ["seo crawl --sitemap-url <url> --health --json"]
+		}
+	},
 	{
 		id: "client_error",
 		title: "Client error",
@@ -9411,6 +10485,7 @@ const RULE_RECOMMENDATIONS = {
 	nofollow: "review",
 	x_robots_noindex: "review",
 	robots_blocked: "review",
+	crawler_access_blocked: "review",
 	orphan_page: "review",
 	redirected_url: "review",
 	slow_response: "review",
@@ -9632,6 +10707,14 @@ function auditCrawlRequests(requests) {
 			continue;
 		}
 		if (request.outcome === "skipped") continue;
+		if (request.accessBlock) {
+			issues.push(requestIssue("crawler_access_blocked", request, request.accessBlock.guidance.summary, {
+				requestedUrl: request.requestedUrl,
+				finalUrl: request.finalUrl,
+				accessBlock: request.accessBlock
+			}));
+			continue;
+		}
 		const redirectChain = request.redirectChain ?? [];
 		if (redirectChain.length > 0 || !sameUrl(request.requestedUrl, request.finalUrl)) issues.push(requestIssue("redirected_url", request, request.finalUrl ? `Final URL: ${request.finalUrl}` : void 0, {
 			requestedUrl: request.requestedUrl,
@@ -9667,6 +10750,7 @@ function auditCrawlPages(pages, opts = {}) {
 			}));
 			continue;
 		}
+		if (page.accessBlock) continue;
 		if (page.status >= 500) {
 			issues.push(issue("server_error", page, String(page.status), { status: page.status }));
 			if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue("broken_internal_link", page, String(page.status), {
@@ -9707,6 +10791,7 @@ function auditCrawlPages(pages, opts = {}) {
 			responseTimeMs: page.responseTimeMs,
 			thresholdMs: SLOW_RESPONSE_MS
 		}));
+		if (page.auditScope === "status") continue;
 		if (redirected) continue;
 		if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(issue("large_html", page, `${Math.round((page.sizeBytes ?? 0) / 1024)} KB`, {
 			sizeBytes: page.sizeBytes,
@@ -9942,6 +11027,10 @@ function crawlSkipReasonCountsFromRecord(input = {}) {
 }
 //#endregion
 //#region packages/core/src/analyze/crawler/report.ts
+const MAX_CRAWL_PAGES = 1e4;
+const MAX_CRAWL_DEPTH = 64;
+const MAX_CRAWL_CONCURRENCY = 16;
+const MAX_CRAWL_TIMEOUT_MS = 12e4;
 const SENSITIVE_FIELD_NAMES = /* @__PURE__ */ new Set([
 	"authorization",
 	"cookie",
@@ -9957,33 +11046,53 @@ const SENSITIVE_PARAM_PATTERN = /^(access_?token|auth|api_?key|client_?secret|co
 function uniqueSorted(values = []) {
 	return [...new Set(values.filter(Boolean))].sort();
 }
-function normalizeFetchRate(input) {
+function normalizeFetchRate(input, strategy) {
+	const defaultConcurrency = strategy === "health" ? 4 : 8;
 	return {
-		concurrency: input.fetchRate?.concurrency ?? input.concurrency ?? 8,
+		concurrency: input.fetchRate?.concurrency ?? input.concurrency ?? defaultConcurrency,
 		...input.fetchRate?.intervalCap !== void 0 ? { intervalCap: input.fetchRate.intervalCap } : {},
 		...input.fetchRate?.intervalMs !== void 0 ? { intervalMs: input.fetchRate.intervalMs } : {},
 		...input.fetchRate?.backpressure ? { backpressure: input.fetchRate.backpressure } : {}
 	};
 }
 function normalizeCrawlConfig(input) {
+	const strategy = input.strategy ?? "full";
+	const concurrency = input.concurrency ?? (strategy === "health" ? 4 : 8);
 	return {
 		url: new URL(input.url).toString(),
-		mode: input.mode ?? "site",
+		mode: input.mode ?? (strategy === "health" ? "sitemap" : "site"),
+		strategy,
+		...input.sitemapUrl ? { sitemapUrl: new URL(input.sitemapUrl).toString() } : {},
 		urls: uniqueSorted(input.urls ?? []),
 		maxPages: input.maxPages ?? 500,
 		maxDepth: input.maxDepth ?? 16,
-		concurrency: input.concurrency ?? 8,
+		concurrency,
 		timeoutMs: input.timeoutMs ?? 2e4,
 		include: uniqueSorted(input.include ?? []),
 		exclude: uniqueSorted(input.exclude ?? []),
 		respectRobots: input.respectRobots ?? true,
 		useSitemap: input.useSitemap ?? true,
-		checkExternal: input.checkExternal ?? true,
-		checkAgentDiscovery: input.checkAgentDiscovery ?? false,
-		js: normalizeJavaScriptRenderingMode(input.js),
-		refresh: input.refresh ?? false,
-		fetchRate: normalizeFetchRate(input)
-	};
+		checkExternal: strategy === "health" ? false : input.checkExternal ?? true,
+		checkAgentDiscovery: strategy === "health" ? false : input.checkAgentDiscovery ?? false,
+		js: strategy === "health" ? "off" : normalizeJavaScriptRenderingMode(input.js),
+		refresh: strategy === "health" ? true : input.refresh ?? false,
+		fetchRate: normalizeFetchRate(input, strategy)
+	};
+}
+function assertIntegerRange(value, label, minimum, maximum) {
+	if (!Number.isInteger(value) || value < minimum || value > maximum) throw new RangeError(`${label} must be an integer from ${minimum} to ${maximum}.`);
+}
+function assertCrawlConfigLimits(config) {
+	if (config.strategy === "health" && config.mode !== "sitemap") throw new RangeError("The health strategy requires sitemap mode.");
+	if (config.strategy === "health" && !config.useSitemap) throw new RangeError("The health strategy requires sitemap discovery.");
+	assertIntegerRange(config.maxPages, "maxPages", 1, MAX_CRAWL_PAGES);
+	assertIntegerRange(config.maxDepth, "maxDepth", 0, 64);
+	assertIntegerRange(config.concurrency, "concurrency", 1, 16);
+	assertIntegerRange(config.timeoutMs, "timeoutMs", 1, MAX_CRAWL_TIMEOUT_MS);
+	if (config.fetchRate.concurrency !== void 0) assertIntegerRange(config.fetchRate.concurrency, "fetchRate.concurrency", 1, 16);
+	if (config.strategy === "health" && config.mode !== "sitemap") throw new RangeError("Health strategy requires sitemap mode.");
+	if (config.sitemapUrl && config.mode !== "sitemap") throw new RangeError("sitemapUrl requires sitemap mode.");
+	return config;
 }
 function crawlConfigHash(config) {
 	const normalized = normalizeCrawlConfig(config);
@@ -10100,10 +11209,12 @@ function summarizeCrawlReport(input) {
 	}
 	for (const issue of input.issues) byCategory[issue.category] = (byCategory[issue.category] ?? 0) + 1;
 	const runStats = crawlRunStats(input.pages, input.stats);
+	const indexabilityPages = input.pages.filter((page) => page.auditScope !== "status");
 	return {
 		totalPages: input.pages.length,
-		indexablePages: input.pages.filter((page) => page.indexable).length,
-		nonIndexablePages: input.pages.filter((page) => !page.indexable).length,
+		statusOnlyPages: input.pages.filter((page) => page.auditScope === "status").length,
+		indexablePages: indexabilityPages.filter((page) => page.indexable).length,
+		nonIndexablePages: indexabilityPages.filter((page) => !page.indexable).length,
 		statusErrors: input.pages.filter((page) => page.status === 0 || page.status >= 400).length,
 		...runStats,
 		attemptedRequests: input.requestEvidenceStatus === "unavailable" ? 0 : requests.length,
@@ -10121,6 +11232,23 @@ function summarizeCrawlReport(input) {
 		byCategory
 	};
 }
+const ACCESS_BLOCK_SAMPLE_LIMIT = 10;
+function crawlAccessSummary(requests) {
+	const blocked = requests.flatMap((request) => request.outcome === "response" && request.accessBlock ? [{
+		url: request.requestedUrl,
+		evidence: request.accessBlock
+	}] : []);
+	const providers = {};
+	for (const item of blocked) providers[item.evidence.provider] = (providers[item.evidence.provider] ?? 0) + 1;
+	return {
+		crawler: SEO_CRAWLER_IDENTITY,
+		blockedRequests: blocked.length,
+		providers,
+		samples: blocked.slice(0, ACCESS_BLOCK_SAMPLE_LIMIT),
+		sampleLimit: ACCESS_BLOCK_SAMPLE_LIMIT,
+		truncated: blocked.length > ACCESS_BLOCK_SAMPLE_LIMIT
+	};
+}
 function crawlRunStats(pages, stats = {}) {
 	const discovered = /* @__PURE__ */ new Set();
 	let observedInternalLinks = 0;
@@ -10214,6 +11342,7 @@ function createCrawlReport(input) {
 		status: input.status ?? "completed",
 		configHash: crawlConfigHash(config),
 		config,
+		access: crawlAccessSummary(requests),
 		summary: summarizeCrawlReport({
 			pages,
 			requests,
@@ -10236,16 +11365,19 @@ function createCrawlReport(input) {
 	};
 }
 function normalizeLoadedCrawlReport(report) {
+	const config = sanitizeCrawlConfig(normalizeCrawlConfig(report.config));
 	const requestEvidenceStatus = report.requestEvidenceStatus ?? (Object.hasOwn(report, "requests") ? "available" : "unavailable");
 	const issues = sanitizeIssues(report.issues ?? []);
 	const requests = sortRequests(sanitizeTenantValue(report.requests ?? []) ?? []);
 	const pages = sortPages(sanitizePages(deriveInternalLinkAuthority(report.pages ?? [])));
 	return {
 		...sanitizeTenantValue(report),
+		config,
+		access: crawlAccessSummary(requests),
 		requestEvidenceStatus,
 		requests,
 		definitionId: report.definitionId ?? crawlDefinitionId({
-			config: report.config,
+			config,
 			site: report.site,
 			googleAnalyticsPropertyId: report.googleAnalyticsPropertyId
 		}),
@@ -10266,6 +11398,8 @@ function normalizeLoadedCrawlReport(report) {
 //#endregion
 //#region packages/core/src/analyze/crawler/report-store.ts
 const CRAWL_REPORT_STORAGE_VERSION = 6;
+const AUTOMATIC_BASELINE_SITE_RETENTION = 2;
+const AUTOMATIC_BASELINE_TOTAL_RETENTION = 50;
 function toMeta(row) {
 	return {
 		id: row.id,
@@ -10304,14 +11438,61 @@ function reportFromJson(value) {
 	if (parsed && typeof parsed === "object" && "kind" in parsed && parsed.kind === "seo.crawl_report" && "report" in parsed) return normalizeLoadedCrawlReport(parsed.report);
 	return normalizeLoadedCrawlReport(parsed);
 }
-function saveCrawlReportToSqlite(report) {
-	if (getDb().prepare(`INSERT INTO crawl_reports
-      (id, config_hash, site_url, url, status, total_pages, issue_count, created_at, report_json)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-      ON CONFLICT(id) DO NOTHING`).run(report.id, report.configHash, report.site ?? null, report.config.url, report.status, report.summary.totalPages, report.issues.length, Date.parse(report.generatedAt), JSON.stringify(storageEnvelope(report))).changes === 0) {
-		const existing = getDb().prepare("SELECT * FROM crawl_reports WHERE id = ?").get(report.id);
+function pruneAutomaticBaselines() {
+	getDb().prepare(`WITH ranked AS (
+        SELECT id,
+          ROW_NUMBER() OVER (
+            PARTITION BY site_url
+            ORDER BY created_at DESC, id DESC
+          ) AS site_rank,
+          ROW_NUMBER() OVER (
+            ORDER BY created_at DESC, id DESC
+          ) AS total_rank
+        FROM crawl_reports
+        WHERE retention_class = 'baseline'
+      )
+      DELETE FROM crawl_reports
+      WHERE id IN (
+        SELECT id FROM ranked
+        WHERE site_rank > ? OR total_rank > ?
+      )`).run(2, 50);
+}
+function crawlReportSiteWhere(value) {
+	const siteValues = /* @__PURE__ */ new Set([value]);
+	const origins = /* @__PURE__ */ new Set();
+	try {
+		origins.add(new URL(value).origin);
+	} catch {
+		const host = value.replace(/^sc-domain:/, "").replace(/\/$/, "");
+		if (host && !host.includes("/")) {
+			siteValues.add(`sc-domain:${host}`);
+			origins.add(`https://${host}`);
+			origins.add(`http://${host}`);
+		}
+	}
+	const clauses = [...siteValues].map(() => "site_url = ?");
+	const params = [...siteValues];
+	for (const origin of origins) {
+		clauses.push("url = ?", "url LIKE ?");
+		params.push(`${origin}/`, `${origin}/%`);
+	}
+	return {
+		sql: `(${clauses.join(" OR ")})`,
+		params
+	};
+}
+function saveCrawlReportToSqlite(report, options = {}) {
+	const database = getDb();
+	const retention = options.retention ?? "saved";
+	if (database.prepare(`INSERT INTO crawl_reports
+      (id, config_hash, site_url, url, status, total_pages, issue_count, created_at,
+       retention_class, report_json)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(id) DO NOTHING`).run(report.id, report.configHash, report.site ?? null, report.config.url, report.status, report.summary.totalPages, report.issues.length, Date.parse(report.generatedAt), retention, JSON.stringify(storageEnvelope(report))).changes === 0) {
+		const existing = database.prepare("SELECT * FROM crawl_reports WHERE id = ?").get(report.id);
 		if (existing) return toMeta(existing);
 	}
+	if (retention === "baseline") pruneAutomaticBaselines();
 	return {
 		id: report.id,
 		configHash: report.configHash,
@@ -10326,10 +11507,13 @@ function saveCrawlReportToSqlite(report) {
 }
 function listCrawlReportsFromSqlite(input = {}) {
 	const limit = input.limit ?? 20;
-	return (input.site ? getDb().prepare(`SELECT * FROM crawl_reports
-          WHERE site_url = ?
-          ORDER BY created_at DESC, id DESC
-          LIMIT ?`).all(input.site, limit) : getDb().prepare(`SELECT * FROM crawl_reports
+	return (input.site ? (() => {
+		const where = crawlReportSiteWhere(input.site);
+		return getDb().prepare(`SELECT * FROM crawl_reports
+            WHERE ${where.sql}
+            ORDER BY created_at DESC, id DESC
+            LIMIT ?`).all(...where.params, limit);
+	})() : getDb().prepare(`SELECT * FROM crawl_reports
           ORDER BY created_at DESC, id DESC
           LIMIT ?`).all(limit)).map(toMeta);
 }
@@ -10342,10 +11526,13 @@ function deleteCrawlReportFromSqlite(id) {
 	return getDb().prepare("DELETE FROM crawl_reports WHERE id = ?").run(id).changes > 0;
 }
 function latestCrawlReportFromSqlite(site) {
-	const row = site ? getDb().prepare(`SELECT report_json FROM crawl_reports
-          WHERE site_url = ?
-          ORDER BY created_at DESC, id DESC
-          LIMIT 1`).get(site) : getDb().prepare(`SELECT report_json FROM crawl_reports
+	const row = site ? (() => {
+		const where = crawlReportSiteWhere(site);
+		return getDb().prepare(`SELECT report_json FROM crawl_reports
+            WHERE ${where.sql}
+            ORDER BY created_at DESC, id DESC
+            LIMIT 1`).get(...where.params);
+	})() : getDb().prepare(`SELECT report_json FROM crawl_reports
           ORDER BY created_at DESC, id DESC
           LIMIT 1`).get();
 	if (!row) return void 0;
@@ -10359,8 +11546,8 @@ const sqliteCrawlReportStore = {
 	latest: latestCrawlReportFromSqlite
 };
 const crawlReportStore = sqliteCrawlReportStore;
-function saveCrawlReport(report, store = crawlReportStore) {
-	return store.save(report);
+function saveCrawlReport(report, store = crawlReportStore, options = {}) {
+	return store.save(report, options);
 }
 function listCrawlReports(input = {}, store = crawlReportStore) {
 	return store.list(input);
@@ -10406,6 +11593,86 @@ function cancellationRace(signal) {
 	});
 }
 //#endregion
+//#region packages/core/src/analyze/crawler/crawl-url-queue.ts
+function before(left, right) {
+	return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
+}
+var CrawlUrlQueue = class {
+	#items = [];
+	#positions = /* @__PURE__ */ new Map();
+	get size() {
+		return this.#items.length;
+	}
+	has(url) {
+		return this.#positions.has(url);
+	}
+	push(item) {
+		if (this.has(item.url)) throw new Error(`URL is already queued: ${item.url}`);
+		const index = this.#items.length;
+		this.#items.push(item);
+		this.#positions.set(item.url, index);
+		this.#bubbleUp(index);
+	}
+	decreaseDepth(url, depth) {
+		const index = this.#positions.get(url);
+		if (index === void 0) return false;
+		const item = this.#items[index];
+		if (!item || depth >= item.depth) return false;
+		item.depth = depth;
+		this.#bubbleUp(index);
+		return true;
+	}
+	take() {
+		const first = this.#items[0];
+		if (!first) return void 0;
+		const last = this.#items.pop();
+		this.#positions.delete(first.url);
+		if (last && last !== first) {
+			this.#items[0] = last;
+			this.#positions.set(last.url, 0);
+			this.#bubbleDown(0);
+		}
+		return first;
+	}
+	#swap(left, right) {
+		const leftItem = this.#items[left];
+		const rightItem = this.#items[right];
+		if (!leftItem || !rightItem) return;
+		this.#items[left] = rightItem;
+		this.#items[right] = leftItem;
+		this.#positions.set(rightItem.url, left);
+		this.#positions.set(leftItem.url, right);
+	}
+	#bubbleUp(start) {
+		let index = start;
+		while (index > 0) {
+			const parent = Math.floor((index - 1) / 2);
+			const item = this.#items[index];
+			const parentItem = this.#items[parent];
+			if (!item || !parentItem || !before(item, parentItem)) break;
+			this.#swap(index, parent);
+			index = parent;
+		}
+	}
+	#bubbleDown(start) {
+		let index = start;
+		while (true) {
+			const left = index * 2 + 1;
+			const right = left + 1;
+			let next = index;
+			const item = this.#items[next];
+			const leftItem = this.#items[left];
+			const rightItem = this.#items[right];
+			if (leftItem && item && before(leftItem, item)) next = left;
+			const nextItem = this.#items[next];
+			if (rightItem && nextItem && before(rightItem, nextItem)) next = right;
+			if (next === index) return;
+			this.#swap(index, next);
+			index = next;
+		}
+	}
+};
+//#endregion
 //#region packages/core/src/analyze/crawler/external-link-checks.ts
 const EXTERNAL_LINK_SELECTION_LIMIT = 200;
 const EXTERNAL_LINK_STATES = [
@@ -10607,18 +11874,13 @@ function resolveFetchedAlias(url, aliases) {
 	}
 	return current;
 }
-function resolveLinkGraphAliases(linkGraph, aliases) {
-	const resolved = {};
-	for (const [source, targets] of Object.entries(linkGraph)) {
-		const resolvedSource = resolveFetchedAlias(source, aliases);
-		let resolvedTargets = resolved[resolvedSource];
-		if (!resolvedTargets) {
-			resolvedTargets = /* @__PURE__ */ new Set();
-			resolved[resolvedSource] = resolvedTargets;
-		}
-		for (const target of targets) resolvedTargets.add(resolveFetchedAlias(target, aliases));
+function resolveLinkCountAliases(counts, aliases) {
+	const resolved = /* @__PURE__ */ new Map();
+	for (const [url, count] of counts) {
+		const target = resolveFetchedAlias(url, aliases);
+		resolved.set(target, (resolved.get(target) ?? 0) + count);
 	}
-	return Object.fromEntries(Object.entries(resolved).map(([source, targets]) => [source, [...targets].sort((left, right) => left < right ? -1 : left > right ? 1 : 0)]));
+	return resolved;
 }
 //#endregion
 //#region packages/core/src/analyze/crawler/request-evidence.ts
@@ -10630,7 +11892,9 @@ function observationFromPage(requestedUrl, page) {
 		status: page.status,
 		contentType: page.contentType,
 		durationMs: page.responseTimeMs,
-		redirectChain: page.fetchDiagnostics?.redirectChain
+		redirectChain: page.fetchDiagnostics?.redirectChain,
+		accessBlock: page.accessBlock ?? page.fetchDiagnostics?.accessBlock,
+		robotsTxt: page.robotsTxt
 	};
 	if (page.extractionStatus === "failed") return {
 		...observation,
@@ -10643,18 +11907,225 @@ function observationFromPage(requestedUrl, page) {
 	};
 }
 //#endregion
-//#region packages/core/src/analyze/crawler/site-crawl-providers.ts
-function crawlMetricsWindow(now) {
-	return {
-		...defaultDateRange(28, now),
-		days: 28
-	};
-}
-async function joinAnalytics(input) {
-	let analytics;
+//#region packages/core/src/analyze/crawler/site-crawl-ai-signals.ts
+const MAX_AUXILIARY_RESPONSE_BYTES = 2 * 1024 * 1024;
+const AI_BOT_USER_AGENTS = [
+	"GPTBot",
+	"OAI-SearchBot",
+	"ChatGPT-User",
+	"ClaudeBot",
+	"Claude-User",
+	"Claude-SearchBot",
+	"PerplexityBot",
+	"Perplexity-User",
+	"Google-Extended",
+	"CCBot",
+	"Applebot",
+	"Applebot-Extended",
+	"Bingbot",
+	"Amazonbot",
+	"Bytespider",
+	"Meta-ExternalAgent",
+	"Meta-ExternalFetcher",
+	"DuckAssistBot",
+	"MistralAI-User",
+	"Diffbot",
+	"cohere-ai"
+];
+const AGENT_RESOURCE_PATHS = [
+	"/.well-known/agent.json",
+	"/agent.json",
+	"/.well-known/mcp.json",
+	"/.well-known/ai-plugin.json",
+	"/.well-known/openapi.json",
+	"/openapi.json"
+];
+async function checkLlmsTxt(input) {
+	const llmsUrl = new URL("/llms.txt", input.url).toString();
+	const controller = abortController({
+		timeoutMs: input.timeoutMs,
+		signal: input.signal
+	});
 	try {
-		analytics = await input.fetchLandingPageValues({
-			propertyId: input.propertyId,
+		const response = await input.fetch(llmsUrl, {
+			profile: "bot",
+			redirect: "follow",
+			signal: controller.signal
+		});
+		const contentType = response.headers.get("content-type") ?? "";
+		const exists = response.status >= 200 && response.status < 300 && !/\btext\/html\b/i.test(contentType);
+		await response.body?.cancel().catch(() => void 0);
+		return {
+			url: response.url || llmsUrl,
+			exists,
+			status: response.status
+		};
+	} catch {
+		return {
+			url: llmsUrl,
+			exists: false
+		};
+	} finally {
+		controller.cleanup();
+	}
+}
+function parseRobots(robotsUrl, text) {
+	return robotsParserModule(robotsUrl, text);
+}
+function declaredUserAgents(text) {
+	const declared = /* @__PURE__ */ new Set();
+	for (const match of text.matchAll(/^\s*user-agent\s*:\s*(.+?)\s*$/gim)) {
+		const value = match[1]?.trim().toLowerCase();
+		if (value) declared.add(value);
+	}
+	return declared;
+}
+function contentSignalsFromRobots(text) {
+	const values = /* @__PURE__ */ new Set();
+	for (const match of text.matchAll(/^\s*content-signal\s*:\s*(.+?)\s*$/gim)) {
+		const value = match[1]?.trim();
+		if (value) values.add(value);
+	}
+	return [...values].sort();
+}
+function sitemapUrlsFromRobots(text) {
+	const urls = /* @__PURE__ */ new Set();
+	for (const match of text.matchAll(/^\s*sitemap\s*:\s*(\S+)\s*$/gim)) {
+		const value = match[1]?.trim();
+		if (!value) continue;
+		try {
+			urls.add(new URL(value).toString());
+		} catch {}
+	}
+	return [...urls];
+}
+async function checkRobotsAiAccess(input) {
+	const robotsUrl = new URL("/robots.txt", input.url).toString();
+	const controller = abortController({
+		timeoutMs: input.timeoutMs,
+		signal: input.signal
+	});
+	try {
+		const response = await input.fetch(robotsUrl, {
+			profile: "bot",
+			redirect: "follow",
+			signal: controller.signal
+		});
+		const text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, "robots.txt response");
+		const available = response.status >= 200 && response.status < 300;
+		const absent = response.status >= 400 && response.status < 500 && response.status !== 429;
+		const availability = available ? "available" : [401, 403].includes(response.status) ? "access-blocked" : response.status === 429 ? "rate-limited" : absent ? "absent" : "unreachable";
+		const exists = available;
+		const declared = exists ? declaredUserAgents(text) : /* @__PURE__ */ new Set();
+		const parsed = parseRobots(robotsUrl, exists ? text : "");
+		return {
+			url: response.url || robotsUrl,
+			exists,
+			availability,
+			status: response.status,
+			...["rate-limited", "unreachable"].includes(availability) ? { error: `robots.txt returned HTTP ${response.status}.` } : {},
+			sitemapUrls: exists ? sitemapUrlsFromRobots(text) : [],
+			contentSignals: exists ? contentSignalsFromRobots(text) : [],
+			botAccess: AI_BOT_USER_AGENTS.map((userAgent) => {
+				const lower = userAgent.toLowerCase();
+				return {
+					userAgent,
+					allowed: availability === "rate-limited" || availability === "unreachable" ? null : parsed.isAllowed(input.url, userAgent) ?? true,
+					declared: declared.has(lower),
+					coveredByWildcard: declared.has("*")
+				};
+			})
+		};
+	} catch (error) {
+		return {
+			url: robotsUrl,
+			exists: false,
+			availability: "unreachable",
+			error: error instanceof Error ? error.message : String(error),
+			sitemapUrls: [],
+			botAccess: AI_BOT_USER_AGENTS.map((userAgent) => ({
+				userAgent,
+				allowed: null,
+				declared: false,
+				coveredByWildcard: false
+			}))
+		};
+	} finally {
+		controller.cleanup();
+	}
+}
+async function checkAgentResource(input) {
+	const url = new URL(input.path, input.baseUrl).toString();
+	const controller = abortController({
+		timeoutMs: input.timeoutMs,
+		signal: input.signal
+	});
+	try {
+		const response = await input.fetch(url, {
+			profile: "bot",
+			redirect: "follow",
+			signal: controller.signal
+		});
+		const contentType = response.headers.get("content-type") ?? "";
+		const text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, "Agent resource response");
+		const exists = response.status >= 200 && response.status < 300;
+		let validJson;
+		if (exists && /\bjson\b/i.test(contentType)) try {
+			JSON.parse(text);
+			validJson = true;
+		} catch {
+			validJson = false;
+		}
+		return {
+			url: response.url || url,
+			exists,
+			status: response.status,
+			contentType,
+			...validJson === void 0 ? {} : { validJson }
+		};
+	} catch {
+		return {
+			url,
+			exists: false
+		};
+	} finally {
+		controller.cleanup();
+	}
+}
+async function checkAgentResources(input) {
+	const resources = [];
+	for (const path of AGENT_RESOURCE_PATHS) {
+		if (input.signal?.aborted) break;
+		resources.push(await checkAgentResource({
+			baseUrl: input.url,
+			path,
+			timeoutMs: input.timeoutMs,
+			fetch: input.fetch,
+			signal: input.signal
+		}));
+	}
+	return resources;
+}
+function collectCrawlAiSignals(input) {
+	return Promise.all([
+		checkLlmsTxt(input),
+		checkRobotsAiAccess(input),
+		checkAgentResources(input)
+	]);
+}
+//#endregion
+//#region packages/core/src/analyze/crawler/site-crawl-providers.ts
+function crawlMetricsWindow(now) {
+	return {
+		...defaultDateRange(28, now),
+		days: 28
+	};
+}
+async function joinAnalytics(input) {
+	let analytics;
+	try {
+		analytics = await input.fetchLandingPageValues({
+			propertyId: input.propertyId,
 			startDate: input.window.startDate,
 			endDate: input.window.endDate,
 			limit: input.limit
@@ -11008,7 +12479,9 @@ function responseObservation(requestedUrl, fetched, extraction, extractionError)
 		status: fetched.status,
 		contentType: headerValue(fetched.headers, "content-type"),
 		durationMs: fetched.diagnostics.durationMs,
-		redirectChain: fetched.diagnostics.redirectChain
+		redirectChain: fetched.diagnostics.redirectChain,
+		accessBlock: fetched.diagnostics.accessBlock,
+		robotsTxt: fetched.robotsTxt
 	};
 	if (extraction === "failed") {
 		if (!extractionError) throw new Error("Failed extraction observations require an error.");
@@ -11033,13 +12506,13 @@ function unextractedPage(fetched, contentType) {
 		finalUrl: fetched.finalUrl,
 		status: fetched.status,
 		contentType,
-		responseHeaders: safeResponseHeaders(fetched.headers),
 		responseTimeMs: fetched.diagnostics.durationMs,
 		sizeBytes: Buffer.byteLength(fetched.html),
 		usedJs: fetched.usedJs,
 		fetchSource: fetched.diagnostics.source,
 		cacheState: fetched.diagnostics.cache,
 		fetchDiagnostics: fetched.diagnostics,
+		accessBlock: fetched.diagnostics.accessBlock,
 		blocked: fetched.diagnostics.blocked,
 		robotsTxt: fetched.robotsTxt,
 		xRobotsTag,
@@ -11060,6 +12533,35 @@ function unextractedPage(fetched, contentType) {
 		compression: headerValue(fetched.headers, "content-encoding")
 	};
 }
+function statusOnlyPage(fetched) {
+	const contentType = headerValue(fetched.headers, "content-type");
+	return {
+		url: fetched.finalUrl,
+		finalUrl: fetched.finalUrl,
+		status: fetched.status,
+		contentType,
+		responseHeaders: safeResponseHeaders(fetched.headers),
+		responseTimeMs: fetched.diagnostics.durationMs,
+		usedJs: false,
+		fetchSource: "network",
+		cacheState: "bypass",
+		fetchDiagnostics: fetched.diagnostics,
+		auditScope: "status",
+		accessBlock: fetched.diagnostics.accessBlock,
+		blocked: fetched.diagnostics.blocked,
+		contentAuditAllowed: false,
+		robotsTxt: fetched.robotsTxt,
+		indexable: false,
+		indexability: "Not evaluated by status-only health probe",
+		declaredIndexability: "unknown",
+		extractionStatus: "not-applicable",
+		wordCount: 0,
+		contentHash: "",
+		outgoingInternalCount: 0,
+		outgoingExternalCount: 0,
+		isHttps: new URL(fetched.finalUrl).protocol === "https:"
+	};
+}
 function indexabilityReason(page) {
 	if (page.status < 200 || page.status >= 300) return `Status ${page.status}`;
 	if (hasMetaRobotsDirective(page.metaRobots, "noindex")) return "Meta robots noindex";
@@ -11113,13 +12615,26 @@ async function crawlOne(url, opts = {}) {
 		};
 	}
 	const contentType = headerValue(fetched.headers, "content-type");
+	if (fetched.diagnostics.accessBlock) {
+		const page = unextractedPage(fetched, contentType);
+		page.contentAuditAllowed = false;
+		page.indexable = false;
+		page.indexability = "Crawler access blocked";
+		page.declaredIndexability = "unknown";
+		page.extractionStatus = "not-applicable";
+		return {
+			request: responseObservation(url, fetched, "not-applicable"),
+			page,
+			urls: []
+		};
+	}
 	if (!isHtmlContentType(contentType)) return {
 		request: responseObservation(url, fetched, contentType ? "not-applicable" : "unknown-media-type"),
 		page: unextractedPage(fetched, contentType),
 		urls: []
 	};
 	try {
-		const extracted = await extractPage(fetched);
+		const extracted = await extractPage(fetched, "crawler");
 		const base = new URL(extracted.finalUrl);
 		const internalLinks = extracted.links.map((link) => sameOriginUrl(link.href, base)).filter((value) => Boolean(value));
 		const externalLinks = extracted.links.filter((link) => !link.internal).map((link) => link.href).filter((href) => /^https?:\/\//.test(href));
@@ -11240,6 +12755,62 @@ async function crawlOne(url, opts = {}) {
 		};
 	}
 }
+async function crawlStatusOnly(url, opts = {}) {
+	const startedAt = Date.now();
+	try {
+		const fetched = await fetchPageStatus(url, {
+			...opts,
+			js: "off",
+			refresh: true
+		});
+		return {
+			request: responseObservation(url, fetched, "not-applicable"),
+			page: statusOnlyPage(fetched),
+			urls: []
+		};
+	} catch (error) {
+		if (error instanceof OriginBackpressureError) return {
+			request: {
+				requestedUrl: url,
+				outcome: "skipped",
+				reason: "origin-backpressure",
+				error: error.message,
+				extraction: "not-applicable"
+			},
+			urls: []
+		};
+		if (error instanceof RobotsAccessError) return {
+			request: {
+				requestedUrl: url,
+				outcome: "skipped",
+				reason: error.reason,
+				robotsTxt: {
+					url: error.evidence.url,
+					allowed: error.evidence.allowed,
+					availability: error.evidence.availability,
+					status: error.evidence.status,
+					error: error.evidence.error,
+					matchedLine: error.evidence.matchedLine
+				},
+				extraction: "not-applicable"
+			},
+			urls: []
+		};
+		const message = error instanceof Error ? error.message : String(error);
+		return {
+			request: {
+				requestedUrl: url,
+				outcome: "failure",
+				durationMs: Date.now() - startedAt,
+				failureKind: fetchFailureKind(error, opts.signal),
+				error: message,
+				extraction: "not-applicable"
+			},
+			urls: [],
+			warning: `${url}: ${message}`
+		};
+	}
+}
 //#endregion
 //#region packages/core/src/analyze/monitoring/sitemaps.ts
 const INVALID_LOC_SAMPLE_LIMIT = 10;
@@ -11482,6 +13053,7 @@ async function fetchSitemapUrls(input) {
 	const nestedSitemaps = [];
 	const discoveredSitemaps = /* @__PURE__ */ new Set();
 	const scheduledSitemaps = /* @__PURE__ */ new Set();
+	const fetchedSitemaps = /* @__PURE__ */ new Set();
 	const invalidLocSamples = [];
 	const malformedLastmodSamples = [];
 	const futureLastmodSamples = [];
@@ -11528,12 +13100,18 @@ async function fetchSitemapUrls(input) {
 		const sitemapUrl = queue.shift();
 		if (!sitemapUrl) continue;
 		const fetched = await fetchSitemapXml(sitemapUrl);
+		fetchedSitemaps.add(sitemapUrl);
 		documents.push(fetched.document);
 		if (fetched.document.warning) warnings.push(fetched.document.warning);
 		if (fetched.document.dataStatus !== "unavailable") sitemapsFetched += 1;
 		if (!fetched.xml || !fetched.document.root) continue;
 		const $ = load(fetched.xml, { xmlMode: true });
 		$(fetched.document.root === "urlset" ? "urlset > url" : "").each((_, element) => {
+			if (urls.length >= limit) {
+				urlLimitExceeded = true;
+				omittedUrlsAtLeast += 1;
+				return false;
+			}
 			urlLocs += 1;
 			const entry = $(element);
 			const value = entry.children("loc").first().text().trim();
@@ -11558,12 +13136,7 @@ async function fetchSitemapUrls(input) {
 				return;
 			}
 			seenUrls.add(url);
-			if (urls.length < limit) {
-				urls.push(url);
-				return;
-			}
-			urlLimitExceeded = true;
-			omittedUrlsAtLeast += 1;
+			urls.push(url);
 		});
 		$(fetched.document.root === "sitemapindex" ? "sitemapindex > sitemap" : "").each((_, element) => {
 			sitemapLocs += 1;
@@ -11596,8 +13169,13 @@ async function fetchSitemapUrls(input) {
 				queue.push(child);
 			} else nestedSitemapLimitExceeded = true;
 		});
+		if (urls.length >= limit && queue.length > 0) {
+			urlLimitExceeded = true;
+			omittedUrlsAtLeast = Math.max(omittedUrlsAtLeast, 1);
+			break;
+		}
 	}
-	const unprocessedSitemaps = nestedSitemaps.filter((sitemap) => !scheduledSitemaps.has(sitemap)).length;
+	const unprocessedSitemaps = nestedSitemaps.filter((sitemap) => !fetchedSitemaps.has(sitemap)).length;
 	const possiblyTruncated = urlLimitExceeded || nestedSitemapLimitExceeded || unprocessedSitemaps > 0;
 	if (invalidLocCount) warnings.push(`Ignored ${invalidLocCount} invalid sitemap <loc> ${invalidLocCount === 1 ? "entry" : "entries"}.`);
 	if (possiblyTruncated) warnings.push("Sitemap discovery exceeded a configured URL or sitemap boundary; the returned inventory is incomplete.");
@@ -11777,6 +13355,7 @@ function resolveCrawlSiteDependencies(dependencies = {}) {
 	const emptyPageTopQuery = async () => void 0;
 	return {
 		fetchPage: dependencies.fetchPage ?? crawlOne,
+		fetchStatusPage: dependencies.fetchStatusPage ?? crawlStatusOnly,
 		fetchSitemapUrls: dependencies.fetchSitemapUrls ?? fetchSitemapUrls,
 		fetch: dependencies.fetch ?? publicHttpFetch,
 		queryPageMetrics: (injectedMode === "per-page" ? dependencies.queryPageMetrics : void 0) ?? (injectedMode ? emptyPageMetrics : queryPageMetrics),
@@ -11802,6 +13381,10 @@ function isSameOrigin(url, origin) {
 	}
 }
 function sitemapRootUrls(input) {
+	if (input.sitemapUrl) return [{
+		url: new URL(input.sitemapUrl).toString(),
+		source: "explicit"
+	}];
 	const origin = new URL(input.url).origin;
 	const declared = input.declaredSitemapUrls.filter((url) => isSameOrigin(url, origin));
 	const ignored = input.declaredSitemapUrls.length - declared.length;
@@ -11852,247 +13435,71 @@ async function sitemapSeeds(input) {
 				sitemapUrl: fallbackUrl,
 				limit: input.maxPages
 			});
-			input.warnings.push(...sitemap.warnings);
-			for (const url of sitemap.urls) urls.add(url);
-			roots.push({
-				url: sitemap.sitemapUrl,
-				source: "default-path",
-				dataStatus: sitemap.source.sitemapsFetched > 0 ? sitemap.dataStatus : "unavailable",
-				urlsReturned: sitemap.urls.length,
-				sitemapsFetched: sitemap.source.sitemapsFetched,
-				lastmods: sitemap.source.lastmods,
-				documents: sitemap.source.documents,
-				possiblyTruncated: sitemap.truncation.possiblyTruncated,
-				warnings: sitemap.warnings
-			});
-		}
-	}
-	return {
-		urls: [...urls],
-		discovery: {
-			dataStatus: sitemapDiscoveryStatus(roots),
-			urlsReturned: urls.size,
-			roots
-		}
-	};
-}
-//#endregion
-//#region packages/core/src/analyze/crawler/site-crawl.ts
-const AI_BOT_USER_AGENTS = [
-	"GPTBot",
-	"OAI-SearchBot",
-	"ChatGPT-User",
-	"ClaudeBot",
-	"PerplexityBot",
-	"Google-Extended",
-	"CCBot",
-	"Applebot",
-	"Bingbot"
-];
-const AGENT_RESOURCE_PATHS = [
-	"/.well-known/agent.json",
-	"/agent.json",
-	"/.well-known/mcp.json",
-	"/.well-known/ai-plugin.json",
-	"/.well-known/openapi.json",
-	"/openapi.json"
-];
-function normalizeUrl$1(value, base) {
-	try {
-		const url = base ? new URL(value, base) : new URL(value);
-		url.hash = "";
-		if (!["http:", "https:"].includes(url.protocol)) return void 0;
-		return url.toString();
-	} catch {
-		return;
-	}
-}
-function sameOrigin(url, origin) {
-	try {
-		return new URL(url).origin === origin;
-	} catch {
-		return false;
-	}
-}
-function isLikelyAsset(url) {
-	const path = new URL(url).pathname.toLowerCase();
-	return /\.(avif|bmp|css|csv|dmg|eot|gif|gz|ico|jpeg|jpg|js|json|map|mov|mp3|mp4|ogg|otf|pdf|png|svg|tar|ttf|wav|webm|webp|woff|woff2|zip)$/.test(path);
-}
-function globMatch(pattern, value) {
-	if (!pattern.includes("*")) return value.includes(pattern);
-	const parts = pattern.split("*");
-	let position = 0;
-	for (const part of parts) {
-		if (!part) continue;
-		const found = value.slice(position).indexOf(part);
-		if (found === -1) return false;
-		position += found + part.length;
-	}
-	return true;
-}
-function passesFilters(url, include, exclude) {
-	return (!include.length || include.some((pattern) => globMatch(pattern, url))) && !exclude.some((pattern) => globMatch(pattern, url));
-}
-async function checkLlmsTxt(input) {
-	const llmsUrl = new URL("/llms.txt", input.url).toString();
-	const controller = abortController({
-		timeoutMs: input.timeoutMs,
-		signal: input.signal
-	});
-	try {
-		const response = await input.fetch(llmsUrl, {
-			profile: "bot",
-			redirect: "follow",
-			signal: controller.signal
-		});
-		const contentType = response.headers.get("content-type") ?? "";
-		const exists = response.status >= 200 && response.status < 300 && !/\btext\/html\b/i.test(contentType);
-		await response.body?.cancel().catch(() => void 0);
-		return {
-			url: response.url || llmsUrl,
-			exists,
-			status: response.status
-		};
-	} catch {
-		return {
-			url: llmsUrl,
-			exists: false
-		};
-	} finally {
-		controller.cleanup();
-	}
-}
-function parseRobots(robotsUrl, text) {
-	return robotsParserModule(robotsUrl, text);
-}
-function declaredUserAgents(text) {
-	const declared = /* @__PURE__ */ new Set();
-	for (const match of text.matchAll(/^\s*user-agent\s*:\s*(.+?)\s*$/gim)) {
-		const value = match[1]?.trim().toLowerCase();
-		if (value) declared.add(value);
-	}
-	return declared;
-}
-function sitemapUrlsFromRobots(text) {
-	const urls = /* @__PURE__ */ new Set();
-	for (const match of text.matchAll(/^\s*sitemap\s*:\s*(\S+)\s*$/gim)) {
-		const value = match[1]?.trim();
-		if (!value) continue;
-		try {
-			urls.add(new URL(value).toString());
-		} catch {}
-	}
-	return [...urls];
-}
-async function checkRobotsAiAccess(input) {
-	const robotsUrl = new URL("/robots.txt", input.url).toString();
-	const controller = abortController({
-		timeoutMs: input.timeoutMs,
-		signal: input.signal
-	});
-	try {
-		const response = await input.fetch(robotsUrl, {
-			profile: "bot",
-			redirect: "follow",
-			signal: controller.signal
-		});
-		const text = await response.text();
-		const available = response.status >= 200 && response.status < 300;
-		const absent = response.status >= 400 && response.status < 500 && response.status !== 429;
-		const availability = available ? "available" : [401, 403].includes(response.status) ? "access-blocked" : response.status === 429 ? "rate-limited" : absent ? "absent" : "unreachable";
-		const exists = available;
-		const declared = exists ? declaredUserAgents(text) : /* @__PURE__ */ new Set();
-		const parsed = parseRobots(robotsUrl, exists ? text : "");
-		return {
-			url: response.url || robotsUrl,
-			exists,
-			availability,
-			status: response.status,
-			...["rate-limited", "unreachable"].includes(availability) ? { error: `robots.txt returned HTTP ${response.status}.` } : {},
-			sitemapUrls: exists ? sitemapUrlsFromRobots(text) : [],
-			botAccess: AI_BOT_USER_AGENTS.map((userAgent) => {
-				const lower = userAgent.toLowerCase();
-				return {
-					userAgent,
-					allowed: availability === "rate-limited" || availability === "unreachable" ? null : parsed.isAllowed(input.url, userAgent) ?? true,
-					declared: declared.has(lower),
-					coveredByWildcard: declared.has("*")
-				};
-			})
-		};
-	} catch (error) {
-		return {
-			url: robotsUrl,
-			exists: false,
-			availability: "unreachable",
-			error: error instanceof Error ? error.message : String(error),
-			sitemapUrls: [],
-			botAccess: AI_BOT_USER_AGENTS.map((userAgent) => ({
-				userAgent,
-				allowed: null,
-				declared: false,
-				coveredByWildcard: false
-			}))
-		};
-	} finally {
-		controller.cleanup();
+			input.warnings.push(...sitemap.warnings);
+			for (const url of sitemap.urls) urls.add(url);
+			roots.push({
+				url: sitemap.sitemapUrl,
+				source: "default-path",
+				dataStatus: sitemap.source.sitemapsFetched > 0 ? sitemap.dataStatus : "unavailable",
+				urlsReturned: sitemap.urls.length,
+				sitemapsFetched: sitemap.source.sitemapsFetched,
+				lastmods: sitemap.source.lastmods,
+				documents: sitemap.source.documents,
+				possiblyTruncated: sitemap.truncation.possiblyTruncated,
+				warnings: sitemap.warnings
+			});
+		}
 	}
+	return {
+		urls: [...urls],
+		discovery: {
+			dataStatus: sitemapDiscoveryStatus(roots),
+			urlsReturned: urls.size,
+			roots
+		}
+	};
 }
-async function checkAgentResource(input) {
-	const url = new URL(input.path, input.baseUrl).toString();
-	const controller = abortController({
-		timeoutMs: input.timeoutMs,
-		signal: input.signal
-	});
+//#endregion
+//#region packages/core/src/analyze/crawler/site-crawl.ts
+function normalizeUrl$1(value, base) {
 	try {
-		const response = await input.fetch(url, {
-			profile: "bot",
-			redirect: "follow",
-			signal: controller.signal
-		});
-		const contentType = response.headers.get("content-type") ?? "";
-		const text = await response.text();
-		const exists = response.status >= 200 && response.status < 300;
-		let validJson;
-		if (exists && /\bjson\b/i.test(contentType)) try {
-			JSON.parse(text);
-			validJson = true;
-		} catch {
-			validJson = false;
-		}
-		return {
-			url: response.url || url,
-			exists,
-			status: response.status,
-			contentType,
-			...validJson === void 0 ? {} : { validJson }
-		};
+		const url = base ? new URL(value, base) : new URL(value);
+		url.hash = "";
+		if (!["http:", "https:"].includes(url.protocol)) return void 0;
+		return url.toString();
 	} catch {
-		return {
-			url,
-			exists: false
-		};
-	} finally {
-		controller.cleanup();
+		return;
 	}
 }
-async function checkAgentResources(input) {
-	const resources = [];
-	for (const path of AGENT_RESOURCE_PATHS) {
-		if (input.signal?.aborted) break;
-		resources.push(await checkAgentResource({
-			baseUrl: input.url,
-			path,
-			timeoutMs: input.timeoutMs,
-			fetch: input.fetch,
-			signal: input.signal
-		}));
+function sameOrigin(url, origin) {
+	try {
+		return new URL(url).origin === origin;
+	} catch {
+		return false;
 	}
-	return resources;
+}
+function isLikelyAsset(url) {
+	const path = new URL(url).pathname.toLowerCase();
+	return /\.(avif|bmp|css|csv|dmg|eot|gif|gz|ico|jpeg|jpg|js|json|map|mov|mp3|mp4|ogg|otf|pdf|png|svg|tar|ttf|wav|webm|webp|woff|woff2|zip)$/.test(path);
+}
+function globMatch(pattern, value) {
+	if (!pattern.includes("*")) return value.includes(pattern);
+	const parts = pattern.split("*");
+	let position = 0;
+	for (const part of parts) {
+		if (!part) continue;
+		const found = value.slice(position).indexOf(part);
+		if (found === -1) return false;
+		position += found + part.length;
+	}
+	return true;
+}
+function passesFilters(url, include, exclude) {
+	return (!include.length || include.some((pattern) => globMatch(pattern, url))) && !exclude.some((pattern) => globMatch(pattern, url));
 }
 async function crawlSite(input, dependencies) {
 	const deps = resolveCrawlSiteDependencies(dependencies);
-	const config = normalizeCrawlConfig(input);
+	const config = assertCrawlConfigLimits(normalizeCrawlConfig(input));
 	const { searchMetricsLimit, analyticsLimit } = crawlProviderLimits(input);
 	const signal = input.signal;
 	let cancelled = Boolean(signal?.aborted);
@@ -12100,17 +13507,19 @@ async function crawlSite(input, dependencies) {
 	const site = input.site;
 	const origin = new URL(config.url).origin;
 	const warnings = [];
-	const queue = [];
+	const queue = new CrawlUrlQueue();
 	const visited = /* @__PURE__ */ new Set();
-	const queued = /* @__PURE__ */ new Set();
 	const discovered = /* @__PURE__ */ new Set();
-	const linkGraph = {};
+	const incomingLinkCounts = /* @__PURE__ */ new Map();
 	const fetchedAliases = /* @__PURE__ */ new Map();
 	const pages = [];
 	const requests = [];
 	const documentIndexes = /* @__PURE__ */ new Map();
 	const inFlight = /* @__PURE__ */ new Set();
 	const followLinks = config.mode === "site";
+	const healthStrategy = config.strategy === "health";
+	let activeConcurrency = healthStrategy ? 1 : config.concurrency;
+	let healthyProbeStreak = 0;
 	let queuedUrls = 0;
 	let skippedUrls = 0;
 	const skipReasonCounts = {};
@@ -12134,7 +13543,7 @@ async function crawlSite(input, dependencies) {
 			generatedAt: deps.now().toISOString(),
 			discoveredUrls: discovered.size,
 			queuedUrls,
-			pendingUrls: queue.length,
+			pendingUrls: queue.size,
 			inFlightUrls: inFlight.size,
 			crawledUrls: pages.length,
 			skippedUrls,
@@ -12148,36 +13557,27 @@ async function crawlSite(input, dependencies) {
 	const flushStatusEvents = async () => {
 		await statusEventChain;
 	};
-	const renderer = config.js === "off" ? void 0 : createPageRenderer({ concurrency: config.concurrency });
+	const renderer = config.js === "off" || healthStrategy ? void 0 : createPageRenderer({ concurrency: config.concurrency });
+	const robotsSession = healthStrategy ? createRobotsSession({
+		refresh: true,
+		writeCache: false,
+		timeoutMs: config.timeoutMs,
+		signal
+	}) : void 0;
+	const robotsResolver = robotsSession?.resolve;
 	try {
 		emitStatus("started", {
 			url: config.url,
 			message: `Started crawl for ${config.url}.`
 		});
-		const aiSignals = isCancelled() ? void 0 : await Promise.all([
-			checkLlmsTxt({
-				url: config.url,
-				timeoutMs: config.timeoutMs,
-				fetch: deps.fetch,
-				signal
-			}),
-			checkRobotsAiAccess({
-				url: config.url,
-				timeoutMs: config.timeoutMs,
-				fetch: deps.fetch,
-				signal
-			}),
-			checkAgentResources({
-				url: config.url,
-				timeoutMs: config.timeoutMs,
-				fetch: deps.fetch,
-				signal
-			})
-		]);
-		const llmsTxt = aiSignals?.[0] ?? {
-			url: new URL("/llms.txt", config.url).toString(),
-			exists: false
-		};
+		const aiSignals = isCancelled() || healthStrategy ? void 0 : await collectCrawlAiSignals({
+			url: config.url,
+			timeoutMs: config.timeoutMs,
+			fetch: deps.fetch,
+			signal
+		});
+		const llmsTxt = aiSignals?.[0];
+		const healthSitemapUrls = healthStrategy && !config.sitemapUrl && !isCancelled() ? await robotsSession?.sitemapUrls(origin) : void 0;
 		const robotsTxt = aiSignals?.[1];
 		const agentResources = aiSignals?.[2];
 		cancelled = isCancelled();
@@ -12192,12 +13592,9 @@ async function crawlSite(input, dependencies) {
 				});
 				return;
 			}
-			discovered.add(normalized);
 			if (visited.has(normalized)) return;
-			if (queued.has(normalized)) {
-				const existing = queue.find((item) => item.url === normalized);
-				if (existing && depth < existing.depth) existing.depth = depth;
-				queue.sort((left, right) => left.depth - right.depth || (left.url < right.url ? -1 : left.url > right.url ? 1 : 0));
+			if (queue.has(normalized)) {
+				queue.decreaseDepth(normalized, depth);
 				return;
 			}
 			if (!sameOrigin(normalized, origin)) {
@@ -12227,7 +13624,7 @@ async function crawlSite(input, dependencies) {
 				});
 				return;
 			}
-			if (queue.length + visited.size + inFlight.size >= config.maxPages * 5) {
+			if (queue.size + visited.size + inFlight.size >= config.maxPages * 5) {
 				recordSkip("queue-safety-limit");
 				queueSafetySkippedUrls += 1;
 				emitStatus("url_skipped", {
@@ -12237,13 +13634,12 @@ async function crawlSite(input, dependencies) {
 				});
 				return;
 			}
-			queued.add(normalized);
+			discovered.add(normalized);
 			queuedUrls += 1;
 			queue.push({
 				url: normalized,
 				depth
 			});
-			queue.sort((left, right) => left.depth - right.depth || (left.url < right.url ? -1 : left.url > right.url ? 1 : 0));
 			emitStatus("url_queued", {
 				url: normalized,
 				depth
@@ -12256,8 +13652,9 @@ async function crawlSite(input, dependencies) {
 		if (!isCancelled() && config.useSitemap && (config.mode === "site" || config.mode === "sitemap")) {
 			const sitemap = await sitemapSeeds({
 				url: config.url,
+				sitemapUrl: config.sitemapUrl,
 				maxPages: config.maxPages,
-				declaredSitemapUrls: robotsTxt?.sitemapUrls ?? [],
+				declaredSitemapUrls: robotsTxt?.sitemapUrls ?? healthSitemapUrls ?? [],
 				warnings,
 				fetchSitemapUrls: deps.fetchSitemapUrls
 			});
@@ -12265,8 +13662,8 @@ async function crawlSite(input, dependencies) {
 			sitemapDiscovery = sitemap.discovery;
 		}
 		const nextQueueItem = () => {
-			while (queue.length) {
-				const item = queue.shift();
+			while (queue.size) {
+				const item = queue.take();
 				if (!item) continue;
 				const normalized = normalizeUrl$1(item.url, config.url);
 				if (!normalized) continue;
@@ -12280,15 +13677,17 @@ async function crawlSite(input, dependencies) {
 		};
 		const startTask = (item) => {
 			let task;
+			const fetcher = healthStrategy ? deps.fetchStatusPage : deps.fetchPage;
 			task = {
 				...item,
-				promise: deps.fetchPage(item.url, {
+				promise: fetcher(item.url, {
 					js: config.js,
 					refresh: config.refresh,
 					timeoutMs: config.timeoutMs,
 					rate: config.fetchRate,
 					signal,
 					respectRobots: config.respectRobots,
+					robotsResolver,
 					renderer
 				}).then((result) => ({
 					task,
@@ -12314,8 +13713,8 @@ async function crawlSite(input, dependencies) {
 				depth: item.depth
 			});
 		};
-		while (!isCancelled() && (queue.length || inFlight.size) && pages.length < config.maxPages) {
-			while (!isCancelled() && inFlight.size < config.concurrency && pages.length + inFlight.size < config.maxPages) {
+		while (!isCancelled() && (queue.size || inFlight.size) && pages.length < config.maxPages) {
+			while (!isCancelled() && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
 				const item = nextQueueItem();
 				if (!item) break;
 				startTask(item);
@@ -12336,6 +13735,13 @@ async function crawlSite(input, dependencies) {
 				extraction: "not-applicable"
 			});
 			requests.push(request);
+			if (healthStrategy) if (request.outcome === "response" && request.status < 400 && !request.accessBlock) {
+				healthyProbeStreak += 1;
+				activeConcurrency = Math.min(config.concurrency, 1 + Math.floor(healthyProbeStreak / 20));
+			} else {
+				healthyProbeStreak = 0;
+				activeConcurrency = 1;
+			}
 			if (request.outcome === "skipped") {
 				recordSkip(request.reason === "robots-disallowed" ? "robots-disallowed" : request.reason === "robots-deferred" ? "robots-uncertain" : "origin-backpressure");
 				if (request.reason === "robots-deferred") {
@@ -12398,40 +13804,49 @@ async function crawlSite(input, dependencies) {
 				};
 				const existing = documentIndexes.get(finalUrl);
 				if (existing) {
-					const links = /* @__PURE__ */ new Set([...existing.links, ...normalizedLinks]);
 					const minDepth = Math.min(existing.minDepth, task.depth);
 					const replace = !existing.directlyRequested && directlyRequested || existing.directlyRequested === directlyRequested && sourceRequest < existing.sourceRequest;
 					if (replace) {
+						const existingPage = pages[existing.index];
+						const previousLinks = existing.pendingAliasLinks ?? (existingPage && existingPage.outgoingInternalCount === existingPage.sampleInternalLinks?.length ? new Set(existingPage.sampleInternalLinks) : void 0);
+						const links = new Set(normalizedLinks);
+						if (previousLinks) {
+							for (const target of previousLinks) {
+								const nextCount = (incomingLinkCounts.get(target) ?? 0) - 1;
+								if (nextCount > 0) incomingLinkCounts.set(target, nextCount);
+								else incomingLinkCounts.delete(target);
+							}
+							observedInternalLinks -= previousLinks.size;
+						}
+						for (const target of links) incomingLinkCounts.set(target, (incomingLinkCounts.get(target) ?? 0) + 1);
+						observedInternalLinks += links.size;
+						syncLinkEvidence(page, links);
 						page.crawlDepth = minDepth;
 						pages[existing.index] = page;
 					} else {
 						const existingPage = pages[existing.index];
 						if (existingPage) existingPage.crawlDepth = minDepth;
 					}
-					const retainedPage = pages[existing.index];
-					if (retainedPage) syncLinkEvidence(retainedPage, links);
-					observedInternalLinks += links.size - existing.links.size;
 					documentIndexes.set(finalUrl, {
 						...existing,
 						directlyRequested: existing.directlyRequested || directlyRequested,
 						sourceRequest: replace ? sourceRequest : existing.sourceRequest,
-						links,
+						pendingAliasLinks: void 0,
 						minDepth
 					});
-					linkGraph[finalUrl] = [...links].sort((left, right) => left < right ? -1 : left > right ? 1 : 0);
 				} else {
 					const links = new Set(normalizedLinks);
 					syncLinkEvidence(page, links);
 					pages.push(page);
 					observedInternalLinks += links.size;
+					for (const target of links) incomingLinkCounts.set(target, (incomingLinkCounts.get(target) ?? 0) + 1);
 					documentIndexes.set(finalUrl, {
 						index: pages.length - 1,
 						directlyRequested,
 						sourceRequest,
-						links,
+						pendingAliasLinks: !directlyRequested ? links : void 0,
 						minDepth: task.depth
 					});
-					linkGraph[finalUrl] = [...links].sort((left, right) => left < right ? -1 : left > right ? 1 : 0);
 				}
 			};
 			if (config.respectRobots && page.robotsTxt?.allowed === false) {
@@ -12459,27 +13874,16 @@ async function crawlSite(input, dependencies) {
 			});
 			if (followLinks && task.depth < config.maxDepth) for (const url of result.urls) enqueue(url, task.depth + 1);
 		}
-		const resolvedLinkGraph = resolveLinkGraphAliases(linkGraph, fetchedAliases);
-		const pageIndexes = new Map(pages.map((page, index) => [page.url, index]));
-		const depths = pages.map((page) => page.crawlDepth ?? 0);
-		for (let pass = 0; pass < pages.length; pass += 1) {
-			let changed = false;
-			for (const [sourceUrl, targets] of Object.entries(resolvedLinkGraph)) {
-				const sourceIndex = pageIndexes.get(sourceUrl);
-				if (sourceIndex === void 0) continue;
-				for (const target of targets) {
-					const targetIndex = pageIndexes.get(target);
-					if (targetIndex === void 0) continue;
-					const nextDepth = (depths[sourceIndex] ?? 0) + 1;
-					if (nextDepth >= (depths[targetIndex] ?? Number.POSITIVE_INFINITY)) continue;
-					depths[targetIndex] = nextDepth;
-					changed = true;
-				}
-			}
-			if (!changed) break;
+		const resolvedInlinkCounts = resolveLinkCountAliases(incomingLinkCounts, fetchedAliases);
+		const maxInlinks = Math.max(0, ...pages.map((page) => resolvedInlinkCounts.get(page.finalUrl) ?? 0));
+		for (const page of pages) {
+			const internalInlinkCount = resolvedInlinkCounts.get(page.finalUrl) ?? 0;
+			page.internalInlinkCount = internalInlinkCount;
+			page.internalLinkAuthorityScore = maxInlinks ? Math.round(internalInlinkCount / maxInlinks * 100) : 0;
 		}
-		for (const [index, page] of pages.entries()) page.crawlDepth = depths[index];
 		cancelled = isCancelled();
+		const robotsDisallowed = skipReasonCounts["robots-disallowed"] ?? 0;
+		if (robotsDisallowed > 0) warnings.push(`${robotsDisallowed} URL${robotsDisallowed === 1 ? "" : "s"} disallowed ${SEO_CRAWLER_USER_AGENT} in robots.txt. If the audit should reach them, allow the SEO-Skill token on only those public paths.`);
 		if (cancelled) {
 			warnings.push("Crawl cancelled; returning a partial report.");
 			emitStatus("cancelled", {
@@ -12487,15 +13891,17 @@ async function crawlSite(input, dependencies) {
 				message: "Crawl cancelled; returning a partial report."
 			});
 		}
-		const pageLimitReached = pages.length >= config.maxPages && (queue.length > 0 || inFlight.size > 0);
+		const pageLimitReached = pages.length >= config.maxPages && (queue.size > 0 || inFlight.size > 0);
 		const sitemapEvidencePartial = sitemapDiscovery?.dataStatus === "partial";
-		const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.length > 0 || inFlight.size > 0 || sitemapEvidencePartial;
-		const failed = !cancelled && pages.length === 0 && requests.length > 0 && requests.every((request) => request.outcome === "failure");
+		const accessEvidencePartial = requests.some((request) => request.outcome === "response" && request.accessBlock);
+		const requiredSitemapUnavailable = config.mode === "sitemap" && sitemapDiscovery?.dataStatus === "unavailable";
+		const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
+		const failed = !cancelled && pages.length === 0 && (requiredSitemapUnavailable || requests.length > 0 && requests.every((request) => request.outcome === "failure"));
 		const requestEvidenceStatus = cancelled && inFlight.size > 0 ? "partial" : "available";
 		const dataSources = await crawlDataSources({
 			cancelled,
-			site,
-			googleAnalyticsPropertyId: input.googleAnalyticsPropertyId,
+			site: healthStrategy ? void 0 : site,
+			googleAnalyticsPropertyId: healthStrategy ? void 0 : input.googleAnalyticsPropertyId,
 			pages,
 			warnings,
 			searchMetricsLimit,
@@ -12538,7 +13944,7 @@ async function crawlSite(input, dependencies) {
 			concurrency: Math.min(config.concurrency, 4)
 		}) : void 0;
 		for (const page of pages) {
-			if (!page.geo) continue;
+			if (!page.geo || !llmsTxt) continue;
 			page.geo = {
 				...page.geo,
 				hasLlmsTxt: llmsTxt.exists,
@@ -12555,12 +13961,11 @@ async function crawlSite(input, dependencies) {
 			pages,
 			requests,
 			requestEvidenceStatus,
-			linkGraph: resolvedLinkGraph,
-			ai: {
-				llmsTxt,
+			...!healthStrategy ? { ai: {
+				...llmsTxt ? { llmsTxt } : {},
 				...robotsTxt ? { robotsTxt } : {},
 				...agentResources ? { agentResources } : {}
-			},
+			} } : {},
 			...sitemapDiscovery ? { sitemapDiscovery } : {},
 			...externalLinkVerification ? { externalLinkVerification } : {},
 			...agentDiscovery ? { agentDiscovery } : {},
@@ -12604,7 +14009,7 @@ const REPORT_BASELINE_MAX_AGE_MS = 10080 * 60 * 1e3;
 const defaultDependencies$13 = {
 	latestCrawlReport,
 	crawlSite,
-	saveCrawlReport,
+	saveCrawlReport: (report) => saveCrawlReport(report, crawlReportStore, { retention: "baseline" }),
 	now: () => /* @__PURE__ */ new Date()
 };
 function normalizePathname(value) {
@@ -13921,6 +15326,8 @@ function renderCrawlPretty(report, fixes = topFixes(report)) {
 		`Crawl report for ${report.config.url}`,
 		"",
 		`Status: ${report.status}`,
+		`Strategy: ${report.config.strategy === "health" ? "sitemap health pass" : "full audit"}`,
+		`Crawler: ${report.access.crawler.userAgent}`,
 		`Documents: ${report.summary.totalPages} retained; ${requestSummary}, ${report.summary.discoveredUrls} URLs discovered, ${report.summary.failedUrls} failed, ${report.summary.skippedUrls} skipped`,
 		`Issues: ${report.issues.length} (${report.summary.highIssues} high, ${report.summary.mediumIssues} medium, ${report.summary.lowIssues} low)`
 	];
@@ -13928,6 +15335,14 @@ function renderCrawlPretty(report, fixes = topFixes(report)) {
 		const external = report.externalLinkVerification;
 		lines.push(`External links: ${external.fetchedUrls} fetched, ${external.failedUrls} request failure${external.failedUrls === 1 ? "" : "s"}, ${external.deferredUrls} deferred (${external.dataStatus})`);
 	}
+	if (report.access.blockedRequests > 0) {
+		lines.push("", `Access blocks: ${report.access.blockedRequests}`);
+		for (const sample of report.access.samples.slice(0, 3)) {
+			const requestId = sample.evidence.requestId ? `, request ID ${sample.evidence.requestId}` : "";
+			lines.push(`- ${sample.url}: ${sample.evidence.guidance.summary} (${sample.evidence.provider}${requestId})`, `  Action: ${sample.evidence.guidance.recommendedAction}`, `  Security: ${sample.evidence.guidance.securityNote}`);
+			if (sample.evidence.guidance.documentationUrl) lines.push(`  Provider docs: ${sample.evidence.guidance.documentationUrl}`);
+		}
+	}
 	if (fixes.length) {
 		lines.push("", "Prioritised fixes");
 		for (const fix of fixes.slice(0, 5)) {
@@ -13942,12 +15357,108 @@ function renderCrawlPretty(report, fixes = topFixes(report)) {
 			if (observation.sampleUrls[0]) lines.push(`  First URL: ${observation.sampleUrls[0]}`);
 		}
 	}
-	lines.push("", "Next commands", `- seo crawl ${report.config.url} --json`);
+	lines.push("", "Next commands", report.config.strategy === "health" ? `- seo crawl ${report.config.url} --max-pages ${report.config.maxPages} --json` : `- seo crawl ${report.config.url} --json`);
 	for (const command of [...new Set(fixes.slice(0, 3).map((fix) => fix.verification.command))]) lines.push(`- ${command}`);
 	if (report.warnings.length) lines.push("", "Warnings", ...report.warnings.map((item) => `- ${item}`));
 	if (report.caveats.length) lines.push("", "Caveats", ...report.caveats.map((item) => `- ${item}`));
 	return `${lines.join("\n")}\n`;
 }
+function renderCrawlJunit(report) {
+	const issuesByUrl = /* @__PURE__ */ new Map();
+	for (const issue of report.issues) {
+		const existing = issuesByUrl.get(issue.url) ?? [];
+		existing.push(issue);
+		issuesByUrl.set(issue.url, existing);
+	}
+	const sitemapCases = (report.sitemapDiscovery?.roots ?? []).flatMap((root) => {
+		return (root.documents.length ? root.documents : [{
+			url: root.url,
+			dataStatus: root.dataStatus,
+			compression: "none",
+			warning: root.warnings.join(" ") || void 0
+		}]).map((document) => {
+			const failures = [
+				...document.dataStatus !== "complete" ? [document.warning ?? `Sitemap evidence is ${document.dataStatus}.`] : [],
+				...document.redirected ? [`Sitemap redirected to ${document.finalUrl ?? "an unknown URL"}.`] : [],
+				...document.status !== void 0 && (document.status < 200 || document.status >= 300) ? [`Sitemap returned HTTP ${document.status}.`] : []
+			];
+			const output = [
+				`dataStatus=${document.dataStatus}`,
+				...document.status !== void 0 ? [`status=${document.status}`] : [],
+				...document.finalUrl ? [`finalUrl=${document.finalUrl}`] : [],
+				...document.root ? [`root=${document.root}`] : [],
+				...document.warning ? [`warning=${document.warning}`] : []
+			].join("\n");
+			return {
+				xml: [
+					`  <testcase classname="seo.sitemap-document" name="${escapeXml(document.url)}" time="0.000">`,
+					...failures.length ? [`    <failure message="${escapeXml(failures[0] ?? "Sitemap check failed")}">${escapeXml(failures.join("\n"))}</failure>`] : [],
+					`    <system-out>${escapeXml(output)}</system-out>`,
+					"  </testcase>"
+				].join("\n"),
+				failed: failures.length > 0
+			};
+		});
+	});
+	const requestCases = report.requests.map((request) => {
+		const failures = [
+			...(issuesByUrl.get(request.requestedUrl) ?? []).map((issue) => `${issue.severity}: ${issue.title}${issue.detail ? `: ${issue.detail}` : ""}`),
+			...request.outcome === "failure" ? [`${request.failureKind}: ${request.error}`] : [],
+			...request.outcome === "skipped" ? [request.reason === "origin-backpressure" ? request.error : `${request.reason}: ${request.robotsTxt.matchedLine ?? request.robotsTxt.availability}`] : []
+		];
+		const output = request.outcome === "response" ? [
+			`status=${request.status}`,
+			`finalUrl=${request.finalUrl}`,
+			...request.robotsTxt ? [
+				`robotsUrl=${request.robotsTxt.url}`,
+				`robotsAllowed=${request.robotsTxt.allowed ?? "unknown"}`,
+				`robotsAvailability=${request.robotsTxt.availability}`,
+				...request.robotsTxt.status !== void 0 ? [`robotsStatus=${request.robotsTxt.status}`] : [],
+				...request.robotsTxt.matchedLine ? [`robotsMatchedLine=${request.robotsTxt.matchedLine}`] : [],
+				...request.robotsTxt.error ? [`robotsError=${request.robotsTxt.error}`] : []
+			] : [],
+			...request.accessBlock ? [
+				`accessProvider=${request.accessBlock.provider}`,
+				`crawler=${request.accessBlock.crawler.userAgent}`,
+				...request.accessBlock.requestId ? [`requestId=${request.accessBlock.requestId}`] : []
+			] : []
+		].join("\n") : request.outcome === "failure" ? request.error : request.reason === "origin-backpressure" ? `reason=${request.reason}\nerror=${request.error}` : [
+			`reason=${request.reason}`,
+			`robotsUrl=${request.robotsTxt.url}`,
+			`robotsAllowed=${request.robotsTxt.allowed ?? "unknown"}`,
+			`robotsAvailability=${request.robotsTxt.availability}`,
+			...request.robotsTxt.status !== void 0 ? [`robotsStatus=${request.robotsTxt.status}`] : [],
+			...request.robotsTxt.matchedLine ? [`robotsMatchedLine=${request.robotsTxt.matchedLine}`] : [],
+			...request.robotsTxt.error ? [`robotsError=${request.robotsTxt.error}`] : []
+		].join("\n");
+		const duration = ((request.durationMs ?? 0) / 1e3).toFixed(3);
+		return {
+			xml: [
+				`  <testcase classname="seo.sitemap-health" name="${escapeXml(request.requestedUrl)}" time="${duration}">`,
+				...failures.length ? [`    <failure message="${escapeXml(failures[0] ?? "Health check failed")}">${escapeXml(failures.join("\n"))}</failure>`] : [],
+				`    <system-out>${escapeXml(output)}</system-out>`,
+				"  </testcase>"
+			].join("\n"),
+			failed: failures.length > 0
+		};
+	});
+	const cases = [...sitemapCases, ...requestCases];
+	const failures = cases.filter((item) => item.failed).length;
+	const elapsed = report.requests.reduce((sum, request) => sum + (request.durationMs ?? 0), 0);
+	return [
+		"<?xml version=\"1.0\" encoding=\"UTF-8\"?>",
+		`<testsuite name="seo sitemap health" tests="${cases.length}" failures="${failures}" errors="0" skipped="0" time="${(elapsed / 1e3).toFixed(3)}">`,
+		"  <properties>",
+		`    <property name="report.id" value="${escapeXml(report.id)}"/>`,
+		`    <property name="report.status" value="${escapeXml(report.status)}"/>`,
+		`    <property name="crawl.strategy" value="${escapeXml(report.config.strategy)}"/>`,
+		`    <property name="crawler.userAgent" value="${escapeXml(report.access.crawler.userAgent)}"/>`,
+		"  </properties>",
+		...cases.map((item) => item.xml),
+		"</testsuite>",
+		""
+	].join("\n");
+}
 function renderCrawlHtml(report, fixes = topFixes(report)) {
 	const reviews = reviewObservations(report);
 	const issueRows = report.issueGroups.map((group) => {
@@ -13958,6 +15469,7 @@ function renderCrawlHtml(report, fixes = topFixes(report)) {
 	const reviewItems = reviews.slice(0, 10).map((observation) => `<li><strong>${escapeHtml(observation.title)}</strong><br>${escapeHtml(observation.howToFix)}<br><span>Check before scheduling work. ${escapeHtml(observation.howToVerify)}</span></li>`).join("\n");
 	const caveatItems = report.caveats.map((caveat) => `<li>${escapeHtml(caveat)}</li>`).join("\n");
 	const warningItems = report.warnings.map((warning) => `<li>${escapeHtml(warning)}</li>`).join("\n");
+	const accessItems = report.access.samples.map((sample) => `<li><strong>${escapeHtml(sample.url)}</strong><br>${escapeHtml(sample.evidence.guidance.summary)}<br>${escapeHtml(sample.evidence.guidance.recommendedAction)}<br><span>${escapeHtml(sample.evidence.guidance.securityNote)}</span></li>`).join("\n");
 	return `<!doctype html>
 <html lang="en">
 <head>
@@ -13980,6 +15492,7 @@ function renderCrawlHtml(report, fixes = topFixes(report)) {
 <body>
   <h1>Crawl report</h1>
   <p>${escapeHtml(report.config.url)}</p>
+  <p>Strategy: ${escapeHtml(report.config.strategy === "health" ? "sitemap health pass" : "full audit")}<br>Crawler: <code>${escapeHtml(report.access.crawler.userAgent)}</code></p>
   <section class="summary">
     <div class="metric"><span>Status</span><strong>${escapeHtml(report.status)}</strong></div>
     <div class="metric"><span>Documents</span><strong>${report.summary.totalPages}</strong></div>
@@ -13992,6 +15505,7 @@ function renderCrawlHtml(report, fixes = topFixes(report)) {
     <div class="metric"><span>Issues</span><strong>${report.issues.length}</strong></div>
     <div class="metric"><span>High</span><strong>${report.summary.highIssues}</strong></div>
   </section>
+  ${accessItems ? `<h2>Access blocks</h2><ul>${accessItems}</ul>` : ""}
   <h2>Prioritised fixes</h2>
   ${fixItems ? `<ol>${fixItems}</ol>` : "<p>No prioritised fixes found.</p>"}
   ${reviewItems ? `<h2>Review observations</h2>
@@ -14018,10 +15532,17 @@ function renderCrawlMarkdownTickets(report, fixes = topFixes(report)) {
 		`Source: ${report.config.url}`,
 		`Report: ${report.id}`,
 		`Status: ${report.status}`,
+		`Strategy: ${report.config.strategy}`,
+		`Crawler: ${report.access.crawler.userAgent}`,
 		""
 	];
 	if (report.caveats.length) lines.push("## Caveats", "", ...report.caveats.map((item) => `- ${item}`), "");
 	if (report.warnings.length) lines.push("## Warnings", "", ...report.warnings.map((item) => `- ${item}`), "");
+	if (report.access.blockedRequests > 0) lines.push("## Access blocks", "", ...report.access.samples.flatMap((sample) => [
+		`- ${sample.url}: ${sample.evidence.guidance.summary}`,
+		`  - Action: ${sample.evidence.guidance.recommendedAction}`,
+		`  - Security: ${sample.evidence.guidance.securityNote}`
+	]), "");
 	for (const [index, fix] of fixes.entries()) lines.push(`## ${index + 1}. ${fix.title}`, "", `- [ ] Fix ${fix.count} affected URL${fix.count === 1 ? "" : "s"}`, `- Severity: ${fix.severity}`, `- Rule: ${fix.ruleId}`, `- Why this ranks: ${fix.whyThisRanks}`, `- Plain-English fix: ${fix.howToFix}`, `- Verify: ${fix.howToVerify}`, `- Command: ${fix.verification.command}`, "", `Affected URLs:`, ...fix.sampleUrls.slice(0, 10).map((url) => `- ${url}`), "");
 	if (!fixes.length) lines.push("No implementation tickets were generated.");
 	if (reviews.length) lines.push("## Review observations", "", "These are not implementation tickets. Check the evidence before deciding whether to work on them.", "", ...reviews.slice(0, 10).flatMap((observation) => [`- ${observation.title} (${observation.severity}, ${observation.count} URL${observation.count === 1 ? "" : "s"}): ${observation.howToFix}`, `  - First URL: ${observation.sampleUrls[0] ?? "No sample URL recorded"}`]), "");
@@ -14030,6 +15551,9 @@ function renderCrawlMarkdownTickets(report, fixes = topFixes(report)) {
 function escapeHtml(value) {
 	return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll("\"", "&quot;").replaceAll("'", "&#39;");
 }
+function escapeXml(value) {
+	return escapeHtml(value);
+}
 //#endregion
 //#region packages/core/src/analyze/crawler/history-provenance.ts
 function snapshotCaps(report) {
@@ -14345,6 +15869,14 @@ function compareCrawlReports(input) {
 	};
 }
 //#endregion
+//#region packages/core/src/phrasing.ts
+function plural(count, singular, pluralLabel = `${singular}s`) {
+	return count === 1 ? singular : pluralLabel;
+}
+function countLabel(count, singular, pluralLabel = `${singular}s`) {
+	return `${count.toLocaleString("en-GB")} ${plural(count, singular, pluralLabel)}`;
+}
+//#endregion
 //#region packages/core/src/analyze/crawler/okf.ts
 const OKF_DEFAULT_CONCEPTS = 500;
 const OKF_MAX_CONCEPTS = 5e3;
@@ -14711,7 +16243,7 @@ function explainOkfValidation(validation) {
 	return {
 		title: "OKF bundle",
 		valid: validation.valid,
-		summary: validation.valid ? `This bundle passes seo OKF checks with ${validation.concepts} concept files.` : `This OKF bundle has ${errors.length} validation error${errors.length === 1 ? "" : "s"}.`,
+		summary: validation.valid ? `This bundle passes seo OKF checks with ${countLabel(validation.concepts, "concept file")}.` : `This OKF bundle has ${countLabel(errors.length, "validation error")}.`,
 		files: validation.files,
 		concepts: validation.concepts,
 		errors: errors.length,
@@ -14863,6 +16395,31 @@ const agentDiscoverySchema = z.object({
 		missingMarkdownPages: z.number().int().nonnegative(),
 		consistent: z.boolean().nullable()
 	}),
+	endpointDiscovery: z.object({
+		linkHeader: z.object({
+			url: z.string().url(),
+			status: z.number().int().optional(),
+			entries: z.array(z.object({
+				url: z.string().url(),
+				rel: z.array(z.string()),
+				type: z.string().optional()
+			})),
+			registeredRels: z.array(z.string()),
+			emergingRels: z.array(z.string()),
+			error: z.string().optional()
+		}),
+		endpoints: z.array(z.object({
+			id: z.string(),
+			url: z.string().url(),
+			status: z.number().int().optional(),
+			exists: z.boolean(),
+			contentType: z.string().optional(),
+			validJson: z.boolean().optional(),
+			presentFields: z.array(z.string()).optional(),
+			missingFields: z.array(z.string()).optional(),
+			error: z.string().optional()
+		}))
+	}).optional(),
 	protocolVariants: z.object({
 		http: z.object({
 			url: z.string().url(),
@@ -14934,6 +16491,46 @@ const analyticsSchema = z.object({
 	totalUsers: z.number(),
 	conversions: z.number()
 });
+const crawlerIdentitySchema = z.object({
+	name: z.string(),
+	robotsToken: z.string(),
+	version: z.string(),
+	userAgent: z.string(),
+	documentationUrl: z.string().url()
+});
+const accessBlockEvidenceSchema = z.object({
+	provider: z.enum(["cloudflare", "unknown"]),
+	kind: z.enum([
+		"challenge",
+		"firewall",
+		"rate-limit",
+		"authentication"
+	]),
+	status: z.number().int(),
+	crawler: crawlerIdentitySchema,
+	indicators: z.array(z.string()),
+	requestId: z.string().optional(),
+	guidance: z.object({
+		summary: z.string(),
+		recommendedAction: z.string(),
+		securityNote: z.string(),
+		documentationUrl: z.string().url().optional()
+	})
+});
+const crawlRobotsEvidenceSchema = z.object({
+	url: z.string().url(),
+	allowed: z.boolean().nullable(),
+	availability: z.enum([
+		"available",
+		"absent",
+		"access-blocked",
+		"rate-limited",
+		"unreachable"
+	]),
+	status: z.number().int().optional(),
+	error: z.string().optional(),
+	matchedLine: z.string().optional()
+});
 const geoSignalsSchema = z.object({
 	semanticHtml: z.boolean(),
 	structuredData: z.boolean(),
@@ -15107,6 +16704,7 @@ const crawlPageSnapshotSchema = z.object({
 			consecutiveErrors: z.number().int(),
 			recentP95Ms: z.number().optional()
 		}).optional(),
+		accessBlock: accessBlockEvidenceSchema.optional(),
 		robotsTxt: z.object({
 			url: z.string().url(),
 			cache: z.enum([
@@ -15189,23 +16787,12 @@ const crawlPageSnapshotSchema = z.object({
 		}).optional()
 	}).optional(),
 	blocked: z.boolean().optional(),
+	auditScope: z.enum(["full", "status"]).optional(),
+	accessBlock: accessBlockEvidenceSchema.optional(),
 	contentAuditAllowed: z.boolean().optional(),
 	crawlDepth: z.number().int().optional(),
 	error: z.string().optional(),
-	robotsTxt: z.object({
-		url: z.string().url(),
-		allowed: z.boolean().nullable(),
-		availability: z.enum([
-			"available",
-			"absent",
-			"access-blocked",
-			"rate-limited",
-			"unreachable"
-		]),
-		status: z.number().int().optional(),
-		error: z.string().optional(),
-		matchedLine: z.string().optional()
-	}).optional(),
+	robotsTxt: crawlRobotsEvidenceSchema.optional(),
 	title: z.string().optional(),
 	metaDescription: z.string().optional(),
 	canonical: z.string().url().optional(),
@@ -15260,8 +16847,16 @@ const crawlPageSnapshotSchema = z.object({
 	extractionError: z.string().optional(),
 	wordCount: z.number().int(),
 	contentExtraction: z.object({
-		requested: z.enum(["defuddle", "readability"]),
-		used: z.enum(["defuddle", "readability"]),
+		requested: z.enum([
+			"crawler",
+			"defuddle",
+			"readability"
+		]),
+		used: z.enum([
+			"crawler",
+			"defuddle",
+			"readability"
+		]),
 		fallback: z.boolean(),
 		fallbackReason: z.enum(["defuddle_error", "defuddle_empty"]).optional(),
 		fallbackDetail: z.string().optional(),
@@ -15395,7 +16990,9 @@ const crawlResponseObservationBaseSchema = z.object({
 		url: z.string().url(),
 		status: z.number().int(),
 		location: z.string().url().optional()
-	})).optional()
+	})).optional(),
+	accessBlock: accessBlockEvidenceSchema.optional(),
+	robotsTxt: crawlRobotsEvidenceSchema.optional()
 });
 const crawlRequestObservationSchema = z.union([
 	crawlResponseObservationBaseSchema.extend({ extraction: z.enum([
@@ -15412,20 +17009,7 @@ const crawlRequestObservationSchema = z.union([
 		outcome: z.literal("skipped"),
 		durationMs: z.number().optional(),
 		reason: z.enum(["robots-disallowed", "robots-deferred"]),
-		robotsTxt: z.object({
-			url: z.string().url(),
-			allowed: z.boolean().nullable(),
-			availability: z.enum([
-				"available",
-				"absent",
-				"access-blocked",
-				"rate-limited",
-				"unreachable"
-			]),
-			status: z.number().int().optional(),
-			error: z.string().optional(),
-			matchedLine: z.string().optional()
-		}),
+		robotsTxt: crawlRobotsEvidenceSchema,
 		extraction: z.literal("not-applicable")
 	}),
 	z.object({
@@ -15484,6 +17068,7 @@ const crawlAiSignalsSchema = z.object({
 		status: z.number().int().optional(),
 		error: z.string().optional(),
 		sitemapUrls: z.array(z.string().url()),
+		contentSignals: z.array(z.string()).optional(),
 		botAccess: z.array(z.object({
 			userAgent: z.string(),
 			allowed: z.boolean().nullable(),
@@ -15541,7 +17126,11 @@ const crawlSitemapDiscoverySchema = z.object({
 	urlsReturned: z.number().int().nonnegative(),
 	roots: z.array(z.object({
 		url: z.string().url(),
-		source: z.enum(["robots-txt", "default-path"]),
+		source: z.enum([
+			"explicit",
+			"robots-txt",
+			"default-path"
+		]),
 		dataStatus: z.enum([
 			"complete",
 			"partial",
@@ -15625,6 +17214,8 @@ const crawlConfigSchema = z.object({
 		"list",
 		"sitemap"
 	]),
+	strategy: z.enum(["full", "health"]),
+	sitemapUrl: z.string().url().optional(),
 	urls: z.array(z.string().url()),
 	maxPages: z.number().int(),
 	maxDepth: z.number().int(),
@@ -15659,6 +17250,7 @@ const crawlConfigSchema = z.object({
 });
 const crawlReportSummarySchema = z.object({
 	totalPages: z.number().int(),
+	statusOnlyPages: z.number().int(),
 	indexablePages: z.number().int(),
 	nonIndexablePages: z.number().int(),
 	statusErrors: z.number().int(),
@@ -15749,6 +17341,17 @@ const crawlReportBaseSchema = z.object({
 	]),
 	configHash: z.string(),
 	config: crawlConfigSchema,
+	access: z.object({
+		crawler: crawlerIdentitySchema,
+		blockedRequests: z.number().int().nonnegative(),
+		providers: z.partialRecord(z.enum(["cloudflare", "unknown"]), z.number().int().nonnegative()),
+		samples: z.array(z.object({
+			url: z.string().url(),
+			evidence: accessBlockEvidenceSchema
+		})),
+		sampleLimit: z.number().int().positive(),
+		truncated: z.boolean()
+	}).optional(),
 	summary: crawlReportSummarySchema,
 	pages: z.array(crawlPageSnapshotSchema),
 	issues: z.array(crawlIssueSchema),
@@ -16064,14 +17667,6 @@ async function ctrUnderperformersReport(input, dependencies = defaultDependencie
 	};
 }
 //#endregion
-//#region packages/core/src/phrasing.ts
-function plural(count, singular, pluralLabel = `${singular}s`) {
-	return count === 1 ? singular : pluralLabel;
-}
-function countLabel(count, singular, pluralLabel = `${singular}s`) {
-	return `${count.toLocaleString("en-GB")} ${plural(count, singular, pluralLabel)}`;
-}
-//#endregion
 //#region packages/core/src/analyze/diagnosis-priorities.ts
 function topDelta(report) {
 	const top = report.items[0];
@@ -16098,14 +17693,14 @@ function buildDiagnosisPriorities(input) {
 		const topGroup = input.decay.groups[0];
 		priorities.push({
 			label: "Refresh decaying content",
-			reason: topGroup ? `${input.decay.selection.eligibleRows} observed retained query/page declines found; ${topGroup.label} declined by ${topGroup.totalClickLoss.toFixed(0)} clicks.` : `${input.decay.selection.eligibleRows} observed retained query/page declines found.`,
+			reason: topGroup ? `${countLabel(input.decay.selection.eligibleRows, "observed retained query/page decline")} found; ${topGroup.label} declined by ${topGroup.totalClickLoss.toFixed(0)} clicks.` : `${countLabel(input.decay.selection.eligibleRows, "observed retained query/page decline")} found.`,
 			action: topGroup ? topGroup.recommendation : "Start with declines that continued outside the update window. Check indexability first, then ranking and CTR causes.",
 			confidence: input.decay.dataStatus === "partial" ? "low" : "medium"
 		});
 	}
 	if (input.cannibal.items.length) priorities.push({
 		label: "Review multi-URL query candidates",
-		reason: `${input.cannibal.selection.eligibleClusters} multi-URL query candidates found.`,
+		reason: `${countLabel(input.cannibal.selection.eligibleClusters, "multi-URL query candidate")} found.`,
 		action: "Confirm whether each URL set satisfies the same intent and inspect technical state. Consolidate only verified duplicate or same-intent pages; otherwise clarify the distinction.",
 		confidence: "low"
 	});
@@ -16123,7 +17718,7 @@ function buildDiagnosisPriorities(input) {
 		const top = input.striking.items[0];
 		priorities.push({
 			label: "Investigate striking-distance candidates",
-			reason: `${input.striking.items.length} query/page rows have an average GSC position above 10 and at most 20.`,
+			reason: `${countLabel(input.striking.items.length, "query/page row")} ${input.striking.items.length === 1 ? "has" : "have"} an average GSC position above 10 and at most 20.`,
 			action: top?.recommendation.action ?? "Check technical state, query coverage, competing URLs, and relevant internal links before choosing a change.",
 			confidence: input.striking.source.possiblyTruncated ? "low" : top?.recommendation.confidence ?? "low"
 		});
@@ -17956,8 +19551,7 @@ function analyzeQuickWinsFromRows(input) {
 			selected.selection.belowMinimumRows++;
 			continue;
 		}
-		const targetUrl = row.keys[1] ?? "";
-		const excludedTargetRows = selected.rows.filter((candidate) => samePageUrl(candidate.keys[1] ?? "", targetUrl)).length;
+		const excludedTargetRows = benchmarkContext.urlRowCount(row);
 		const benchmark = benchmarkContext.forUrl(row);
 		if (row.ctr >= benchmark.ctr) {
 			selected.selection.atOrAboveTargetRows++;
@@ -18151,7 +19745,7 @@ async function quickWinsReport(input, dependencies = defaultDependencies$8) {
 		brandTerms: input.brandTerms,
 		includeBrand: input.includeBrand
 	});
-	const verificationRequested = input.verifyContent === true || input.verifyLimit !== void 0;
+	const verificationRequested = input.verifyContent === true || input.verifyContent !== false && input.verifyLimit !== void 0;
 	const verificationItems = verificationRequested ? analysis.items.slice(0, verifyLimit) : [];
 	const fetches = /* @__PURE__ */ new Map();
 	const cachedFetch = (url, options) => {
@@ -18250,7 +19844,7 @@ async function quickWinsReport(input, dependencies = defaultDependencies$8) {
 			repeatedQueryGroups: groups.length,
 			templatePatterns: templateActions.length,
 			brandFiltering: input.includeBrand ? "included" : "excluded",
-			verdict: analysis.dataStatus === "empty" ? "Search Console returned no retained query/page rows for this date window." : analysis.dataStatus === "filtered" ? "No retained rows met the quick-win report criteria." : `${analysis.summary.eligibleRows} eligible CTR-target rows found; ${analysis.summary.returnedRows} returned for review.`
+			verdict: analysis.dataStatus === "empty" ? "Search Console returned no retained query/page rows for this date window." : analysis.dataStatus === "filtered" ? "No retained rows met the quick-win report criteria." : `${countLabel(analysis.summary.eligibleRows, "eligible CTR-target row")} found; ${countLabel(analysis.summary.returnedRows, "row")} returned for review.`
 		},
 		caveats: [
 			`Date window: ${range.startDate} to ${range.endDate}, using final GSC data where available.`,
@@ -18541,7 +20135,7 @@ function templatesFromGroups(groups) {
 function verdict$1(analysis) {
 	if (analysis.dataStatus === "empty") return "Search Console returned no retained query/page rows for this date window.";
 	if (analysis.dataStatus === "filtered") return "No retained query/page rows met the report filters.";
-	return `${analysis.summary.eligibleRows} eligible query/page rows found; ${analysis.summary.returnedRows} returned for review.`;
+	return `${countLabel(analysis.summary.eligibleRows, "eligible query/page row")} found; ${countLabel(analysis.summary.returnedRows, "row")} returned for review.`;
 }
 async function strikingDistance(input, dependencies = defaultDependencies$7) {
 	const days = reportDays(input.days);
@@ -18563,7 +20157,7 @@ async function strikingDistance(input, dependencies = defaultDependencies$7) {
 		includeBrand: input.includeBrand
 	});
 	const verifyLimit = boundedInteger$2(input.verifyLimit, DEFAULT_VERIFY_LIMIT, 0, analysis.limit);
-	const verificationRequested = input.verifyContent === true || input.verifyLimit !== void 0;
+	const verificationRequested = input.verifyContent === true || input.verifyContent !== false && input.verifyLimit !== void 0;
 	const attempted = verificationRequested ? Math.min(verifyLimit, analysis.items.length) : 0;
 	const items = await Promise.all(analysis.items.map(async (item, index) => {
 		if (index >= attempted) return item;
@@ -21141,6 +22735,7 @@ function topCrawlRecommendations(items) {
 }
 //#endregion
 //#region packages/core/src/analyze/monitoring/crawl-store.ts
+const CRAWL_RUN_RETENTION = 20;
 function toRun(row) {
 	return {
 		id: row.id,
@@ -21212,34 +22807,52 @@ function insertCrawlRun(run, pages, recommendations = []) {
 		for (const page of pages) insertPage.run(run.id, page.url, page.finalUrl, page.status, page.title ?? null, page.metaDescription ?? null, page.canonical ?? null, page.metaRobots ?? null, page.xRobotsTag ?? null, page.h1 ?? null, page.indexable ? 1 : 0, page.wordCount, page.contentHash, page.outgoingInternalCount, JSON.stringify(page));
 		for (const recommendation of recommendations) insertRecommendation.run(run.id, run.site, recommendation.url, recommendation.severity, recommendation.category, recommendation.title, recommendation.action, recommendation.confidence, createdAt);
 	})();
+	db.prepare(`DELETE FROM crawl_runs
+    WHERE site_url = ? AND start_url = ?
+      AND id NOT IN (
+        SELECT id FROM crawl_runs
+        WHERE site_url = ? AND start_url = ?
+        ORDER BY created_at DESC, id DESC
+        LIMIT ?
+      )`).run(run.site, run.startUrl, run.site, run.startUrl, 20);
 }
 function latestCrawlSummaries(site, limit = 5) {
-	return getDb().prepare(`WITH page_counts AS (
+	return getDb().prepare(`WITH recent_runs AS (
+        SELECT *
+        FROM crawl_runs
+        WHERE site_url = ?
+        ORDER BY created_at DESC
+        LIMIT ?
+      ),
+      page_counts AS (
         SELECT
-          run_id,
-          COALESCE(SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), 0) AS status_errors,
-          COALESCE(SUM(CASE WHEN indexable = 0 THEN 1 ELSE 0 END), 0) AS non_indexable
+          crawl_pages.run_id,
+          COALESCE(SUM(CASE WHEN crawl_pages.status >= 400 THEN 1 ELSE 0 END), 0) AS status_errors,
+          COALESCE(SUM(CASE WHEN crawl_pages.indexable = 0 THEN 1 ELSE 0 END), 0) AS non_indexable
         FROM crawl_pages
-        GROUP BY run_id
+        INNER JOIN recent_runs ON recent_runs.id = crawl_pages.run_id
+        GROUP BY crawl_pages.run_id
       ),
       recommendation_counts AS (
         SELECT
-          run_id,
+          crawl_recommendations.run_id,
           COUNT(*) AS recommendation_count,
-          COALESCE(SUM(CASE WHEN severity = 'high' THEN 1 ELSE 0 END), 0) AS high_recommendation_count
+          COALESCE(SUM(CASE WHEN crawl_recommendations.severity = 'high' THEN 1 ELSE 0 END), 0) AS high_recommendation_count
         FROM crawl_recommendations
-        GROUP BY run_id
+        INNER JOIN recent_runs ON recent_runs.id = crawl_recommendations.run_id
+        GROUP BY crawl_recommendations.run_id
       ),
       recommendation_rank AS (
         SELECT
           crawl_recommendations.*,
           ROW_NUMBER() OVER (
-            PARTITION BY run_id
+            PARTITION BY crawl_recommendations.run_id
             ORDER BY
-              CASE severity WHEN 'high' THEN 3 WHEN 'medium' THEN 2 ELSE 1 END DESC,
-              created_at DESC
+              CASE crawl_recommendations.severity WHEN 'high' THEN 3 WHEN 'medium' THEN 2 ELSE 1 END DESC,
+              crawl_recommendations.created_at DESC
           ) AS rank
         FROM crawl_recommendations
+        INNER JOIN recent_runs ON recent_runs.id = crawl_recommendations.run_id
       )
       SELECT
         crawl_runs.*,
@@ -21251,14 +22864,12 @@ function latestCrawlSummaries(site, limit = 5) {
         top.title AS top_recommendation_title,
         top.action AS top_recommendation_action,
         top.severity AS top_recommendation_severity
-      FROM crawl_runs
+      FROM recent_runs AS crawl_runs
       LEFT JOIN page_counts ON page_counts.run_id = crawl_runs.id
       LEFT JOIN recommendation_counts ON recommendation_counts.run_id = crawl_runs.id
       LEFT JOIN recommendation_rank top ON top.run_id = crawl_runs.id AND top.rank = 1
-      WHERE crawl_runs.site_url = ?
       GROUP BY crawl_runs.id
-      ORDER BY crawl_runs.created_at DESC
-      LIMIT ?`).all(site, limit).map((row) => ({
+      ORDER BY crawl_runs.created_at DESC`).all(site, limit).map((row) => ({
 		...toRun(row),
 		statusErrors: row.status_errors,
 		nonIndexable: row.non_indexable,
@@ -22320,6 +23931,7 @@ function indexWatchFailureItem(input) {
 //#endregion
 //#region packages/core/src/analyze/monitoring/index-watch.ts
 const MAX_DIRECT_INSPECTIONS = 100;
+const INDEX_WATCH_ATTEMPT_RETENTION = 20;
 function previousFromRow(row) {
 	if (!row.inspected_at) return void 0;
 	return {
@@ -22334,6 +23946,36 @@ function previousFromRow(row) {
 		lastCrawlTime: row.last_crawl_time ?? void 0
 	};
 }
+function pruneIndexWatchSnapshots(rootSite, url, database = getDb()) {
+	return database.prepare(`WITH attempts AS (
+        SELECT id,
+          ROW_NUMBER() OVER (
+            PARTITION BY root_site_url, url
+            ORDER BY inspected_at DESC, id DESC
+          ) AS attempt_rank
+        FROM index_watch_snapshots
+        WHERE root_site_url = ? AND url = ?
+      ), latest_success AS (
+        SELECT id FROM (
+          SELECT id,
+            ROW_NUMBER() OVER (
+              ORDER BY inspected_at DESC, id DESC
+            ) AS success_rank
+          FROM index_watch_snapshots
+          WHERE root_site_url = ? AND url = ?
+            AND inspection_status = 'succeeded'
+            AND error_code IS NULL
+        )
+        WHERE success_rank = 1
+      )
+      DELETE FROM index_watch_snapshots
+      WHERE id IN (
+        SELECT attempts.id
+        FROM attempts
+        LEFT JOIN latest_success ON latest_success.id = attempts.id
+        WHERE attempts.attempt_rank > ? AND latest_success.id IS NULL
+      )`).run(rootSite, url, rootSite, url, 20).changes;
+}
 const defaultDependencies$2 = {
 	inspectUrl,
 	now: () => /* @__PURE__ */ new Date(),
@@ -22347,12 +23989,14 @@ const defaultDependencies$2 = {
 			return row ? previousFromRow(row) : void 0;
 		},
 		insert(item) {
-			getDb().prepare(`INSERT INTO index_watch_snapshots
+			const db = getDb();
+			db.prepare(`INSERT INTO index_watch_snapshots
         (id, site_url, root_site_url, property_site_url, url, verdict,
          coverage_state, indexing_state, robots_txt_state, page_fetch_state,
          google_canonical, user_canonical, last_crawl_time, inspection_status,
          error_code, error_message, inspected_at)
         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(randomUUID(), item.property, item.rootSite, item.property, item.url, item.verdict ?? null, item.coverageState ?? null, item.indexingState ?? null, item.robotsTxtState ?? null, item.pageFetchState ?? null, item.googleCanonical ?? null, item.userCanonical ?? null, item.lastCrawlTime ?? null, item.inspectionStatus, item.errorCode ?? null, item.errorMessage ?? null, Date.parse(item.inspectedAt));
+			pruneIndexWatchSnapshots(item.rootSite, item.url, db);
 		}
 	}
 };
@@ -22896,6 +24540,7 @@ async function indexMonitor(input) {
 }
 //#endregion
 //#region packages/core/src/analyze/monitoring/link-recover-store.ts
+const LINK_RECOVER_RUN_RETENTION = 20;
 function toSummary(row) {
 	return {
 		id: row.id,
@@ -22938,6 +24583,14 @@ function insertLinkRecoverRun(report) {
 		insertRun.run(id, report.site, createdAt, report.range.startDate, report.range.endDate, report.range.days, report.summary.checked, report.summary.recoverable, report.summary.high, report.summary.medium, report.summary.low, report.summary.clicksAtRisk, report.summary.impressionsAtRisk, top?.issue ?? null, top?.url ?? null, top?.recommendation.action ?? null);
 		for (const item of report.items) insertItem.run(id, report.site, item.url, item.finalUrl, item.issue, JSON.stringify(item.issues), item.severity, item.clicks, item.impressions, item.position, item.recommendation.action, createdAt);
 	})();
+	db.prepare(`DELETE FROM link_recover_runs
+    WHERE site_url = ?
+      AND id NOT IN (
+        SELECT id FROM link_recover_runs
+        WHERE site_url = ?
+        ORDER BY created_at DESC, id DESC
+        LIMIT ?
+      )`).run(report.site, report.site, 20);
 	return id;
 }
 function latestLinkRecoverSummary(site) {
@@ -23852,9 +25505,11 @@ function cacheSet(report, now) {
 		...report,
 		raw: void 0
 	};
+	const reportJson = JSON.stringify(stored);
 	getDb().prepare(`INSERT OR REPLACE INTO performance_reports
       (id, url, strategy, report_json, created_at, expires_at)
-      VALUES (?, ?, ?, ?, ?, ?)`).run(report.id, report.url, report.strategy, JSON.stringify(stored), Date.parse(report.generatedAt), now + CACHE_TTL_MS);
+      VALUES (?, ?, ?, ?, ?, ?)`).run(report.id, report.url, report.strategy, reportJson, Date.parse(report.generatedAt), now + CACHE_TTL_MS);
+	noteCacheWrite(Buffer.byteLength(reportJson));
 }
 async function fallbackReport(input) {
 	const fetched = await fetchPage(input.url, {
@@ -25403,7 +27058,7 @@ function clusterRecommendation(input) {
 	return `This ${clusterName} cluster already gets clicks. Use it to refine page structure, FAQs, and internal anchors rather than creating duplicate pages.`;
 }
 function clusterSummary(input) {
-	return `${input.queries} queries, ${input.impressions.toFixed(0)} impressions, ${input.clicks.toFixed(0)} clicks, average position ${input.position.toFixed(1)}.`;
+	return `${countLabel(input.queries, "query", "queries")}, ${countLabel(input.impressions, "impression")}, ${countLabel(input.clicks, "click")}, average position ${input.position.toFixed(1)}.`;
 }
 function clusterLabel(rows) {
 	return [...rows].sort((a, b) => b.impressions - a.impressions || compareQueryClusterText(a.query, b.query))[0]?.query ?? "";
@@ -25876,7 +27531,7 @@ function contentOpportunityBullets(report) {
 	if (report.cannibalization.selection.suppressedQueries) bullets.push(cannibalSuppressionLine(report));
 	if (report.summary.strikingDistanceItems) {
 		const top = report.strikingDistance.items[0];
-		bullets.push(top ? `${report.summary.strikingDistanceItems} query/page ${plural$1(report.summary.strikingDistanceItems, "candidate")} have an average GSC position above 10 and at most 20. Start by investigating "${top.query}" on ${top.url}; it averages position ${top.position.toFixed(1)} with ${top.impressions.toFixed(0)} impressions.` : `${report.summary.strikingDistanceItems} query/page ${plural$1(report.summary.strikingDistanceItems, "candidate")} have an average GSC position above 10 and at most 20.`);
+		bullets.push(top ? `${countLabel(report.summary.strikingDistanceItems, "query/page candidate")} ${plural$1(report.summary.strikingDistanceItems, "has", "have")} an average GSC position above 10 and at most 20. Start by investigating "${top.query}" on ${top.url}; it averages position ${top.position.toFixed(1)} with ${top.impressions.toFixed(0)} impressions.` : `${countLabel(report.summary.strikingDistanceItems, "query/page candidate")} ${plural$1(report.summary.strikingDistanceItems, "has", "have")} an average GSC position above 10 and at most 20.`);
 	}
 	if (!skippedReason(report, "quick-win opportunities") && report.summary.quickWinItems) {
 		const top = report.quickWins.items[0];
@@ -26357,10 +28012,10 @@ function applyVerification(item, coverage) {
 function verdict(report) {
 	if (report.dataStatus === "empty") return "Google Search Console returned no retained query/page rows for this window.";
 	if (report.dataStatus === "filtered") return "No pages met the position, brand, and minimum-impression criteria.";
-	if (report.technical > 0) return `${report.eligible} eligible average-position pages found; ${report.technical} returned pages have verified technical issues to fix before content changes.`;
-	if (report.failed > 0) return `${report.eligible} eligible average-position pages found; ${report.failed} returned pages could not be verified and need a fetch check before content changes.`;
-	if (report.content > 0) return `${report.eligible} eligible average-position pages found; ${report.content} returned pages have verified content or search-framing gaps.`;
-	return `${report.eligible} eligible average-position pages found; ${report.returned} are returned in priority order for investigation.`;
+	if (report.technical > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.technical, "returned page")} ${plural(report.technical, "has", "have")} verified technical issues to fix before content changes.`;
+	if (report.failed > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.failed, "returned page")} could not be verified and ${plural(report.failed, "needs", "need")} a fetch check before content changes.`;
+	if (report.content > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.content, "returned page")} ${plural(report.content, "has", "have")} verified content or search-framing gaps.`;
+	return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.returned, "page")} ${plural(report.returned, "is", "are")} returned in priority order for investigation.`;
 }
 function reportRecommendations(items) {
 	const first = items[0];
@@ -26407,7 +28062,7 @@ async function secondPage(input, dependencies = defaultDependencies) {
 	});
 	const generatedAt = now.toISOString();
 	const warnings = [];
-	const verificationRequested = input.verifyContent === true || input.verifyLimit !== void 0;
+	const verificationRequested = input.verifyContent === true || input.verifyContent !== false && input.verifyLimit !== void 0;
 	const verificationItems = verificationRequested ? analysis.items.slice(0, verifyLimit) : [];
 	for (const item of verificationItems) {
 		const coverage = await verifyQueryContent({
@@ -27155,7 +28810,7 @@ async function refreshPrioritiesWorkflow(input) {
 	const diagnosis = await diagnoseProperty({
 		...input,
 		verifyContent: input.verifyContent ?? true,
-		verifyLimit: input.verifyLimit ?? Math.min(input.limit ?? 5, 5)
+		verifyLimit: input.verifyContent === false ? void 0 : input.verifyLimit ?? Math.min(input.limit ?? 5, 5)
 	});
 	const range = defaultDateRange$1(input.days ?? 28);
 	const analytics = await fetchLandingPageValues({
@@ -28249,11 +29904,76 @@ function ga4MatchReason(match, site) {
 	return `Matched Google Analytics web stream ${stream} (${match.defaultUri}): its hostname ${location} ${site}.`;
 }
 //#endregion
+//#region packages/core/src/storage/log-retention.ts
+const MEBIBYTE = 1024 * 1024;
+const LOG_MAX_FILE_BYTES = 8 * MEBIBYTE;
+const LOG_MAX_TOTAL_BYTES = 64 * MEBIBYTE;
+const LOG_MAX_AGE_MS = 336 * 60 * 60 * 1e3;
+function logFiles(directory) {
+	if (!existsSync(directory)) return [];
+	return readdirSync(directory, { withFileTypes: true }).filter((entry) => entry.isFile() && entry.name.endsWith(".log")).map((entry) => {
+		const path = join(directory, entry.name);
+		const stats = statSync(path);
+		return {
+			path,
+			size: stats.size,
+			modifiedAt: stats.mtimeMs
+		};
+	});
+}
+function rotatedPath(path, now) {
+	const directory = dirname(path);
+	const stem = basename(path, ".log");
+	let suffix = 0;
+	while (true) {
+		const candidate = join(directory, `${stem}.${now}${suffix ? `-${suffix}` : ""}.log`);
+		if (!existsSync(candidate)) return candidate;
+		suffix += 1;
+	}
+}
+function pruneLogs(options = {}) {
+	const directory = options.directory ?? getSeoCliPaths().logDir;
+	const now = options.now ?? Date.now();
+	const maxFileBytes = options.maxFileBytes ?? 8388608;
+	const maxTotalBytes = options.maxTotalBytes ?? 67108864;
+	const maxAgeMs = options.maxAgeMs ?? 12096e5;
+	mkdirSync(directory, {
+		recursive: true,
+		mode: 448
+	});
+	let rotated = 0;
+	let removed = 0;
+	for (const file of logFiles(directory)) {
+		if (file.size <= maxFileBytes) continue;
+		renameSync(file.path, rotatedPath(file.path, now));
+		rotated += 1;
+	}
+	for (const file of logFiles(directory)) {
+		if (file.modifiedAt >= now - maxAgeMs) continue;
+		rmSync(file.path, { force: true });
+		removed += 1;
+	}
+	const retained = logFiles(directory).sort((left, right) => left.modifiedAt - right.modifiedAt || (left.path < right.path ? -1 : left.path > right.path ? 1 : 0));
+	let sizeBytes = retained.reduce((total, file) => total + file.size, 0);
+	for (const file of retained) {
+		if (sizeBytes <= maxTotalBytes) break;
+		rmSync(file.path, { force: true });
+		sizeBytes -= file.size;
+		removed += 1;
+	}
+	return {
+		rotated,
+		removed,
+		sizeBytes
+	};
+}
+//#endregion
 //#region packages/core/src/logger.ts
 let loggerInstance;
 function createLogger(runId) {
 	if (loggerInstance) return loggerInstance.child({ runId });
 	const paths = ensureSeoCliDirs();
+	pruneLogs({ directory: paths.logDir });
 	const date = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
 	const fileStream = createWriteStream(join(paths.logDir, `${date}.log`), { flags: "a" });
 	const destination = process.stderr.isTTY ? pretty({
@@ -28321,6 +30041,7 @@ function parseSemicolonCsv(text) {
 //#endregion
 //#region packages/core/src/providers/semrush/cache.ts
 const BASE_URL = "https://api.semrush.com/";
+const MAX_RESPONSE_BYTES = 10 * 1024 * 1024;
 function estimateUsd(units) {
 	return units / 1e3 * .05;
 }
@@ -28350,14 +30071,17 @@ async function cachedSemrushCall(endpoint, params, map, ttlMs, creditsPerLine, r
 	const url = new URL(BASE_URL);
 	url.search = new URLSearchParams(requestParams).toString();
 	const response = await fetch$1(url);
-	const text = await response.text();
+	const text = await readBoundedResponseText(response, MAX_RESPONSE_BYTES, "Semrush response");
 	if (!response.ok || text.startsWith("ERROR ::")) throw new Error(text || `Semrush request failed with ${response.status}.`);
 	const rows = parseSemicolonCsv(text);
 	const data = map(rows);
 	const credits = Math.max(0, rows.length - 1) * creditsPerLine;
+	const requestJson = JSON.stringify(requestParams);
+	const responseJson = JSON.stringify(data);
 	db.prepare(`INSERT OR REPLACE INTO semrush_cache
     (endpoint, query_hash, request_json, response_json, credits_used, fetched_at, expires_at)
-    VALUES (?, ?, ?, ?, ?, ?, ?)`).run(endpoint, queryHash, JSON.stringify(requestParams), JSON.stringify(data), credits, Date.now(), Date.now() + ttlMs);
+    VALUES (?, ?, ?, ?, ?, ?, ?)`).run(endpoint, queryHash, requestJson, responseJson, credits, Date.now(), Date.now() + ttlMs);
+	noteCacheWrite(Buffer.byteLength(requestJson) + Buffer.byteLength(responseJson));
 	return {
 		data,
 		usage: {
@@ -28554,9 +30278,6 @@ var JsonFileStorageAdapter = class {
 };
 const localCredentialsProvider = new LocalCredentialsProvider();
 //#endregion
-//#region packages/core/src/version.ts
-const SEO_VERSION = version;
-//#endregion
 //#region packages/core/src/telemetry.ts
 const TELEMETRY_ENDPOINT = "https://seoskill.dev/api/t";
 const TELEMETRY_SCHEMA_VERSION = 1;
@@ -28953,6 +30674,6 @@ function isoDate(daysAgo = 0) {
 	return date.toISOString().slice(0, 10);
 }
 //#endregion
-export { AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, AI_SEARCH_SCORECARD_METHODOLOGY_ID, AI_SEARCH_SCORECARD_METHODOLOGY_VERSION, AUDIT_PAGE_RULE_IDS, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CRAWL_REPORT_STORAGE_VERSION, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_DEFAULT_MIN_IMPRESSIONS, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DataForSeoProvider, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LocalCredentialsProvider, OKF_DEFAULT_CONCEPTS, OKF_MAX_CONCEPTS, REPORT_BASELINE_MAX_AGE_MS, REPORT_BASELINE_MAX_DEPTH, REPORT_BASELINE_MAX_PAGES, SCORECARD_CHECK_WEIGHTS, SCORECARD_STATUS_CREDIT, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SemrushProvider, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_NOTICE, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, affectedUrls, agentReadiness, aiPromptsForQuery, aiReadiness, aiReferralSourceForValue, aiReferralsReport, aiSearchScorecard, allocateIndexUrlsToProperties, analyticsConnectionsSchema, analyzeAiReferralRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertUrlMatchesGscProperty, attachCrawlRecommendations, auditCrawlPages, auditCrawlRequests, auditLlmsTxt, auditPage, authStatus, boundedSitemapInventory, buildOkfBundle, buildPseoAuditReportFromRows, buildTelemetryPayload, cannibalReport, changeMeasurementCsvRows, checkDatabaseReadiness, classifyCommunityIntent, clearCache, clientProfileSchema, clusterPseoTemplates, clusterQueryRows, collectAgentDiscovery, combineRobotsValues, communityIntentReport, compareCrawlPages, compareCrawlReports, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlDiff, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportStore, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createContentGroup, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, decayClusterDrafts, decayComparisonRange, decayingReport, deleteChange, deleteClient, deleteContentGroup, deleteCrawlReport, deleteTokens, deriveBrandTerms, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensureSeoCliDirs, entityReadiness, estimateSerpTitleWidth, expectedCtrForPosition, explainOkfValidation, explainRule, fetchCruxFieldData, fetchPage, fetchSearchStatusIncidents, fetchSitemapUrls, fileSize, finalGscDateRange, findOverlappingSearchUpdates, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, generateLlmsTxt, geoGaps, geoGapsReport, getAuthModeStatus, getCacheStats, getChange, getClient, getClientConfig, getContentGroup, getDb, getKeywordProvider, getPreviousRun, getPrivacySnapshot, getRepeatedLinkRecoverUrls, getRunPages, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleAnalyticsPropertyId, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoveragePlan, indexCoverageSignals, indexMonitor, indexWatch, inferPseoTemplate, inferTemplateMovement, initializeTelemetry, insertCrawlRun, insertLinkRecoverRun, inspectUrl, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isCompatibleTechnicalBaseline, isCurrentTechnicalBaseline, isEntityEvidenceSchemaType, isSiteIdentitySchemaType, isSkippableReportError, isTelemetryCi, isTelemetryReport, isoDate, isoWeek, latestCrawlReport, latestCrawlSummaries, latestFinalGscDate, latestIndexWatchSummary, latestLinkRecoverSummary, linkRecover, linkRecoverRecommendation, listChanges, listClients, listContentGroups, listCrawlReports, listGa4AccountSummaries, listGa4DataStreams, listRules, listSearchUpdates, listSites, loadCrawlReport, localCredentialsProvider, loginWithLoopback, makeRunId, matchGa4WebStreams, measureChange, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, monitoringStatus, monthlyReport, monthlyReportWorkflow, narrativeCsvFiles, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, okfConceptLimit, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, parsePseoPath, performanceAudit, performanceReportIsCacheable, planDueIndexUrls, planIndexCoverageFromUrls, plural, propertyMatchesUrl, providerPreferenceSchema, pseoAuditOptions, pseoAuditReport, pseoCsvFiles, pseoIndexStatus, pseoPresentation, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, readConfig, readOauthClient, readTokens, recommendCrawlDiffItem, recommendationForRule, recordChange, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, renderCrawlCsv, renderCrawlHtml, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderPseoMarkdown, renderWorkflowMarkdown, reportNarrative, reportPresentation, requestHeaders, resolveSegmentRanges, resolveSeoCliPaths, resolveTechnicalBaseline, reviewObservations, roundedPosition, runDoctor, runGa4Report, saveClient, saveCrawlReport, secondPage, segmentComparisonRange, segmentImpact, segmentRangeDays, selectAffectedUrls, selectDueIndexUrls, seoErrorEnvelope, seoToAiQueryReport, sessionSourceInListFilter, setDefaultClient, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteSchema, sqliteCrawlReportStore, strikingDistance, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, technicalWatchWorkflow, telemetryErrorCategory, templateForUrl, toSeoError, tokenSchema, topCrawlRecommendations, topFixes, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, unavailableSegmentImpactReport, unicodeTokens, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateOkfFiles, verifyQueryContent, workflowPresentation, writeConfig, writeOauthClient, writeTokens, xRobotsDirectives };
+export { AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, AI_SEARCH_SCORECARD_METHODOLOGY_ID, AI_SEARCH_SCORECARD_METHODOLOGY_VERSION, AUDIT_PAGE_RULE_IDS, AUTOMATIC_BASELINE_SITE_RETENTION, AUTOMATIC_BASELINE_TOTAL_RETENTION, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CRAWL_REPORT_STORAGE_VERSION, CRAWL_RUN_RETENTION, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_DEFAULT_MIN_IMPRESSIONS, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DataForSeoProvider, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, INDEX_WATCH_ATTEMPT_RETENTION, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LINK_RECOVER_RUN_RETENTION, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, OKF_DEFAULT_CONCEPTS, OKF_MAX_CONCEPTS, REPORT_BASELINE_MAX_AGE_MS, REPORT_BASELINE_MAX_DEPTH, REPORT_BASELINE_MAX_PAGES, ResponseSizeLimitError, SCORECARD_CHECK_WEIGHTS, SCORECARD_STATUS_CREDIT, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SemrushProvider, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_NOTICE, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, affectedUrls, agentReadiness, aiPromptsForQuery, aiReadiness, aiReferralSourceForValue, aiReferralsReport, aiSearchScorecard, allocateIndexUrlsToProperties, analyticsConnectionsSchema, analyzeAiReferralRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, attachCrawlRecommendations, auditCrawlPages, auditCrawlRequests, auditLlmsTxt, auditPage, authStatus, boundedSitemapInventory, buildOkfBundle, buildPseoAuditReportFromRows, buildTelemetryPayload, cannibalReport, changeMeasurementCsvRows, checkDatabaseReadiness, classifyCommunityIntent, clearCache, clientProfileSchema, clusterPseoTemplates, clusterQueryRows, collectAgentDiscovery, combineRobotsValues, communityIntentReport, compareCrawlPages, compareCrawlReports, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlDiff, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportStore, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createContentGroup, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, decayClusterDrafts, decayComparisonRange, decayingReport, deleteChange, deleteClient, deleteContentGroup, deleteCrawlReport, deleteTokens, deriveBrandTerms, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensureSeoCliDirs, entityReadiness, estimateSerpTitleWidth, expectedCtrForPosition, explainOkfValidation, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fetchText, fileSize, finalGscDateRange, findOverlappingSearchUpdates, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, generateLlmsTxt, geoGaps, geoGapsReport, getAuthModeStatus, getCacheStats, getChange, getClient, getClientConfig, getContentGroup, getDb, getKeywordProvider, getPreviousRun, getPrivacySnapshot, getRepeatedLinkRecoverUrls, getRunPages, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleAnalyticsPropertyId, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoveragePlan, indexCoverageSignals, indexMonitor, indexWatch, inferPseoTemplate, inferTemplateMovement, initializeTelemetry, insertCrawlRun, insertLinkRecoverRun, inspectUrl, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isCompatibleTechnicalBaseline, isCurrentTechnicalBaseline, isEntityEvidenceSchemaType, isSiteIdentitySchemaType, isSkippableReportError, isTelemetryCi, isTelemetryReport, isoDate, isoWeek, latestCrawlReport, latestCrawlSummaries, latestFinalGscDate, latestIndexWatchSummary, latestLinkRecoverSummary, linkEntries, linkRecover, linkRecoverRecommendation, listChanges, listClients, listContentGroups, listCrawlReports, listGa4AccountSummaries, listGa4DataStreams, listRules, listSearchUpdates, listSites, loadCrawlReport, localCredentialsProvider, loginWithLoopback, makeRunId, matchGa4WebStreams, measureChange, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, monitoringStatus, monthlyReport, monthlyReportWorkflow, narrativeCsvFiles, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, noteCacheWrite, okfConceptLimit, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, parsePseoPath, performanceAudit, performanceReportIsCacheable, planDueIndexUrls, planIndexCoverageFromUrls, plural, propertyMatchesUrl, providerPreferenceSchema, pruneIndexWatchSnapshots, pruneLogs, pseoAuditOptions, pseoAuditReport, pseoCsvFiles, pseoIndexStatus, pseoPresentation, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, readBoundedResponseText, readConfig, readOauthClient, readTokens, recommendCrawlDiffItem, recommendationForRule, recordChange, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderPseoMarkdown, renderWorkflowMarkdown, reportNarrative, reportPresentation, requestHeaders, resolveSegmentRanges, resolveSeoCliPaths, resolveTechnicalBaseline, reviewObservations, roundedPosition, runDoctor, runGa4Report, safeError, saveClient, saveCrawlReport, secondPage, segmentComparisonRange, segmentImpact, segmentRangeDays, selectAffectedUrls, selectDueIndexUrls, seoErrorEnvelope, seoToAiQueryReport, sessionSourceInListFilter, setDefaultClient, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteSchema, sqliteCrawlReportStore, strikingDistance, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, technicalWatchWorkflow, telemetryErrorCategory, templateForUrl, toSeoError, tokenSchema, topCrawlRecommendations, topFixes, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, unavailableSegmentImpactReport, unicodeTokens, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateOkfFiles, verifyQueryContent, workflowPresentation, writeConfig, writeOauthClient, writeTokens, xRobotsDirectives };
 
 //# sourceMappingURL=index.js.map