seo 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +9 -4
- package/README.md +2 -2
- package/dist/cli.js +79 -71
- package/dist/cli.js.map +1 -1
- package/dist/{dist-eiewVfiS.js → dist-BG734YzY.js} +1 -1
- package/dist/{dist-CTBhWxn6.js → dist-BnXs3Jen.js} +62 -62
- package/dist/dist-BnXs3Jen.js.map +1 -0
- package/dist/index.d.ts +2544 -41
- package/dist/index.js +1599 -180
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +1663 -164
- package/dist/mcp.js.map +1 -1
- package/dist/{package-BTOI9tde.js → package-Jl7x3QBx.js} +2 -2
- package/dist/{package-BTOI9tde.js.map → package-Jl7x3QBx.js.map} +1 -1
- package/evals/agent-readiness.json +58 -0
- package/package.json +5 -2
- package/skills/seo/SKILL.md +3 -2
- package/dist/dist-CTBhWxn6.js.map +0 -1
package/dist/mcp.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { n as load, t as version } from "./package-
|
|
2
|
+
import { n as load, t as version } from "./package-Jl7x3QBx.js";
|
|
3
3
|
import { Headers, fetch as fetch$1 } from "undici";
|
|
4
4
|
import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
5
5
|
import { homedir } from "node:os";
|
|
@@ -1732,7 +1732,7 @@ function effectiveSnippetControl(input) {
|
|
|
1732
1732
|
}
|
|
1733
1733
|
//#endregion
|
|
1734
1734
|
//#region packages/core/dist/extract/canonical.js
|
|
1735
|
-
function headerValue$
|
|
1735
|
+
function headerValue$4(headers, name) {
|
|
1736
1736
|
const lower = name.toLowerCase();
|
|
1737
1737
|
return Object.entries(headers).find(([key]) => key.toLowerCase() === lower)?.[1];
|
|
1738
1738
|
}
|
|
@@ -1816,7 +1816,7 @@ function parameter(value, name) {
|
|
|
1816
1816
|
return match?.[1] ?? match?.[2] ?? match?.[3];
|
|
1817
1817
|
}
|
|
1818
1818
|
function headerCandidates(headers, baseUrl) {
|
|
1819
|
-
const link = headerValue$
|
|
1819
|
+
const link = headerValue$4(headers, "link");
|
|
1820
1820
|
if (!link) return [];
|
|
1821
1821
|
const candidates = [];
|
|
1822
1822
|
for (const item of splitLinkHeader(link)) {
|
|
@@ -2485,7 +2485,7 @@ function safeText$1(value) {
|
|
|
2485
2485
|
const trimmed = value?.replace(/\s+/g, " ").trim();
|
|
2486
2486
|
return trimmed ? trimmed : void 0;
|
|
2487
2487
|
}
|
|
2488
|
-
function headerValue$
|
|
2488
|
+
function headerValue$3(headers, name) {
|
|
2489
2489
|
const target = name.toLowerCase();
|
|
2490
2490
|
return Object.entries(headers).find(([key]) => key.toLowerCase() === target)?.[1];
|
|
2491
2491
|
}
|
|
@@ -2584,6 +2584,7 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
2584
2584
|
});
|
|
2585
2585
|
else invalidHreflangCount += 1;
|
|
2586
2586
|
}
|
|
2587
|
+
const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
|
|
2587
2588
|
const openGraph = Object.fromEntries($("meta[property^=\"og:\"]").toArray().map((element) => [$(element).attr("property") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
2588
2589
|
const twitter = Object.fromEntries($("meta[name^=\"twitter:\"]").toArray().map((element) => [$(element).attr("name") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
2589
2590
|
const structuredData = extractStructuredData($, fetchResult.finalUrl);
|
|
@@ -2631,13 +2632,14 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
2631
2632
|
title: safeText$1($("title").first().text()),
|
|
2632
2633
|
metaDescription: safeText$1($("meta[name=\"description\"]").attr("content")),
|
|
2633
2634
|
metaRobots,
|
|
2634
|
-
xRobotsTag: safeText$1(headerValue$
|
|
2635
|
+
xRobotsTag: safeText$1(headerValue$3(fetchResult.headers, "x-robots-tag")),
|
|
2635
2636
|
canonical: canonicalEvidence.selectedRaw,
|
|
2636
2637
|
canonicalEvidence,
|
|
2637
2638
|
lang: safeText$1($("html").attr("lang")),
|
|
2638
2639
|
hasViewport: Boolean($("meta[name=\"viewport\"]").attr("content")),
|
|
2639
2640
|
headings,
|
|
2640
2641
|
links,
|
|
2642
|
+
markdownAlternates,
|
|
2641
2643
|
hreflang,
|
|
2642
2644
|
jsonLd: structuredData.jsonLd,
|
|
2643
2645
|
invalidJsonLdCount: structuredData.invalidJsonLdSamples.length,
|
|
@@ -3442,7 +3444,7 @@ function safeText(value) {
|
|
|
3442
3444
|
const normalized = value?.replace(/\s+/g, " ").trim();
|
|
3443
3445
|
return normalized ? normalized.slice(0, MAX_TEXT_VALUE_LENGTH) : void 0;
|
|
3444
3446
|
}
|
|
3445
|
-
function headerValue$
|
|
3447
|
+
function headerValue$2(headers, name) {
|
|
3446
3448
|
const expected = name.toLowerCase();
|
|
3447
3449
|
return Object.entries(headers).find(([key]) => key.toLowerCase() === expected)?.[1];
|
|
3448
3450
|
}
|
|
@@ -3495,7 +3497,7 @@ function renderingDocumentSnapshot(result) {
|
|
|
3495
3497
|
const structuredData = extractStructuredData($, result.finalUrl);
|
|
3496
3498
|
const metaRobots = normalizedDirectiveValue([metaValue($, "robots")]);
|
|
3497
3499
|
const googlebotRobots = normalizedDirectiveValue([metaValue($, "googlebot")]);
|
|
3498
|
-
const httpRobots = normalizedDirectiveValue([headerValue$
|
|
3500
|
+
const httpRobots = normalizedDirectiveValue([headerValue$2(result.headers, "x-robots-tag")]);
|
|
3499
3501
|
const headings = $("h1, h2, h3, h4, h5, h6").toArray().map((element) => ({
|
|
3500
3502
|
level: Number.parseInt(element.tagName.slice(1), 10),
|
|
3501
3503
|
text: safeText($(element).text())
|
|
@@ -6030,8 +6032,1088 @@ function matchesFilters(issue, filters) {
|
|
|
6030
6032
|
return true;
|
|
6031
6033
|
}
|
|
6032
6034
|
//#endregion
|
|
6033
|
-
//#region packages/core/dist/analyze/crawler/
|
|
6035
|
+
//#region packages/core/dist/analyze/crawler/agent-discovery.js
|
|
6036
|
+
const MAX_BODY_BYTES = 2e6;
|
|
6037
|
+
const MAX_LLMS_LINKS = 100;
|
|
6038
|
+
const MAX_CURATED_LLMS_BYTES = 1e5;
|
|
6039
|
+
const MAX_SKILLS = 25;
|
|
6040
|
+
function sha256(value) {
|
|
6041
|
+
return createHash("sha256").update(value).digest("hex");
|
|
6042
|
+
}
|
|
6043
|
+
function headerValue$1(headers, name) {
|
|
6044
|
+
if (!headers) return void 0;
|
|
6045
|
+
const target = name.toLowerCase();
|
|
6046
|
+
return Object.entries(headers).find(([key]) => key.toLowerCase() === target)?.[1];
|
|
6047
|
+
}
|
|
6048
|
+
function linkEntries(value, base) {
|
|
6049
|
+
if (!value) return [];
|
|
6050
|
+
return value.split(/,(?=\s*<)/u).map((entry) => {
|
|
6051
|
+
const match = entry.match(/^\s*<([^>]+)>\s*(.*)$/u);
|
|
6052
|
+
if (!match?.[1]) return void 0;
|
|
6053
|
+
try {
|
|
6054
|
+
const url = new URL(match[1], base).toString();
|
|
6055
|
+
const parameters = match[2] ?? "";
|
|
6056
|
+
const rel = parameters.match(/(?:^|;)\s*rel=(?:"([^"]+)"|([^;\s]+))/iu);
|
|
6057
|
+
const type = parameters.match(/(?:^|;)\s*type=(?:"([^"]+)"|([^;\s]+))/iu);
|
|
6058
|
+
return {
|
|
6059
|
+
url,
|
|
6060
|
+
rel: (rel?.[1] ?? rel?.[2] ?? "").toLowerCase().split(/\s+/u),
|
|
6061
|
+
type: (type?.[1] ?? type?.[2] ?? "").toLowerCase()
|
|
6062
|
+
};
|
|
6063
|
+
} catch {
|
|
6064
|
+
return;
|
|
6065
|
+
}
|
|
6066
|
+
}).filter((entry) => Boolean(entry));
|
|
6067
|
+
}
|
|
6068
|
+
function normalizedDocumentUrl(value) {
|
|
6069
|
+
const url = new URL(value);
|
|
6070
|
+
url.hash = "";
|
|
6071
|
+
if (url.pathname !== "/") url.pathname = url.pathname.replace(/\/$/u, "");
|
|
6072
|
+
return url.toString();
|
|
6073
|
+
}
|
|
6074
|
+
function sameDocument(left, right) {
|
|
6075
|
+
if (!left || !right) return null;
|
|
6076
|
+
try {
|
|
6077
|
+
return normalizedDocumentUrl(left) === normalizedDocumentUrl(right);
|
|
6078
|
+
} catch {
|
|
6079
|
+
return false;
|
|
6080
|
+
}
|
|
6081
|
+
}
|
|
6082
|
+
function bodyWordCount(value) {
|
|
6083
|
+
return value.replace(/^---[\s\S]*?---\s*/u, "").replace(/```[\s\S]*?```/gu, " ").replace(/[#>*_`[\]()|:-]/gu, " ").split(/\s+/u).filter(Boolean).length;
|
|
6084
|
+
}
|
|
6085
|
+
function repeatedProseLines(markdown) {
|
|
6086
|
+
const counts = /* @__PURE__ */ new Map();
|
|
6087
|
+
let inFrontmatter = false;
|
|
6088
|
+
let fence;
|
|
6089
|
+
for (const [index, rawLine] of markdown.split(/\r?\n/u).entries()) {
|
|
6090
|
+
const line = rawLine.trim();
|
|
6091
|
+
if (index === 0 && line === "---") {
|
|
6092
|
+
inFrontmatter = true;
|
|
6093
|
+
continue;
|
|
6094
|
+
}
|
|
6095
|
+
if (inFrontmatter) {
|
|
6096
|
+
if (line === "---") inFrontmatter = false;
|
|
6097
|
+
continue;
|
|
6098
|
+
}
|
|
6099
|
+
if (!fence && (line.startsWith("```") || line.startsWith("~~~"))) {
|
|
6100
|
+
fence = line.startsWith("```") ? "```" : "~~~";
|
|
6101
|
+
continue;
|
|
6102
|
+
}
|
|
6103
|
+
if (fence) {
|
|
6104
|
+
if (line.startsWith(fence)) fence = void 0;
|
|
6105
|
+
continue;
|
|
6106
|
+
}
|
|
6107
|
+
if (line.length < 40 || /^(?:#{1,6}\s|[-+*]\s|\d+[.)]\s|>|\||`|\]\()/u.test(line)) continue;
|
|
6108
|
+
counts.set(line, (counts.get(line) ?? 0) + 1);
|
|
6109
|
+
}
|
|
6110
|
+
return [...counts.values()].reduce((duplicates, count) => duplicates + Math.max(0, count - 1), 0);
|
|
6111
|
+
}
|
|
6112
|
+
function responseIsNoindex(response, body) {
|
|
6113
|
+
if ((response.headers.get("x-robots-tag") ?? "").toLowerCase().includes("noindex")) return true;
|
|
6114
|
+
return [...body.matchAll(/<meta\s+[^>]*>/giu)].some((match) => {
|
|
6115
|
+
const tag = match[0];
|
|
6116
|
+
const name = tag.match(/\bname\s*=\s*["']([^"']+)["']/iu)?.[1];
|
|
6117
|
+
const content = tag.match(/\bcontent\s*=\s*["']([^"']+)["']/iu)?.[1];
|
|
6118
|
+
return name?.toLowerCase() === "robots" && /\bnoindex\b/iu.test(content ?? "");
|
|
6119
|
+
});
|
|
6120
|
+
}
|
|
6121
|
+
function markdownQuality(markdown, page) {
|
|
6122
|
+
const introProbe = (page.contentSample?.replace(/\s+/gu, " ").trim())?.split(" ").slice(0, 10).join(" ");
|
|
6123
|
+
const normalizedMarkdown = markdown.replace(/\s+/gu, " ");
|
|
6124
|
+
const codeFences = markdown.match(/^```/gmu)?.length ?? 0;
|
|
6125
|
+
const wordCount = bodyWordCount(markdown);
|
|
6126
|
+
const sourceWordCount = page.wordCount ?? 0;
|
|
6127
|
+
return {
|
|
6128
|
+
frontmatterTitle: /^---[\s\S]*?^title:\s*.+$/mu.test(markdown),
|
|
6129
|
+
h1Count: markdown.match(/^#\s+\S.+$/gmu)?.length ?? 0,
|
|
6130
|
+
codeFenceBalanced: codeFences % 2 === 0,
|
|
6131
|
+
tableRows: markdown.match(/^\s*\|.*\|\s*$/gmu)?.length ?? 0,
|
|
6132
|
+
links: markdown.match(/\[[^\]]+\]\([^)]+\)/gu)?.length ?? 0,
|
|
6133
|
+
wordCount,
|
|
6134
|
+
rawHtmlTags: markdown.match(/<(?:div|span|section|article|nav|button)\b/giu)?.length ?? 0,
|
|
6135
|
+
rawSvgTags: markdown.match(/<svg\b/giu)?.length ?? 0,
|
|
6136
|
+
rawScriptTags: markdown.match(/<script\b/giu)?.length ?? 0,
|
|
6137
|
+
rawStyleTags: markdown.match(/<style\b/giu)?.length ?? 0,
|
|
6138
|
+
suspiciousConcatenations: markdown.match(/\b[a-z]{4,}[A-Z][A-Za-z]{3,}\b/gu)?.length ?? 0,
|
|
6139
|
+
repeatedLines: repeatedProseLines(markdown),
|
|
6140
|
+
sourceWordCount,
|
|
6141
|
+
wordRetentionRatio: sourceWordCount > 0 ? Math.round(wordCount / sourceWordCount * 1e3) / 1e3 : null,
|
|
6142
|
+
introductoryCopyRetained: introProbe ? normalizedMarkdown.includes(introProbe) : null,
|
|
6143
|
+
navigationOnly: bodyWordCount(markdown) < 25
|
|
6144
|
+
};
|
|
6145
|
+
}
|
|
6146
|
+
function combinedSignal(timeoutMs, signal) {
|
|
6147
|
+
const controller = new AbortController();
|
|
6148
|
+
const abort = () => controller.abort();
|
|
6149
|
+
const timer = setTimeout(abort, timeoutMs);
|
|
6150
|
+
if (signal?.aborted) controller.abort();
|
|
6151
|
+
else signal?.addEventListener("abort", abort, { once: true });
|
|
6152
|
+
return {
|
|
6153
|
+
signal: controller.signal,
|
|
6154
|
+
cleanup: () => {
|
|
6155
|
+
clearTimeout(timer);
|
|
6156
|
+
signal?.removeEventListener("abort", abort);
|
|
6157
|
+
}
|
|
6158
|
+
};
|
|
6159
|
+
}
|
|
6160
|
+
async function readBoundedText(response) {
|
|
6161
|
+
if (!response.body) return "";
|
|
6162
|
+
const reader = response.body.getReader();
|
|
6163
|
+
const chunks = [];
|
|
6164
|
+
let size = 0;
|
|
6165
|
+
try {
|
|
6166
|
+
while (true) {
|
|
6167
|
+
const result = await reader.read();
|
|
6168
|
+
if (result.done) break;
|
|
6169
|
+
size += result.value.byteLength;
|
|
6170
|
+
if (size > MAX_BODY_BYTES) throw new Error(`Response exceeds ${MAX_BODY_BYTES} bytes.`);
|
|
6171
|
+
chunks.push(result.value);
|
|
6172
|
+
}
|
|
6173
|
+
} finally {
|
|
6174
|
+
await reader.cancel().catch(() => void 0);
|
|
6175
|
+
}
|
|
6176
|
+
return Buffer.concat(chunks).toString("utf8");
|
|
6177
|
+
}
|
|
6178
|
+
async function fetchRepresentation(input) {
|
|
6179
|
+
const controller = combinedSignal(input.timeoutMs, input.signal);
|
|
6180
|
+
try {
|
|
6181
|
+
const response = await input.fetch(input.url, {
|
|
6182
|
+
profile: "bot",
|
|
6183
|
+
redirect: "follow",
|
|
6184
|
+
headers: { accept: input.accept },
|
|
6185
|
+
signal: controller.signal
|
|
6186
|
+
});
|
|
6187
|
+
const body = await readBoundedText(response);
|
|
6188
|
+
const link = linkEntries(response.headers.get("link") ?? void 0, input.url);
|
|
6189
|
+
const tokens = Number(response.headers.get("x-markdown-tokens"));
|
|
6190
|
+
return {
|
|
6191
|
+
observation: {
|
|
6192
|
+
requestedUrl: input.url,
|
|
6193
|
+
finalUrl: response.url || input.url,
|
|
6194
|
+
status: response.status,
|
|
6195
|
+
contentType: response.headers.get("content-type") ?? void 0,
|
|
6196
|
+
bytes: Buffer.byteLength(body),
|
|
6197
|
+
sha256: sha256(body),
|
|
6198
|
+
canonicalUrl: link.find((entry) => entry.rel.includes("canonical"))?.url,
|
|
6199
|
+
alternateUrl: link.find((entry) => entry.rel.includes("alternate") && entry.type === "text/markdown")?.url,
|
|
6200
|
+
varyAccept: (response.headers.get("vary") ?? "").split(",").some((value) => value.trim().toLowerCase() === "accept"),
|
|
6201
|
+
...Number.isInteger(tokens) && tokens >= 0 ? { markdownTokens: tokens } : {},
|
|
6202
|
+
contentSignal: response.headers.get("content-signal") ?? void 0
|
|
6203
|
+
},
|
|
6204
|
+
body
|
|
6205
|
+
};
|
|
6206
|
+
} catch (error) {
|
|
6207
|
+
return { observation: {
|
|
6208
|
+
requestedUrl: input.url,
|
|
6209
|
+
varyAccept: false,
|
|
6210
|
+
error: error instanceof Error ? error.message : String(error)
|
|
6211
|
+
} };
|
|
6212
|
+
} finally {
|
|
6213
|
+
controller.cleanup();
|
|
6214
|
+
}
|
|
6215
|
+
}
|
|
6216
|
+
async function inspectMarkdownPage(input) {
|
|
6217
|
+
const advertisedUrls = [...new Set(input.page.markdownAlternates ?? [])].sort();
|
|
6218
|
+
const httpAlternateUrls = linkEntries(headerValue$1(input.page.responseHeaders, "link"), input.page.finalUrl).filter((entry) => entry.rel.includes("alternate") && entry.type === "text/markdown").map((entry) => entry.url).sort();
|
|
6219
|
+
const observation = {
|
|
6220
|
+
htmlUrl: input.page.finalUrl,
|
|
6221
|
+
advertisedUrls,
|
|
6222
|
+
htmlAlternateUnique: advertisedUrls.length === 1,
|
|
6223
|
+
httpAlternateUrls,
|
|
6224
|
+
explicitMatchesNegotiated: null,
|
|
6225
|
+
repeatedHashStable: null,
|
|
6226
|
+
markdownCanonicalMatchesHtml: null
|
|
6227
|
+
};
|
|
6228
|
+
if (advertisedUrls.length !== 1) return observation;
|
|
6229
|
+
const explicitUrl = advertisedUrls[0];
|
|
6230
|
+
const [explicit, negotiated, repeated] = await Promise.all([
|
|
6231
|
+
fetchRepresentation({
|
|
6232
|
+
url: explicitUrl,
|
|
6233
|
+
accept: "text/markdown",
|
|
6234
|
+
timeoutMs: input.timeoutMs,
|
|
6235
|
+
fetch: input.fetch,
|
|
6236
|
+
signal: input.signal
|
|
6237
|
+
}),
|
|
6238
|
+
fetchRepresentation({
|
|
6239
|
+
url: input.page.finalUrl,
|
|
6240
|
+
accept: "text/markdown",
|
|
6241
|
+
timeoutMs: input.timeoutMs,
|
|
6242
|
+
fetch: input.fetch,
|
|
6243
|
+
signal: input.signal
|
|
6244
|
+
}),
|
|
6245
|
+
fetchRepresentation({
|
|
6246
|
+
url: explicitUrl,
|
|
6247
|
+
accept: "text/markdown",
|
|
6248
|
+
timeoutMs: input.timeoutMs,
|
|
6249
|
+
fetch: input.fetch,
|
|
6250
|
+
signal: input.signal
|
|
6251
|
+
})
|
|
6252
|
+
]);
|
|
6253
|
+
observation.explicit = explicit.observation;
|
|
6254
|
+
observation.negotiated = negotiated.observation;
|
|
6255
|
+
observation.repeated = repeated.observation;
|
|
6256
|
+
observation.explicitMatchesNegotiated = explicit.observation.sha256 && negotiated.observation.sha256 ? explicit.observation.sha256 === negotiated.observation.sha256 : null;
|
|
6257
|
+
observation.repeatedHashStable = explicit.observation.sha256 && repeated.observation.sha256 ? explicit.observation.sha256 === repeated.observation.sha256 : null;
|
|
6258
|
+
observation.markdownCanonicalMatchesHtml = sameDocument(explicit.observation.canonicalUrl, input.page.finalUrl);
|
|
6259
|
+
if (explicit.body) observation.quality = markdownQuality(explicit.body, input.page);
|
|
6260
|
+
return observation;
|
|
6261
|
+
}
|
|
6262
|
+
async function fetchText(input) {
|
|
6263
|
+
const controller = combinedSignal(input.timeoutMs, input.signal);
|
|
6264
|
+
try {
|
|
6265
|
+
const response = await input.fetch(input.url, {
|
|
6266
|
+
profile: "bot",
|
|
6267
|
+
redirect: input.redirect ?? "follow",
|
|
6268
|
+
headers: input.accept ? { accept: input.accept } : void 0,
|
|
6269
|
+
signal: controller.signal
|
|
6270
|
+
});
|
|
6271
|
+
return {
|
|
6272
|
+
response,
|
|
6273
|
+
body: await readBoundedText(response)
|
|
6274
|
+
};
|
|
6275
|
+
} finally {
|
|
6276
|
+
controller.cleanup();
|
|
6277
|
+
}
|
|
6278
|
+
}
|
|
6279
|
+
function safeError(error) {
|
|
6280
|
+
return error instanceof Error ? error.message : String(error);
|
|
6281
|
+
}
|
|
6282
|
+
async function inspectAgentSkills(input) {
|
|
6283
|
+
const indexUrl = new URL("/.well-known/agent-skills/index.json", input.origin).toString();
|
|
6284
|
+
try {
|
|
6285
|
+
const { response, body } = await fetchText({
|
|
6286
|
+
url: indexUrl,
|
|
6287
|
+
timeoutMs: input.timeoutMs,
|
|
6288
|
+
fetch: input.fetch,
|
|
6289
|
+
signal: input.signal,
|
|
6290
|
+
accept: "application/json"
|
|
6291
|
+
});
|
|
6292
|
+
const contentType = response.headers.get("content-type") ?? void 0;
|
|
6293
|
+
const cors = response.headers.get("access-control-allow-origin") ?? void 0;
|
|
6294
|
+
if (response.status < 200 || response.status >= 300) return {
|
|
6295
|
+
indexUrl,
|
|
6296
|
+
status: response.status,
|
|
6297
|
+
contentType,
|
|
6298
|
+
validIndex: false,
|
|
6299
|
+
skills: []
|
|
6300
|
+
};
|
|
6301
|
+
const parsed = JSON.parse(body);
|
|
6302
|
+
const entries = Array.isArray(parsed.skills) ? parsed.skills.slice(0, MAX_SKILLS) : [];
|
|
6303
|
+
const skills = await Promise.all(entries.map(async (entry) => {
|
|
6304
|
+
const name = typeof entry.name === "string" ? entry.name : void 0;
|
|
6305
|
+
const declaredDigest = typeof entry.digest === "string" ? entry.digest : void 0;
|
|
6306
|
+
const relativeUrl = typeof entry.url === "string" ? entry.url : void 0;
|
|
6307
|
+
if (!relativeUrl) return {
|
|
6308
|
+
name,
|
|
6309
|
+
declaredDigest,
|
|
6310
|
+
digestMatches: null,
|
|
6311
|
+
frontmatterValid: null,
|
|
6312
|
+
sameOrigin: false,
|
|
6313
|
+
error: "Skill URL is missing."
|
|
6314
|
+
};
|
|
6315
|
+
const url = new URL(relativeUrl, indexUrl).toString();
|
|
6316
|
+
const sameOrigin = new URL(url).origin === new URL(input.origin).origin;
|
|
6317
|
+
if (!sameOrigin) return {
|
|
6318
|
+
name,
|
|
6319
|
+
url,
|
|
6320
|
+
declaredDigest,
|
|
6321
|
+
digestMatches: null,
|
|
6322
|
+
frontmatterValid: null,
|
|
6323
|
+
sameOrigin,
|
|
6324
|
+
error: "Skill URL is off origin."
|
|
6325
|
+
};
|
|
6326
|
+
try {
|
|
6327
|
+
const skill = await fetchText({
|
|
6328
|
+
url,
|
|
6329
|
+
timeoutMs: input.timeoutMs,
|
|
6330
|
+
fetch: input.fetch,
|
|
6331
|
+
signal: input.signal,
|
|
6332
|
+
accept: "text/markdown,text/plain;q=0.9"
|
|
6333
|
+
});
|
|
6334
|
+
const observedDigest = `sha256:${sha256(skill.body)}`;
|
|
6335
|
+
const frontmatterValid = /^---\s*\n[\s\S]*?^name:\s*\S.+$[\s\S]*?^description:\s*\S.+$[\s\S]*?^---\s*$/mu.test(skill.body);
|
|
6336
|
+
return {
|
|
6337
|
+
name,
|
|
6338
|
+
url,
|
|
6339
|
+
status: skill.response.status,
|
|
6340
|
+
contentType: skill.response.headers.get("content-type") ?? void 0,
|
|
6341
|
+
cors: skill.response.headers.get("access-control-allow-origin") ?? void 0,
|
|
6342
|
+
declaredDigest,
|
|
6343
|
+
observedDigest,
|
|
6344
|
+
digestMatches: declaredDigest ? declaredDigest === observedDigest : null,
|
|
6345
|
+
frontmatterValid,
|
|
6346
|
+
sameOrigin
|
|
6347
|
+
};
|
|
6348
|
+
} catch (error) {
|
|
6349
|
+
return {
|
|
6350
|
+
name,
|
|
6351
|
+
url,
|
|
6352
|
+
declaredDigest,
|
|
6353
|
+
digestMatches: null,
|
|
6354
|
+
frontmatterValid: null,
|
|
6355
|
+
sameOrigin,
|
|
6356
|
+
error: safeError(error)
|
|
6357
|
+
};
|
|
6358
|
+
}
|
|
6359
|
+
}));
|
|
6360
|
+
return {
|
|
6361
|
+
indexUrl,
|
|
6362
|
+
status: response.status,
|
|
6363
|
+
contentType,
|
|
6364
|
+
cors,
|
|
6365
|
+
validIndex: typeof parsed.$schema === "string" && Array.isArray(parsed.skills),
|
|
6366
|
+
skills
|
|
6367
|
+
};
|
|
6368
|
+
} catch (error) {
|
|
6369
|
+
return {
|
|
6370
|
+
indexUrl,
|
|
6371
|
+
validIndex: false,
|
|
6372
|
+
skills: [],
|
|
6373
|
+
error: safeError(error)
|
|
6374
|
+
};
|
|
6375
|
+
}
|
|
6376
|
+
}
|
|
6377
|
+
function markdownLinks(value, base) {
|
|
6378
|
+
const links = [];
|
|
6379
|
+
const invalidLinks = [];
|
|
6380
|
+
for (const match of value.matchAll(/\[([^\u005d]+)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/gu)) {
|
|
6381
|
+
if (!match[1] || !match[2]) continue;
|
|
6382
|
+
try {
|
|
6383
|
+
links.push({
|
|
6384
|
+
label: match[1],
|
|
6385
|
+
url: new URL(match[2], base).toString()
|
|
6386
|
+
});
|
|
6387
|
+
} catch {
|
|
6388
|
+
invalidLinks.push(match[2]);
|
|
6389
|
+
}
|
|
6390
|
+
}
|
|
6391
|
+
return {
|
|
6392
|
+
links,
|
|
6393
|
+
invalidLinks
|
|
6394
|
+
};
|
|
6395
|
+
}
|
|
6396
|
+
async function inspectLlmsTxt(input) {
|
|
6397
|
+
const url = new URL("/llms.txt", input.origin).toString();
|
|
6398
|
+
try {
|
|
6399
|
+
const [first, second] = await Promise.all([fetchText({
|
|
6400
|
+
url,
|
|
6401
|
+
timeoutMs: input.timeoutMs,
|
|
6402
|
+
fetch: input.fetch,
|
|
6403
|
+
signal: input.signal,
|
|
6404
|
+
accept: "text/plain,text/markdown;q=0.9"
|
|
6405
|
+
}), fetchText({
|
|
6406
|
+
url,
|
|
6407
|
+
timeoutMs: input.timeoutMs,
|
|
6408
|
+
fetch: input.fetch,
|
|
6409
|
+
signal: input.signal,
|
|
6410
|
+
accept: "text/plain,text/markdown;q=0.9"
|
|
6411
|
+
})]);
|
|
6412
|
+
const exists = first.response.status >= 200 && first.response.status < 300;
|
|
6413
|
+
const parsed = markdownLinks(first.body, url);
|
|
6414
|
+
const links = parsed.links.slice(0, MAX_LLMS_LINKS);
|
|
6415
|
+
const linkLimitReached = parsed.links.length > links.length;
|
|
6416
|
+
const counts = /* @__PURE__ */ new Map();
|
|
6417
|
+
for (const link of links) counts.set(link.url, (counts.get(link.url) ?? 0) + 1);
|
|
6418
|
+
const duplicateLinks = [...counts].filter(([, count]) => count > 1).map(([link]) => link).sort();
|
|
6419
|
+
const crawlRoutes = new Set(input.pages.map((page) => normalizedDocumentUrl(page.finalUrl)));
|
|
6420
|
+
const linkObservations = await Promise.all(links.map(async (link) => {
|
|
6421
|
+
const sameOrigin = new URL(link.url).origin === new URL(input.origin).origin;
|
|
6422
|
+
try {
|
|
6423
|
+
const target = await fetchText({
|
|
6424
|
+
url: link.url,
|
|
6425
|
+
timeoutMs: input.timeoutMs,
|
|
6426
|
+
fetch: input.fetch,
|
|
6427
|
+
signal: input.signal
|
|
6428
|
+
});
|
|
6429
|
+
const indexableTarget = sameOrigin && target.response.status >= 200 && target.response.status < 300 ? !responseIsNoindex(target.response, target.body) : void 0;
|
|
6430
|
+
const finalUrl = target.response.url || void 0;
|
|
6431
|
+
return {
|
|
6432
|
+
...link,
|
|
6433
|
+
sameOrigin,
|
|
6434
|
+
status: target.response.status,
|
|
6435
|
+
finalUrl,
|
|
6436
|
+
redirected: target.response.redirected,
|
|
6437
|
+
...indexableTarget === void 0 ? {} : { indexableTarget }
|
|
6438
|
+
};
|
|
6439
|
+
} catch (error) {
|
|
6440
|
+
return {
|
|
6441
|
+
...link,
|
|
6442
|
+
sameOrigin,
|
|
6443
|
+
redirected: false,
|
|
6444
|
+
error: safeError(error)
|
|
6445
|
+
};
|
|
6446
|
+
}
|
|
6447
|
+
}));
|
|
6448
|
+
const linkedCrawlRoutes = new Set(linkObservations.filter((link) => link.sameOrigin && !new URL(link.url).pathname.startsWith("/.well-known/") && !new URL(link.url).pathname.endsWith(".md")).map((link) => normalizedDocumentUrl(link.url)));
|
|
6449
|
+
return {
|
|
6450
|
+
url,
|
|
6451
|
+
exists,
|
|
6452
|
+
status: first.response.status,
|
|
6453
|
+
contentType: first.response.headers.get("content-type") ?? void 0,
|
|
6454
|
+
bytes: Buffer.byteLength(first.body),
|
|
6455
|
+
sha256: sha256(first.body),
|
|
6456
|
+
repeatedHashStable: sha256(first.body) === sha256(second.body),
|
|
6457
|
+
headingCount: first.body.match(/^#{1,2}\s+\S/gmu)?.length ?? 0,
|
|
6458
|
+
totalParsedLinks: parsed.links.length,
|
|
6459
|
+
linkLimitReached,
|
|
6460
|
+
links: linkObservations,
|
|
6461
|
+
invalidLinks: parsed.invalidLinks.sort(),
|
|
6462
|
+
duplicateLinks,
|
|
6463
|
+
offSiteLinks: linkObservations.filter((link) => !link.sameOrigin).map((link) => link.url).sort(),
|
|
6464
|
+
redirectedLinks: linkObservations.filter((link) => link.redirected).map((link) => link.url).sort(),
|
|
6465
|
+
nonIndexableLinks: linkObservations.filter((link) => link.indexableTarget === false).map((link) => link.url).sort(),
|
|
6466
|
+
missingCrawlRoutes: [...linkedCrawlRoutes].filter((route) => !crawlRoutes.has(route)).sort(),
|
|
6467
|
+
oversized: Buffer.byteLength(first.body) > MAX_CURATED_LLMS_BYTES
|
|
6468
|
+
};
|
|
6469
|
+
} catch (error) {
|
|
6470
|
+
return {
|
|
6471
|
+
url,
|
|
6472
|
+
exists: false,
|
|
6473
|
+
repeatedHashStable: null,
|
|
6474
|
+
headingCount: 0,
|
|
6475
|
+
totalParsedLinks: 0,
|
|
6476
|
+
linkLimitReached: false,
|
|
6477
|
+
links: [],
|
|
6478
|
+
invalidLinks: [],
|
|
6479
|
+
duplicateLinks: [],
|
|
6480
|
+
offSiteLinks: [],
|
|
6481
|
+
redirectedLinks: [],
|
|
6482
|
+
nonIndexableLinks: [],
|
|
6483
|
+
missingCrawlRoutes: [],
|
|
6484
|
+
oversized: false,
|
|
6485
|
+
error: safeError(error)
|
|
6486
|
+
};
|
|
6487
|
+
}
|
|
6488
|
+
}
|
|
6489
|
+
async function inspectRouteManifest(input) {
|
|
6490
|
+
const url = new URL("/agent-routes.json", input.origin).toString();
|
|
6491
|
+
try {
|
|
6492
|
+
const result = await fetchText({
|
|
6493
|
+
url,
|
|
6494
|
+
timeoutMs: input.timeoutMs,
|
|
6495
|
+
fetch: input.fetch,
|
|
6496
|
+
signal: input.signal,
|
|
6497
|
+
accept: "application/json"
|
|
6498
|
+
});
|
|
6499
|
+
const parsed = JSON.parse(result.body);
|
|
6500
|
+
const declared = Array.isArray(parsed.pages) ? parsed.pages : [];
|
|
6501
|
+
const declaredHtmlRoutes = declared.map((page) => page.htmlPath).filter((path) => typeof path === "string").sort();
|
|
6502
|
+
const declaredMarkdownRoutes = declared.map((page) => page.markdownPath).filter((path) => typeof path === "string").sort();
|
|
6503
|
+
const crawledHtmlRoutes = new Set(input.pages.map((page) => new URL(page.finalUrl).pathname.replace(/\/$/u, "") || "/"));
|
|
6504
|
+
const observedMarkdownRoutes = new Set(input.observations.flatMap((page) => page.advertisedUrls.map((value) => new URL(value).pathname)));
|
|
6505
|
+
return {
|
|
6506
|
+
url,
|
|
6507
|
+
status: result.response.status,
|
|
6508
|
+
valid: Array.isArray(parsed.pages),
|
|
6509
|
+
declaredHtmlRoutes,
|
|
6510
|
+
declaredMarkdownRoutes,
|
|
6511
|
+
missingHtmlRoutes: declaredHtmlRoutes.filter((path) => !crawledHtmlRoutes.has(path.replace(/\/$/u, "") || "/")).sort(),
|
|
6512
|
+
missingMarkdownRoutes: declaredMarkdownRoutes.filter((path) => !observedMarkdownRoutes.has(path)).sort(),
|
|
6513
|
+
orphanMarkdownRoutes: [...observedMarkdownRoutes].filter((path) => !declaredMarkdownRoutes.includes(path)).sort()
|
|
6514
|
+
};
|
|
6515
|
+
} catch (error) {
|
|
6516
|
+
return {
|
|
6517
|
+
url,
|
|
6518
|
+
valid: false,
|
|
6519
|
+
declaredHtmlRoutes: [],
|
|
6520
|
+
declaredMarkdownRoutes: [],
|
|
6521
|
+
missingHtmlRoutes: [],
|
|
6522
|
+
missingMarkdownRoutes: [],
|
|
6523
|
+
orphanMarkdownRoutes: [],
|
|
6524
|
+
error: safeError(error)
|
|
6525
|
+
};
|
|
6526
|
+
}
|
|
6527
|
+
}
|
|
6528
|
+
async function inspectProtocolVariant(input) {
|
|
6529
|
+
try {
|
|
6530
|
+
const result = await fetchText({
|
|
6531
|
+
url: input.url,
|
|
6532
|
+
timeoutMs: input.timeoutMs,
|
|
6533
|
+
fetch: input.fetch,
|
|
6534
|
+
signal: input.signal,
|
|
6535
|
+
redirect: "manual"
|
|
6536
|
+
});
|
|
6537
|
+
return {
|
|
6538
|
+
status: result.response.status,
|
|
6539
|
+
location: result.response.headers.get("location") ?? void 0
|
|
6540
|
+
};
|
|
6541
|
+
} catch (error) {
|
|
6542
|
+
return { error: safeError(error) };
|
|
6543
|
+
}
|
|
6544
|
+
}
|
|
6545
|
+
async function collectAgentDiscovery(input) {
|
|
6546
|
+
const origin = new URL(input.startUrl).origin;
|
|
6547
|
+
const pages = input.pages.filter((page) => page.status >= 200 && page.status < 300 && /^\s*(?:text\/html|application\/xhtml\+xml)\b/iu.test(page.contentType ?? "")).sort((left, right) => left.finalUrl.localeCompare(right.finalUrl));
|
|
6548
|
+
const queue = new PQueue({ concurrency: input.concurrency ?? 4 });
|
|
6549
|
+
const observations = await Promise.all(pages.map((page) => queue.add(() => inspectMarkdownPage({
|
|
6550
|
+
page,
|
|
6551
|
+
timeoutMs: input.timeoutMs,
|
|
6552
|
+
fetch: input.fetch,
|
|
6553
|
+
signal: input.signal
|
|
6554
|
+
}))));
|
|
6555
|
+
const [agentSkills, llmsTxt, routeManifest] = await Promise.all([
|
|
6556
|
+
inspectAgentSkills({
|
|
6557
|
+
...input,
|
|
6558
|
+
origin
|
|
6559
|
+
}),
|
|
6560
|
+
inspectLlmsTxt({
|
|
6561
|
+
...input,
|
|
6562
|
+
origin,
|
|
6563
|
+
pages
|
|
6564
|
+
}),
|
|
6565
|
+
inspectRouteManifest({
|
|
6566
|
+
...input,
|
|
6567
|
+
origin,
|
|
6568
|
+
pages,
|
|
6569
|
+
observations
|
|
6570
|
+
})
|
|
6571
|
+
]);
|
|
6572
|
+
const qZero = await fetchRepresentation({
|
|
6573
|
+
url: input.startUrl,
|
|
6574
|
+
accept: "text/markdown;q=0,text/html;q=1",
|
|
6575
|
+
timeoutMs: input.timeoutMs,
|
|
6576
|
+
fetch: input.fetch,
|
|
6577
|
+
signal: input.signal
|
|
6578
|
+
});
|
|
6579
|
+
const start = new URL(input.startUrl);
|
|
6580
|
+
const httpUrl = new URL(start);
|
|
6581
|
+
httpUrl.protocol = "http:";
|
|
6582
|
+
const wwwUrl = new URL(start);
|
|
6583
|
+
wwwUrl.hostname = start.hostname.startsWith("www.") ? start.hostname.slice(4) : `www.${start.hostname}`;
|
|
6584
|
+
const [http, www] = await Promise.all([inspectProtocolVariant({
|
|
6585
|
+
...input,
|
|
6586
|
+
url: httpUrl.toString()
|
|
6587
|
+
}), inspectProtocolVariant({
|
|
6588
|
+
...input,
|
|
6589
|
+
url: wwwUrl.toString()
|
|
6590
|
+
})]);
|
|
6591
|
+
const httpLocation = http.location ? new URL(http.location, httpUrl).toString() : void 0;
|
|
6592
|
+
const wwwLocation = www.location ? new URL(www.location, wwwUrl).toString() : void 0;
|
|
6593
|
+
const qZeroContentType = qZero.observation.contentType;
|
|
6594
|
+
const advertisedPages = observations.filter((observation) => observation.htmlAlternateUnique).length;
|
|
6595
|
+
const evaluatedPages = observations.filter((observation) => observation.explicit?.status !== void 0).length;
|
|
6596
|
+
const warnings = [...pages.length !== input.pages.length ? [`Agent representations were evaluated for ${pages.length} successful HTML pages from ${input.pages.length} retained documents.`] : [], ...routeManifest.valid ? [] : ["No valid public agent route manifest was available for orphan checks."]].sort();
|
|
6597
|
+
const htmlContentSignals = pages.map((page) => headerValue$1(page.responseHeaders, "content-signal")).filter((value) => Boolean(value));
|
|
6598
|
+
const markdownContentSignals = observations.map((observation) => observation.explicit?.contentSignal).filter((value) => Boolean(value));
|
|
6599
|
+
const contentSignalValues = [.../* @__PURE__ */ new Set([...htmlContentSignals, ...markdownContentSignals])].sort();
|
|
6600
|
+
return {
|
|
6601
|
+
profile: "content",
|
|
6602
|
+
profileApplicability: {
|
|
6603
|
+
content: {
|
|
6604
|
+
status: "evaluated",
|
|
6605
|
+
reason: "This site publishes documents, so access, representations, discovery, identity, and extraction quality were evaluated."
|
|
6606
|
+
},
|
|
6607
|
+
api: {
|
|
6608
|
+
status: "notApplicable",
|
|
6609
|
+
reason: "No public API surface was selected for this content-site run."
|
|
6610
|
+
},
|
|
6611
|
+
application: {
|
|
6612
|
+
status: "notApplicable",
|
|
6613
|
+
reason: "A local MCP product does not make this website a public remote agent endpoint."
|
|
6614
|
+
},
|
|
6615
|
+
commerce: {
|
|
6616
|
+
status: "notApplicable",
|
|
6617
|
+
reason: "No product checkout or transactional surface was selected."
|
|
6618
|
+
}
|
|
6619
|
+
},
|
|
6620
|
+
dataStatus: pages.length === 0 ? "unavailable" : evaluatedPages === pages.length ? "complete" : "partial",
|
|
6621
|
+
markdownAlternates: {
|
|
6622
|
+
eligibleHtmlPages: pages.length,
|
|
6623
|
+
advertisedPages,
|
|
6624
|
+
evaluatedPages,
|
|
6625
|
+
exactByteMatches: observations.filter((observation) => observation.explicitMatchesNegotiated).length,
|
|
6626
|
+
stableResponses: observations.filter((observation) => observation.repeatedHashStable).length,
|
|
6627
|
+
pages: observations
|
|
6628
|
+
},
|
|
6629
|
+
contentNegotiation: {
|
|
6630
|
+
qZeroHonoured: qZeroContentType !== void 0 ? !/^\s*text\/markdown\b/iu.test(qZeroContentType) : null,
|
|
6631
|
+
qZeroStatus: qZero.observation.status,
|
|
6632
|
+
qZeroContentType,
|
|
6633
|
+
error: qZero.observation.error
|
|
6634
|
+
},
|
|
6635
|
+
routeManifest,
|
|
6636
|
+
agentSkills,
|
|
6637
|
+
llmsTxt,
|
|
6638
|
+
contentSignals: {
|
|
6639
|
+
htmlValues: [...new Set(htmlContentSignals)].sort(),
|
|
6640
|
+
markdownValues: [...new Set(markdownContentSignals)].sort(),
|
|
6641
|
+
missingHtmlPages: pages.length - htmlContentSignals.length,
|
|
6642
|
+
missingMarkdownPages: observations.length - markdownContentSignals.length,
|
|
6643
|
+
consistent: pages.length > 0 && observations.length > 0 ? contentSignalValues.length === 1 && htmlContentSignals.length === pages.length && markdownContentSignals.length === observations.length : null
|
|
6644
|
+
},
|
|
6645
|
+
protocolVariants: {
|
|
6646
|
+
http: {
|
|
6647
|
+
url: httpUrl.toString(),
|
|
6648
|
+
status: http.status,
|
|
6649
|
+
location: httpLocation,
|
|
6650
|
+
permanentRedirectToHttps: http.status !== void 0 ? [301, 308].includes(http.status) && Boolean(httpLocation?.startsWith("https://")) : null,
|
|
6651
|
+
error: http.error
|
|
6652
|
+
},
|
|
6653
|
+
www: {
|
|
6654
|
+
url: wwwUrl.toString(),
|
|
6655
|
+
status: www.status,
|
|
6656
|
+
location: wwwLocation,
|
|
6657
|
+
redirectsToPreferredHost: www.status !== void 0 ? [301, 308].includes(www.status) && Boolean(wwwLocation && new URL(wwwLocation).hostname === start.hostname && new URL(wwwLocation).protocol === "https:") : null,
|
|
6658
|
+
error: www.error
|
|
6659
|
+
},
|
|
6660
|
+
hstsOnStartPage: pages.find((page) => normalizedDocumentUrl(page.finalUrl) === normalizedDocumentUrl(input.startUrl))?.hasHsts ?? null
|
|
6661
|
+
},
|
|
6662
|
+
warnings
|
|
6663
|
+
};
|
|
6664
|
+
}
|
|
6665
|
+
//#endregion
|
|
6666
|
+
//#region packages/core/dist/analyze/crawler/agent-readiness.js
|
|
6667
|
+
function check$1(section, input) {
|
|
6668
|
+
return {
|
|
6669
|
+
section,
|
|
6670
|
+
...input
|
|
6671
|
+
};
|
|
6672
|
+
}
|
|
6673
|
+
function section$1(id, title, checks) {
|
|
6674
|
+
return {
|
|
6675
|
+
id,
|
|
6676
|
+
title,
|
|
6677
|
+
checks
|
|
6678
|
+
};
|
|
6679
|
+
}
|
|
6680
|
+
function failedMarkdownUrls(discovery) {
|
|
6681
|
+
return discovery.markdownAlternates.pages.filter((page) => !page.htmlAlternateUnique || page.httpAlternateUrls.length !== 1 || page.explicit?.status !== 200 || !/^\s*text\/markdown\b/iu.test(page.explicit.contentType ?? "") || page.explicitMatchesNegotiated !== true || page.repeatedHashStable !== true || page.markdownCanonicalMatchesHtml !== true || page.explicit?.varyAccept !== true || page.negotiated?.varyAccept !== true).map((page) => page.htmlUrl);
|
|
6682
|
+
}
|
|
6683
|
+
function qualityFailureUrls(discovery) {
|
|
6684
|
+
return discovery.markdownAlternates.pages.filter((page) => {
|
|
6685
|
+
const quality = page.quality;
|
|
6686
|
+
if (!quality) return true;
|
|
6687
|
+
return !quality.frontmatterTitle || quality.h1Count !== 1 || !quality.codeFenceBalanced || quality.rawHtmlTags > 0 || quality.rawSvgTags > 0 || quality.rawScriptTags > 0 || quality.rawStyleTags > 0 || quality.navigationOnly || quality.repeatedLines > 0 || quality.wordRetentionRatio !== null && quality.wordRetentionRatio < .4;
|
|
6688
|
+
}).map((page) => page.htmlUrl);
|
|
6689
|
+
}
|
|
6690
|
+
function profileChecks(discovery) {
|
|
6691
|
+
return [
|
|
6692
|
+
"api",
|
|
6693
|
+
"application",
|
|
6694
|
+
"commerce"
|
|
6695
|
+
].map((profile) => check$1("profile", {
|
|
6696
|
+
id: `profile-${profile}`,
|
|
6697
|
+
status: "notApplicable",
|
|
6698
|
+
title: `${profile[0]?.toUpperCase()}${profile.slice(1)} checks are not applicable`,
|
|
6699
|
+
plainEnglish: discovery.profileApplicability[profile].reason,
|
|
6700
|
+
action: "Choose this profile only when the site exposes that capability publicly.",
|
|
6701
|
+
evidence: {
|
|
6702
|
+
profile,
|
|
6703
|
+
selectedProfile: discovery.profile
|
|
6704
|
+
}
|
|
6705
|
+
}));
|
|
6706
|
+
}
|
|
6707
|
+
function summariseChecks(checks) {
|
|
6708
|
+
const count = (status) => checks.filter((item) => item.status === status).length;
|
|
6709
|
+
return {
|
|
6710
|
+
checks: checks.length,
|
|
6711
|
+
passed: count("pass"),
|
|
6712
|
+
warnings: count("warning"),
|
|
6713
|
+
failed: count("fail"),
|
|
6714
|
+
unknown: count("unknown"),
|
|
6715
|
+
information: count("info"),
|
|
6716
|
+
notApplicable: count("notApplicable")
|
|
6717
|
+
};
|
|
6718
|
+
}
|
|
6719
|
+
function markdownChecks(discovery) {
|
|
6720
|
+
const markdown = discovery.markdownAlternates;
|
|
6721
|
+
const failedUrls = failedMarkdownUrls(discovery);
|
|
6722
|
+
const qualityUrls = qualityFailureUrls(discovery);
|
|
6723
|
+
const coverageComplete = markdown.eligibleHtmlPages > 0 && markdown.advertisedPages === markdown.eligibleHtmlPages && markdown.evaluatedPages === markdown.eligibleHtmlPages;
|
|
6724
|
+
const bytesMatch = markdown.eligibleHtmlPages > 0 && markdown.exactByteMatches === markdown.eligibleHtmlPages;
|
|
6725
|
+
const stable = markdown.eligibleHtmlPages > 0 && markdown.stableResponses === markdown.eligibleHtmlPages;
|
|
6726
|
+
const tokenHeaders = markdown.pages.filter((page) => page.explicit?.bytes !== void 0 && page.explicit?.markdownTokens !== void 0 && page.explicit.markdownTokens === Math.ceil(page.explicit.bytes / 4) && page.explicit.markdownTokens === page.negotiated?.markdownTokens).length;
|
|
6727
|
+
return [
|
|
6728
|
+
check$1("representations", {
|
|
6729
|
+
id: "markdown-coverage",
|
|
6730
|
+
status: markdown.eligibleHtmlPages === 0 ? "unknown" : coverageComplete ? "pass" : "fail",
|
|
6731
|
+
title: coverageComplete ? "Every successful HTML page advertises one Markdown alternative" : "Markdown alternative coverage is incomplete",
|
|
6732
|
+
plainEnglish: `${markdown.advertisedPages} of ${markdown.eligibleHtmlPages} successful HTML pages advertised one Markdown alternative, and ${markdown.evaluatedPages} were fetched.`,
|
|
6733
|
+
action: coverageComplete ? "Keep route generation tied to the same deterministic page manifest." : "Add one text/markdown alternate for each public HTML page and make every advertised URL return successfully.",
|
|
6734
|
+
evidence: {
|
|
6735
|
+
eligibleHtmlPages: markdown.eligibleHtmlPages,
|
|
6736
|
+
advertisedPages: markdown.advertisedPages,
|
|
6737
|
+
evaluatedPages: markdown.evaluatedPages
|
|
6738
|
+
},
|
|
6739
|
+
urls: failedUrls.slice(0, 25)
|
|
6740
|
+
}),
|
|
6741
|
+
check$1("representations", {
|
|
6742
|
+
id: "markdown-token-estimates",
|
|
6743
|
+
status: markdown.evaluatedPages === 0 ? "unknown" : tokenHeaders === markdown.evaluatedPages ? "pass" : "warning",
|
|
6744
|
+
title: tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes the same token estimate" : "Some Markdown responses are missing a stable token estimate",
|
|
6745
|
+
plainEnglish: `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned matching X-Markdown-Tokens values on the explicit and negotiated responses. The value is a quick size estimate, not an exact model-specific token count.`,
|
|
6746
|
+
action: "Generate the estimate from the stored Markdown bytes and return the same value for explicit and negotiated responses.",
|
|
6747
|
+
urls: markdown.pages.filter((page) => page.explicit?.markdownTokens === void 0 || page.explicit.markdownTokens !== page.negotiated?.markdownTokens).map((page) => page.htmlUrl).slice(0, 25)
|
|
6748
|
+
}),
|
|
6749
|
+
check$1("representations", {
|
|
6750
|
+
id: "markdown-negotiation",
|
|
6751
|
+
status: markdown.eligibleHtmlPages === 0 ? "unknown" : bytesMatch && discovery.contentNegotiation.qZeroHonoured ? "pass" : "fail",
|
|
6752
|
+
title: bytesMatch && discovery.contentNegotiation.qZeroHonoured ? "Explicit and negotiated Markdown use the same bytes" : "Content negotiation does not match the explicit Markdown contract",
|
|
6753
|
+
plainEnglish: `${markdown.exactByteMatches} of ${markdown.eligibleHtmlPages} pages returned identical explicit and negotiated Markdown. A request that refuses Markdown ${discovery.contentNegotiation.qZeroHonoured ? "received HTML" : "did not produce confirmed HTML evidence"}.`,
|
|
6754
|
+
action: "Use one stored Markdown artifact for both routes, honour Accept q-values, and send Vary: Accept on negotiated responses.",
|
|
6755
|
+
evidence: {
|
|
6756
|
+
exactByteMatches: markdown.exactByteMatches,
|
|
6757
|
+
qZero: discovery.contentNegotiation
|
|
6758
|
+
},
|
|
6759
|
+
urls: failedUrls.slice(0, 25)
|
|
6760
|
+
}),
|
|
6761
|
+
check$1("representations", {
|
|
6762
|
+
id: "markdown-determinism",
|
|
6763
|
+
status: markdown.eligibleHtmlPages === 0 ? "unknown" : stable ? "pass" : "fail",
|
|
6764
|
+
title: stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
|
|
6765
|
+
plainEnglish: `${markdown.stableResponses} of ${markdown.eligibleHtmlPages} repeated explicit responses kept the same SHA-256 digest.`,
|
|
6766
|
+
action: "Generate Markdown during the build and remove runtime rewriting, timestamps, or random output.",
|
|
6767
|
+
evidence: { stableResponses: markdown.stableResponses },
|
|
6768
|
+
urls: failedUrls.slice(0, 25)
|
|
6769
|
+
}),
|
|
6770
|
+
check$1("representations", {
|
|
6771
|
+
id: "markdown-quality",
|
|
6772
|
+
status: markdown.evaluatedPages === 0 ? "unknown" : qualityUrls.length === 0 ? "pass" : "warning",
|
|
6773
|
+
title: qualityUrls.length === 0 ? "Markdown keeps the useful document structure cleanly" : "Some Markdown alternatives need an extraction review",
|
|
6774
|
+
plainEnglish: qualityUrls.length === 0 ? "The evaluated alternatives retained one H1, frontmatter titles, balanced code fences, useful copy, and no leaked SVG, script, style, or layout tags." : `${qualityUrls.length} Markdown alternative${qualityUrls.length === 1 ? "" : "s"} lost important structure, looked navigation-only, or leaked presentation markup.`,
|
|
6775
|
+
action: "Open the affected Markdown directly and fix the shared converter or component exclusion rules rather than patching generated files.",
|
|
6776
|
+
urls: qualityUrls.slice(0, 25)
|
|
6777
|
+
})
|
|
6778
|
+
];
|
|
6779
|
+
}
|
|
6780
|
+
function discoveryChecks(discovery) {
|
|
6781
|
+
const skills = discovery.agentSkills;
|
|
6782
|
+
const skillsValid = skills.validIndex && skills.skills.length > 0 && skills.skills.every((skill) => skill.sameOrigin && skill.status === 200 && /^\s*text\/markdown\b/iu.test(skill.contentType ?? "") && Boolean(skill.cors) && skill.digestMatches === true && skill.frontmatterValid === true) && Boolean(skills.cors);
|
|
6783
|
+
const llms = discovery.llmsTxt;
|
|
6784
|
+
const llmsBroken = llms.links.filter((link) => !link.status || link.status < 200 || link.status >= 400);
|
|
6785
|
+
const llmsValid = llms.exists && /^\s*(?:text\/plain|text\/markdown)\b/iu.test(llms.contentType ?? "") && llms.headingCount > 0 && llms.links.length > 0 && !llms.linkLimitReached && !llms.oversized && llms.invalidLinks.length === 0 && llms.duplicateLinks.length === 0 && llms.redirectedLinks.length === 0 && llms.nonIndexableLinks.length === 0 && llms.missingCrawlRoutes.length === 0 && llmsBroken.length === 0 && llms.repeatedHashStable === true;
|
|
6786
|
+
return [
|
|
6787
|
+
check$1("discovery", {
|
|
6788
|
+
id: "agent-skills",
|
|
6789
|
+
status: skillsValid ? "pass" : skills.status === 404 ? "info" : "warning",
|
|
6790
|
+
title: skillsValid ? "Agent Skills discovery is valid and digest verified" : "Agent Skills discovery is absent or needs attention",
|
|
6791
|
+
plainEnglish: skillsValid ? `${skills.skills.length} published skill${skills.skills.length === 1 ? "" : "s"} matched the declared SHA-256 digest, valid SKILL.md frontmatter, Markdown content type, and cross-origin access policy.` : "Agent Skills is optional for a content site, but a published index should resolve same-origin Markdown skill files, allow intended cross-origin clients, and verify their declared bytes.",
|
|
6792
|
+
action: "If you publish Agent Skills, keep the discovery index and SHA-256 digests generated from the exact deployed skill bytes.",
|
|
6793
|
+
evidence: skills
|
|
6794
|
+
}),
|
|
6795
|
+
check$1("discovery", {
|
|
6796
|
+
id: "llms-txt",
|
|
6797
|
+
status: llms.exists ? llmsValid ? "pass" : "warning" : "info",
|
|
6798
|
+
title: llms.exists ? llmsValid ? "llms.txt is short, stable, and its links resolve" : "llms.txt exists but needs a content or link review" : "llms.txt is not published",
|
|
6799
|
+
plainEnglish: llms.exists ? `${llms.links.length} declared links were checked. ${llms.invalidLinks.length} were malformed, ${llms.duplicateLinks.length} were duplicated, ${llms.redirectedLinks.length} redirected, ${llms.nonIndexableLinks.length} reached non-indexable pages, ${llmsBroken.length} did not resolve, and ${llms.missingCrawlRoutes.length} were missing from the crawl inventory. ${llms.offSiteLinks.length} linked to other sites.` : "llms.txt is optional and its absence is not a search ranking problem.",
|
|
6800
|
+
action: llms.exists ? "Keep the file curated, deterministic, and limited to useful entry points whose links still resolve." : "Add it only when an intended consumer uses it. Do not treat it as a Google ranking requirement.",
|
|
6801
|
+
evidence: {
|
|
6802
|
+
status: llms.status,
|
|
6803
|
+
contentType: llms.contentType,
|
|
6804
|
+
bytes: llms.bytes,
|
|
6805
|
+
oversized: llms.oversized,
|
|
6806
|
+
totalParsedLinks: llms.totalParsedLinks,
|
|
6807
|
+
linkLimitReached: llms.linkLimitReached,
|
|
6808
|
+
links: llms.links.length,
|
|
6809
|
+
invalidLinks: llms.invalidLinks,
|
|
6810
|
+
duplicateLinks: llms.duplicateLinks,
|
|
6811
|
+
offSiteLinks: llms.offSiteLinks,
|
|
6812
|
+
redirectedLinks: llms.redirectedLinks,
|
|
6813
|
+
nonIndexableLinks: llms.nonIndexableLinks,
|
|
6814
|
+
missingCrawlRoutes: llms.missingCrawlRoutes,
|
|
6815
|
+
repeatedHashStable: llms.repeatedHashStable
|
|
6816
|
+
},
|
|
6817
|
+
urls: [
|
|
6818
|
+
...llmsBroken.map((link) => link.url),
|
|
6819
|
+
...llms.redirectedLinks,
|
|
6820
|
+
...llms.nonIndexableLinks,
|
|
6821
|
+
...llms.missingCrawlRoutes
|
|
6822
|
+
].slice(0, 25)
|
|
6823
|
+
}),
|
|
6824
|
+
check$1("discovery", {
|
|
6825
|
+
id: "content-signals",
|
|
6826
|
+
status: discovery.contentSignals.consistent === null ? "unknown" : discovery.contentSignals.consistent ? "pass" : "warning",
|
|
6827
|
+
title: discovery.contentSignals.consistent ? "HTML and Markdown publish one explicit Content Signals policy" : "Content Signals are missing or inconsistent across representations",
|
|
6828
|
+
plainEnglish: discovery.contentSignals.consistent ? `The same ${discovery.contentSignals.htmlValues[0]} policy was observed on every evaluated HTML and Markdown response.` : `${discovery.contentSignals.missingHtmlPages} HTML responses and ${discovery.contentSignals.missingMarkdownPages} Markdown responses were missing the header, or the observed values differed.`,
|
|
6829
|
+
action: "Choose the site policy deliberately and return the same Content-Signal value on HTML and Markdown responses.",
|
|
6830
|
+
evidence: discovery.contentSignals
|
|
6831
|
+
}),
|
|
6832
|
+
check$1("discovery", {
|
|
6833
|
+
id: "route-manifest",
|
|
6834
|
+
status: !discovery.routeManifest.valid ? "info" : discovery.routeManifest.missingHtmlRoutes.length === 0 && discovery.routeManifest.missingMarkdownRoutes.length === 0 && discovery.routeManifest.orphanMarkdownRoutes.length === 0 ? "pass" : "warning",
|
|
6835
|
+
title: discovery.routeManifest.valid ? "The public route manifest agrees with the crawl" : "No public route manifest was available",
|
|
6836
|
+
plainEnglish: discovery.routeManifest.valid ? `The manifest declared ${discovery.routeManifest.declaredHtmlRoutes.length} HTML routes and ${discovery.routeManifest.declaredMarkdownRoutes.length} Markdown routes. Missing or orphan routes are listed in the evidence.` : "A public route manifest is optional. Without one, this crawl cannot prove that no orphan Markdown files were deployed.",
|
|
6837
|
+
action: "Generate one deterministic route inventory from the build when exact HTML and Markdown parity matters.",
|
|
6838
|
+
evidence: discovery.routeManifest
|
|
6839
|
+
})
|
|
6840
|
+
];
|
|
6841
|
+
}
|
|
6842
|
+
function accessChecks(report, discovery) {
|
|
6843
|
+
const bots = report.ai?.robotsTxt?.botAccess ?? [];
|
|
6844
|
+
const blocked = bots.filter((bot) => bot.allowed === false);
|
|
6845
|
+
const robotsKnown = report.ai?.robotsTxt?.availability === "available" || report.ai?.robotsTxt?.availability === "absent";
|
|
6846
|
+
const protocol = discovery.protocolVariants;
|
|
6847
|
+
return [check$1("access", {
|
|
6848
|
+
id: "crawler-access",
|
|
6849
|
+
status: !robotsKnown ? "unknown" : blocked.length > 0 ? "warning" : "pass",
|
|
6850
|
+
title: blocked.length ? "robots.txt blocks selected crawler tokens" : "No selected crawler token is blocked at the start URL",
|
|
6851
|
+
plainEnglish: robotsKnown ? `${blocked.length} of ${bots.length} selected crawler tokens were blocked by the start URL robots.txt policy.` : "The robots.txt response was not stable enough to make a crawler-access claim.",
|
|
6852
|
+
action: "Keep blocks intentional. A crawler token being allowed here does not prove that the service will crawl, index, or cite the site.",
|
|
6853
|
+
evidence: {
|
|
6854
|
+
availability: report.ai?.robotsTxt?.availability,
|
|
6855
|
+
blocked,
|
|
6856
|
+
botAccess: bots
|
|
6857
|
+
}
|
|
6858
|
+
}), check$1("access", {
|
|
6859
|
+
id: "protocol-canonicalization",
|
|
6860
|
+
status: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "pass" : protocol.http.permanentRedirectToHttps === null ? "unknown" : "warning",
|
|
6861
|
+
title: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "HTTP redirects permanently to HTTPS and HTTPS sends HSTS" : "Protocol canonicalization needs attention or could not be proven",
|
|
6862
|
+
plainEnglish: `The HTTP variant ${protocol.http.permanentRedirectToHttps === true ? "redirected permanently to HTTPS" : "did not produce a confirmed permanent HTTPS redirect"}. The HTTPS start page ${protocol.hstsOnStartPage ? "sent HSTS" : "did not provide confirmed HSTS evidence"}.`,
|
|
6863
|
+
action: "Use one permanent HTTP-to-HTTPS redirect and send HSTS only after HTTPS is stable across the site.",
|
|
6864
|
+
evidence: protocol
|
|
6865
|
+
})];
|
|
6866
|
+
}
|
|
6867
|
+
function identityChecks(report) {
|
|
6868
|
+
const types = new Set(report.pages.flatMap((page) => page.schemaTypes ?? []));
|
|
6869
|
+
const hasSoftware = types.has("SoftwareApplication");
|
|
6870
|
+
const hasWebsite = types.has("WebSite");
|
|
6871
|
+
const hasCreator = types.has("Person") || types.has("Organization");
|
|
6872
|
+
const pageTypes = [
|
|
6873
|
+
"WebPage",
|
|
6874
|
+
"TechArticle",
|
|
6875
|
+
"CollectionPage"
|
|
6876
|
+
].filter((type) => types.has(type));
|
|
6877
|
+
const officialProfiles = [...new Set(report.pages.flatMap((page) => [...page.schemaSameAs ?? [], ...page.socialProfileLinks ?? []]))];
|
|
6878
|
+
return [check$1("identity", {
|
|
6879
|
+
id: "identity-graph",
|
|
6880
|
+
status: hasSoftware && hasWebsite && hasCreator && pageTypes.length > 0 ? "pass" : "warning",
|
|
6881
|
+
title: hasSoftware && hasWebsite && hasCreator && pageTypes.length > 0 ? "Structured identity covers the software, website, creator, and pages" : "The structured identity graph is missing a useful entity layer",
|
|
6882
|
+
plainEnglish: `Software identity: ${hasSoftware ? "present" : "missing"}. Website identity: ${hasWebsite ? "present" : "missing"}. Creator or publisher identity: ${hasCreator ? "present" : "missing"}. Page-level types: ${pageTypes.length ? pageTypes.join(", ") : "missing"}.`,
|
|
6883
|
+
action: "Connect truthful SoftwareApplication, WebSite, creator or publisher, and page nodes. Use sameAs only for official profiles that identify the same entity.",
|
|
6884
|
+
evidence: {
|
|
6885
|
+
schemaTypes: [...types].sort(),
|
|
6886
|
+
officialProfiles: officialProfiles.sort()
|
|
6887
|
+
}
|
|
6888
|
+
})];
|
|
6889
|
+
}
|
|
6890
|
+
function agentReadiness(report) {
|
|
6891
|
+
const discovery = report.agentDiscovery;
|
|
6892
|
+
if (!discovery) {
|
|
6893
|
+
const profiles = {
|
|
6894
|
+
content: {
|
|
6895
|
+
status: "evaluated",
|
|
6896
|
+
reason: "The content profile was selected but discovery evidence was not collected."
|
|
6897
|
+
},
|
|
6898
|
+
api: {
|
|
6899
|
+
status: "notApplicable",
|
|
6900
|
+
reason: "Not selected."
|
|
6901
|
+
},
|
|
6902
|
+
application: {
|
|
6903
|
+
status: "notApplicable",
|
|
6904
|
+
reason: "Not selected."
|
|
6905
|
+
},
|
|
6906
|
+
commerce: {
|
|
6907
|
+
status: "notApplicable",
|
|
6908
|
+
reason: "Not selected."
|
|
6909
|
+
}
|
|
6910
|
+
};
|
|
6911
|
+
const missing = check$1("collection", {
|
|
6912
|
+
id: "agent-discovery-evidence",
|
|
6913
|
+
status: "unknown",
|
|
6914
|
+
title: "Agent discovery evidence was not collected",
|
|
6915
|
+
plainEnglish: "This saved crawl predates the focused content-profile checks or was run without them.",
|
|
6916
|
+
action: "Run the agent-readiness report with a URL or start a fresh crawl with agent discovery enabled."
|
|
6917
|
+
});
|
|
6918
|
+
const scopeChecks = profileChecks({
|
|
6919
|
+
profile: "content",
|
|
6920
|
+
profileApplicability: profiles
|
|
6921
|
+
});
|
|
6922
|
+
const checks = [missing, ...scopeChecks];
|
|
6923
|
+
return {
|
|
6924
|
+
reportId: report.id,
|
|
6925
|
+
url: report.config.url,
|
|
6926
|
+
generatedAt: report.generatedAt,
|
|
6927
|
+
profile: "content",
|
|
6928
|
+
dataStatus: "unavailable",
|
|
6929
|
+
assessment: "evidence-only",
|
|
6930
|
+
headline: "No agent-discovery evidence is stored in this crawl. Run a fresh focused check.",
|
|
6931
|
+
profileApplicability: profiles,
|
|
6932
|
+
summary: summariseChecks(checks),
|
|
6933
|
+
sections: [section$1("collection", "Evidence collection", [missing]), section$1("profile", "Profile scope", scopeChecks)],
|
|
6934
|
+
checks,
|
|
6935
|
+
topActions: [missing],
|
|
6936
|
+
caveats: ["Not evaluated does not mean absent or broken.", "This report does not measure rankings, AI mentions, citations, or selection."]
|
|
6937
|
+
};
|
|
6938
|
+
}
|
|
6939
|
+
const sections = [
|
|
6940
|
+
section$1("profile", "Profile scope", profileChecks(discovery)),
|
|
6941
|
+
section$1("access", "Crawler and protocol access", accessChecks(report, discovery)),
|
|
6942
|
+
section$1("representations", "HTML and Markdown representations", markdownChecks(discovery)),
|
|
6943
|
+
section$1("discovery", "Agent discovery files", discoveryChecks(discovery)),
|
|
6944
|
+
section$1("identity", "Identity evidence", identityChecks(report))
|
|
6945
|
+
];
|
|
6946
|
+
const checks = sections.flatMap((item) => item.checks);
|
|
6947
|
+
const actionable = checks.filter((item) => [
|
|
6948
|
+
"fail",
|
|
6949
|
+
"warning",
|
|
6950
|
+
"unknown"
|
|
6951
|
+
].includes(item.status));
|
|
6952
|
+
const rank = {
|
|
6953
|
+
fail: 0,
|
|
6954
|
+
warning: 1,
|
|
6955
|
+
unknown: 2,
|
|
6956
|
+
pass: 3,
|
|
6957
|
+
info: 4,
|
|
6958
|
+
notApplicable: 5
|
|
6959
|
+
};
|
|
6960
|
+
const topActions = [...actionable].sort((left, right) => rank[left.status] - rank[right.status]).slice(0, 10);
|
|
6961
|
+
const summary = summariseChecks(checks);
|
|
6962
|
+
return {
|
|
6963
|
+
reportId: report.id,
|
|
6964
|
+
url: report.config.url,
|
|
6965
|
+
generatedAt: report.generatedAt,
|
|
6966
|
+
profile: discovery.profile,
|
|
6967
|
+
dataStatus: discovery.dataStatus,
|
|
6968
|
+
assessment: "evidence-only",
|
|
6969
|
+
headline: `${summary.passed} checks passed, ${summary.warnings} need review, and ${summary.failed} failed for the content profile.`,
|
|
6970
|
+
profileApplicability: discovery.profileApplicability,
|
|
6971
|
+
summary,
|
|
6972
|
+
sections,
|
|
6973
|
+
checks,
|
|
6974
|
+
topActions,
|
|
6975
|
+
caveats: [
|
|
6976
|
+
...report.caveats,
|
|
6977
|
+
...discovery.warnings,
|
|
6978
|
+
"A clean content profile does not prove indexing, rankings, AI mentions, citations, or selection.",
|
|
6979
|
+
"A crawler token being allowed does not prove that a service fetched or used the page.",
|
|
6980
|
+
"API, application, and commerce checks were not applicable to this content-site run."
|
|
6981
|
+
]
|
|
6982
|
+
};
|
|
6983
|
+
}
|
|
6984
|
+
//#endregion
|
|
6985
|
+
//#region packages/core/dist/analyze/crawler/entity-readiness.js
|
|
6986
|
+
const ENTITY_EVIDENCE_SCHEMA_TYPES = /* @__PURE__ */ new Set([
|
|
6987
|
+
"organization",
|
|
6988
|
+
"localbusiness",
|
|
6989
|
+
"person",
|
|
6990
|
+
"product",
|
|
6991
|
+
"softwareapplication",
|
|
6992
|
+
"website"
|
|
6993
|
+
]);
|
|
6994
|
+
const SITE_IDENTITY_SCHEMA_TYPES = /* @__PURE__ */ new Set(["organization", "localbusiness"]);
|
|
6995
|
+
const PROFILE_IDENTITY_SCHEMA_TYPES = /* @__PURE__ */ new Set([...SITE_IDENTITY_SCHEMA_TYPES, "person"]);
|
|
6996
|
+
function isEntityEvidenceSchemaType(type) {
|
|
6997
|
+
return ENTITY_EVIDENCE_SCHEMA_TYPES.has(type.toLowerCase());
|
|
6998
|
+
}
|
|
6999
|
+
function isSiteIdentitySchemaType(type) {
|
|
7000
|
+
return SITE_IDENTITY_SCHEMA_TYPES.has(type.toLowerCase());
|
|
7001
|
+
}
|
|
7002
|
+
function isProfileIdentitySchemaType(type) {
|
|
7003
|
+
return PROFILE_IDENTITY_SCHEMA_TYPES.has(type.toLowerCase());
|
|
7004
|
+
}
|
|
7005
|
+
function unique$3(values) {
|
|
7006
|
+
return [...new Set(values.filter((value) => Boolean(value)))];
|
|
7007
|
+
}
|
|
6034
7008
|
function pct$3(part, total) {
|
|
7009
|
+
return total ? Math.round(part / total * 100) : 0;
|
|
7010
|
+
}
|
|
7011
|
+
function entityReadiness(report) {
|
|
7012
|
+
const pages = report.pages.filter((page) => page.indexable && page.status < 400);
|
|
7013
|
+
const total = pages.length || 1;
|
|
7014
|
+
const schemaTypes = {};
|
|
7015
|
+
for (const page of pages) for (const type of page.schemaTypes ?? []) schemaTypes[type] = (schemaTypes[type] ?? 0) + 1;
|
|
7016
|
+
const entitySchemaPages = pages.filter((page) => page.schemaTypes?.some(isEntityEvidenceSchemaType));
|
|
7017
|
+
const sameAsEvidence = pages.flatMap((page) => page.schemaSameAsEvidence ?? []).filter((evidence) => evidence.subjectTypes.some(isProfileIdentitySchemaType));
|
|
7018
|
+
const siteSameAsEvidence = sameAsEvidence.filter((evidence) => evidence.subjectTypes.some(isSiteIdentitySchemaType));
|
|
7019
|
+
const sameAs = unique$3(sameAsEvidence.map((evidence) => evidence.url));
|
|
7020
|
+
const sameAsByType = {};
|
|
7021
|
+
for (const evidence of sameAsEvidence) for (const type of evidence.subjectTypes) {
|
|
7022
|
+
if (!isProfileIdentitySchemaType(type)) continue;
|
|
7023
|
+
sameAsByType[type] = unique$3([...sameAsByType[type] ?? [], evidence.url]);
|
|
7024
|
+
}
|
|
7025
|
+
const socialProfiles = unique$3(pages.flatMap((page) => page.socialProfileLinks ?? []));
|
|
7026
|
+
const authors = unique$3(pages.map((page) => page.author));
|
|
7027
|
+
const authorPages = pages.filter((page) => page.author || page.geo?.hasAuthor);
|
|
7028
|
+
const datePages = pages.filter((page) => page.hasDate || page.geo?.hasDate);
|
|
7029
|
+
const titlePages = pages.filter((page) => page.title && page.h1);
|
|
7030
|
+
const checks = [
|
|
7031
|
+
{
|
|
7032
|
+
coverage: pct$3(entitySchemaPages.length, total),
|
|
7033
|
+
check: {
|
|
7034
|
+
id: "entity-schema",
|
|
7035
|
+
title: "Entity schema is present",
|
|
7036
|
+
plainEnglish: `${pct$3(entitySchemaPages.length, total)}% of evaluated indexable pages include Organization, LocalBusiness, Person, Product, SoftwareApplication, or WebSite schema.`,
|
|
7037
|
+
action: "Add accurate schema only where it describes the software, website, organization, person, product, or local business shown on the page.",
|
|
7038
|
+
evidence: { schemaTypes },
|
|
7039
|
+
urls: pages.filter((page) => !entitySchemaPages.includes(page)).slice(0, 10).map((page) => page.finalUrl)
|
|
7040
|
+
}
|
|
7041
|
+
},
|
|
7042
|
+
{
|
|
7043
|
+
coverage: siteSameAsEvidence.length ? 100 : 0,
|
|
7044
|
+
check: {
|
|
7045
|
+
id: "same-as",
|
|
7046
|
+
title: "Site entity profiles are connected",
|
|
7047
|
+
plainEnglish: siteSameAsEvidence.length ? "The crawl found sameAs links attached to Organization or LocalBusiness structured data." : sameAs.length ? "The crawl found Person sameAs links, but no Organization or LocalBusiness profile evidence for the site entity." : socialProfiles.length ? "The crawl found social-domain links, but they are not enough to prove official profile ownership." : "The crawl did not find entity-scoped sameAs evidence.",
|
|
7048
|
+
action: "Connect official profiles with sameAs on Organization, Person, or LocalBusiness schema. Use only profiles you control.",
|
|
7049
|
+
evidence: {
|
|
7050
|
+
sameAs,
|
|
7051
|
+
siteSameAs: unique$3(siteSameAsEvidence.map((evidence) => evidence.url)),
|
|
7052
|
+
sameAsEvidence,
|
|
7053
|
+
unclassifiedSocialLinks: socialProfiles
|
|
7054
|
+
}
|
|
7055
|
+
}
|
|
7056
|
+
},
|
|
7057
|
+
{
|
|
7058
|
+
coverage: Math.round((pct$3(authorPages.length, total) + pct$3(datePages.length, total)) / 2),
|
|
7059
|
+
check: {
|
|
7060
|
+
id: "authority-freshness",
|
|
7061
|
+
title: "Author and date signals observed",
|
|
7062
|
+
plainEnglish: `${pct$3(authorPages.length, total)}% of evaluated indexable pages expose an author signal and ${pct$3(datePages.length, total)}% expose a date signal.`,
|
|
7063
|
+
action: "For editorial content, identify authors and show accurate published or modified dates when that information is useful to readers.",
|
|
7064
|
+
evidence: {
|
|
7065
|
+
authorCoverage: pct$3(authorPages.length, total),
|
|
7066
|
+
dateCoverage: pct$3(datePages.length, total),
|
|
7067
|
+
authors
|
|
7068
|
+
}
|
|
7069
|
+
}
|
|
7070
|
+
},
|
|
7071
|
+
{
|
|
7072
|
+
coverage: pct$3(titlePages.length, total),
|
|
7073
|
+
check: {
|
|
7074
|
+
id: "entity-naming",
|
|
7075
|
+
title: "Title and H1 coverage observed",
|
|
7076
|
+
plainEnglish: `${pct$3(titlePages.length, total)}% of evaluated indexable pages have both a title and H1.`,
|
|
7077
|
+
action: "Review pages missing either signal. Keep brand, product, and person names accurate wherever they appear."
|
|
7078
|
+
}
|
|
7079
|
+
}
|
|
7080
|
+
].map(({ coverage, check }) => ({
|
|
7081
|
+
...check,
|
|
7082
|
+
status: "info",
|
|
7083
|
+
evidence: {
|
|
7084
|
+
...check.evidence,
|
|
7085
|
+
observedCoveragePercent: coverage
|
|
7086
|
+
}
|
|
7087
|
+
}));
|
|
7088
|
+
const dataStatus = report.status === "completed" && pages.length > 0 ? "complete" : "partial";
|
|
7089
|
+
return {
|
|
7090
|
+
reportId: report.id,
|
|
7091
|
+
url: report.config.url,
|
|
7092
|
+
generatedAt: report.generatedAt,
|
|
7093
|
+
dataStatus,
|
|
7094
|
+
evaluatedPages: pages.length,
|
|
7095
|
+
crawlPages: report.pages.length,
|
|
7096
|
+
assessment: "evidence-only",
|
|
7097
|
+
headline: dataStatus === "partial" ? "Entity evidence is incomplete; treat these findings as scoped to the evaluated pages, not the whole site." : "Entity signals are reported as observations, not a ranking, visibility, or machine-understanding score.",
|
|
7098
|
+
caveats: [
|
|
7099
|
+
...report.caveats,
|
|
7100
|
+
"This report deliberately has no aggregate entity score. Add schema, authors, dates, and sameAs only where they accurately fit the page and a documented use case.",
|
|
7101
|
+
...dataStatus === "partial" ? [`Entity coverage is based on ${pages.length} evaluated indexable page${pages.length === 1 ? "" : "s"} from a ${report.status} crawl.`] : [],
|
|
7102
|
+
...socialProfiles.length && !sameAs.length ? ["Social-domain links are unclassified navigation evidence, not proof that the linked profiles are official."] : []
|
|
7103
|
+
],
|
|
7104
|
+
checks,
|
|
7105
|
+
entities: {
|
|
7106
|
+
schemaTypes,
|
|
7107
|
+
sameAs,
|
|
7108
|
+
sameAsByType,
|
|
7109
|
+
socialProfiles,
|
|
7110
|
+
authors
|
|
7111
|
+
}
|
|
7112
|
+
};
|
|
7113
|
+
}
|
|
7114
|
+
//#endregion
|
|
7115
|
+
//#region packages/core/dist/analyze/crawler/ai-readiness.js
|
|
7116
|
+
function pct$2(part, total) {
|
|
6035
7117
|
if (total <= 0) return 0;
|
|
6036
7118
|
return Math.round(part / total * 100);
|
|
6037
7119
|
}
|
|
@@ -6068,6 +7150,7 @@ function sampleUrls$1(pages, predicate) {
|
|
|
6068
7150
|
return pages.filter(predicate).slice(0, 10).map((page) => page.finalUrl);
|
|
6069
7151
|
}
|
|
6070
7152
|
function aiReadiness(report) {
|
|
7153
|
+
const agentDiscovery = report.agentDiscovery;
|
|
6071
7154
|
const pages = indexablePages$1(report);
|
|
6072
7155
|
const pageCount = pages.length;
|
|
6073
7156
|
const hasIndexablePages = pageCount > 0;
|
|
@@ -6091,7 +7174,7 @@ function aiReadiness(report) {
|
|
|
6091
7174
|
const titlePages = pages.filter((page) => page.title && page.h1);
|
|
6092
7175
|
const metaPages = pages.filter((page) => page.metaDescription);
|
|
6093
7176
|
const langPages = pages.filter((page) => page.lang);
|
|
6094
|
-
const entityPages = pages.filter((page) => page.schemaTypes?.some(
|
|
7177
|
+
const entityPages = pages.filter((page) => page.schemaTypes?.some(isEntityEvidenceSchemaType) || (page.schemaSameAs?.length ?? 0) > 0 || (page.socialProfileLinks?.length ?? 0) > 0);
|
|
6095
7178
|
const restrictedSnippets = pages.map((page) => ({
|
|
6096
7179
|
url: page.finalUrl,
|
|
6097
7180
|
control: effectiveSnippetControl({
|
|
@@ -6108,7 +7191,7 @@ function aiReadiness(report) {
|
|
|
6108
7191
|
section: "agent-access",
|
|
6109
7192
|
maxScore: 20,
|
|
6110
7193
|
evaluated: botAccessEvaluated,
|
|
6111
|
-
score: robotsUnavailable ? 0 : botAccess.length ? blockedBots.length ? scoreFromPercent(pct$
|
|
7194
|
+
score: robotsUnavailable ? 0 : botAccess.length ? blockedBots.length ? scoreFromPercent(pct$2(botAccess.length - blockedBots.length, botAccess.length), 20) : 20 : 8,
|
|
6112
7195
|
title: robotsUnavailable ? "robots.txt crawler-policy evidence is unavailable" : !hasBotAccessEvidence ? "robots.txt crawler-policy evidence was not collected" : blockedBots.length ? "robots.txt blocks selected crawler tokens at the start URL" : "robots.txt allows selected crawler tokens at the start URL",
|
|
6113
7196
|
plainEnglish: robotsUnavailable ? "The robots.txt request failed, so this report cannot say whether selected crawler tokens are allowed at the start URL." : !hasBotAccessEvidence ? "This crawl does not include per-bot robots.txt policy data for the start URL." : blockedBots.length ? `robots.txt policy blocks ${blockedBots.length} selected crawler ${blockedBots.length === 1 ? "token" : "tokens"} at the start URL.` : "robots.txt policy does not block the selected crawler tokens at the start URL. This does not verify actual or site-wide fetchability.",
|
|
6114
7197
|
action: robotsUnavailable ? "Restore a stable robots.txt response, then rerun this report before making crawler-policy claims." : !hasBotAccessEvidence ? "Rerun the crawl with top-level per-bot robots.txt evidence before making crawler-policy claims." : blockedBots.length ? "Only block AI crawlers intentionally. If discovery matters, remove accidental Disallow rules for the blocked user agents." : "Keep robots.txt explicit and intentional so humans and agents can see what is allowed.",
|
|
@@ -6151,10 +7234,11 @@ function aiReadiness(report) {
|
|
|
6151
7234
|
maxScore: 0,
|
|
6152
7235
|
score: 0,
|
|
6153
7236
|
title: "llms.txt is optional",
|
|
6154
|
-
plainEnglish: report.ai?.llmsTxt?.exists ? "An optional llms.txt file was found. Google says it has no positive or negative Search impact." : "No llms.txt file was found. Google says the file is not needed for Search and does not affect visibility.",
|
|
7237
|
+
plainEnglish: report.ai?.llmsTxt?.exists ? agentDiscovery?.llmsTxt ? `An optional llms.txt file was found and its body was checked. It declared ${agentDiscovery.llmsTxt.links.length} links, ${agentDiscovery.llmsTxt.duplicateLinks.length} duplicates, and ${agentDiscovery.llmsTxt.missingCrawlRoutes.length} links outside the crawled route inventory. Google says llms.txt has no positive or negative Search impact.` : "An optional llms.txt file was found. Its body was not validated in this crawl. Google says it has no positive or negative Search impact." : "No llms.txt file was found. Google says the file is not needed for Search and does not affect visibility.",
|
|
6155
7238
|
action: "No SEO action is required. Generate one only for a specific agent or service that consumes it.",
|
|
6156
7239
|
evidence: {
|
|
6157
7240
|
llmsTxt: report.ai?.llmsTxt,
|
|
7241
|
+
validation: agentDiscovery?.llmsTxt,
|
|
6158
7242
|
googleSearchImpact: "none",
|
|
6159
7243
|
guidanceUrl: "https://developers.google.com/search/updates#clarifying-guidance-on-llms-txt-files"
|
|
6160
7244
|
}
|
|
@@ -6177,7 +7261,7 @@ function aiReadiness(report) {
|
|
|
6177
7261
|
evaluated: hasIndexablePages,
|
|
6178
7262
|
score: 0,
|
|
6179
7263
|
title: "Short opening paragraphs are an unscored observation",
|
|
6180
|
-
plainEnglish: hasIndexablePages ? `${pct$
|
|
7264
|
+
plainEnglish: hasIndexablePages ? `${pct$2(answerablePages.length, pageCount)}% of indexable pages have at least one 25-word paragraph near the start. This does not establish content quality, citation likelihood, or Google AI eligibility.` : "No indexable 2xx pages were available, so opening-paragraph coverage was not evaluated.",
|
|
6181
7265
|
action: "Write for the reader and the page intent. Do not create artificial answer blocks or chunk content solely for AI Search.",
|
|
6182
7266
|
urls: sampleUrls$1(pages, (page) => !page.geo?.answerable),
|
|
6183
7267
|
evidence: {
|
|
@@ -6195,7 +7279,7 @@ function aiReadiness(report) {
|
|
|
6195
7279
|
evaluated: hasIndexablePages,
|
|
6196
7280
|
score: 0,
|
|
6197
7281
|
title: "Semantic HTML coverage observed",
|
|
6198
|
-
plainEnglish: hasIndexablePages ? `${pct$
|
|
7282
|
+
plainEnglish: hasIndexablePages ? `${pct$2(semanticPages.length, pageCount)}% of indexable pages use semantic structure.` : "No indexable 2xx pages were available, so semantic HTML coverage was not evaluated.",
|
|
6199
7283
|
action: "Use real headings, lists, tables, nav, main, article, and section elements instead of layout-only divs."
|
|
6200
7284
|
}),
|
|
6201
7285
|
check({
|
|
@@ -6205,7 +7289,7 @@ function aiReadiness(report) {
|
|
|
6205
7289
|
evaluated: hasIndexablePages,
|
|
6206
7290
|
score: 0,
|
|
6207
7291
|
title: "Title, H1, and description coverage observed",
|
|
6208
|
-
plainEnglish: hasIndexablePages ? `${pct$
|
|
7292
|
+
plainEnglish: hasIndexablePages ? `${pct$2(titlePages.length, pageCount)}% of indexable pages have title/H1 coverage and ${pct$2(metaPages.length, pageCount)}% have meta descriptions.` : "No indexable 2xx pages were available, so title, H1, and description coverage was not evaluated.",
|
|
6209
7293
|
action: "Write a unique title and descriptive H1. Add a useful meta description when a summary would help searchers understand the page."
|
|
6210
7294
|
}),
|
|
6211
7295
|
check({
|
|
@@ -6215,7 +7299,7 @@ function aiReadiness(report) {
|
|
|
6215
7299
|
evaluated: hasIndexablePages,
|
|
6216
7300
|
score: 0,
|
|
6217
7301
|
title: "HTML language declaration coverage observed",
|
|
6218
|
-
plainEnglish: hasIndexablePages ? `${pct$
|
|
7302
|
+
plainEnglish: hasIndexablePages ? `${pct$2(langPages.length, pageCount)}% of indexable pages declare a language.` : "No indexable 2xx pages were available, so language declaration coverage was not evaluated.",
|
|
6219
7303
|
action: "Set the html lang attribute so crawlers and assistive tech understand the page language."
|
|
6220
7304
|
})
|
|
6221
7305
|
]),
|
|
@@ -6226,7 +7310,7 @@ function aiReadiness(report) {
|
|
|
6226
7310
|
evaluated: hasIndexablePages,
|
|
6227
7311
|
score: 0,
|
|
6228
7312
|
title: "Entity-signal coverage observed",
|
|
6229
|
-
plainEnglish: hasIndexablePages ? `${pct$
|
|
7313
|
+
plainEnglish: hasIndexablePages ? `${pct$2(entityPages.length, pageCount)}% of indexable pages include entity signals such as Organization, Person, Product, sameAs, or social profile links.` : "No indexable 2xx pages were available, so entity-signal coverage was not evaluated.",
|
|
6230
7314
|
action: "Make the homepage and key pages explicit about the brand, product, people, and official profiles using accurate schema and links.",
|
|
6231
7315
|
urls: sampleUrls$1(pages, (page) => !entityPages.includes(page))
|
|
6232
7316
|
}), check({
|
|
@@ -6270,12 +7354,52 @@ function aiReadiness(report) {
|
|
|
6270
7354
|
section: "technical-ux",
|
|
6271
7355
|
maxScore: 12,
|
|
6272
7356
|
evaluated: report.pages.length > 0,
|
|
6273
|
-
score: scoreFromPercent(pct$
|
|
7357
|
+
score: scoreFromPercent(pct$2(pages.length, report.pages.length || 1), 12),
|
|
6274
7358
|
title: report.pages.length ? "Crawled-page indexability observed" : "Crawled-page indexability was not evaluated",
|
|
6275
|
-
plainEnglish: report.pages.length ? `${pct$
|
|
7359
|
+
plainEnglish: report.pages.length ? `${pct$2(pages.length, report.pages.length)}% of crawled pages are indexable 2xx pages.` : "No crawled pages were available, so status and indexability were not evaluated.",
|
|
6276
7360
|
action: "Fix 4xx/5xx pages, accidental noindex, and blocked internal URLs before expanding AI discovery work."
|
|
6277
7361
|
})
|
|
6278
7362
|
]),
|
|
7363
|
+
section("agent-content", "Agent content observations", [
|
|
7364
|
+
check({
|
|
7365
|
+
id: "markdown-alternates",
|
|
7366
|
+
section: "agent-content",
|
|
7367
|
+
maxScore: 0,
|
|
7368
|
+
score: 0,
|
|
7369
|
+
evaluated: Boolean(agentDiscovery),
|
|
7370
|
+
title: agentDiscovery ? "Markdown alternatives were checked" : "Markdown alternatives were not evaluated",
|
|
7371
|
+
plainEnglish: agentDiscovery ? `${agentDiscovery.markdownAlternates.advertisedPages} of ${agentDiscovery.markdownAlternates.eligibleHtmlPages} successful HTML pages advertised one Markdown alternative. ${agentDiscovery.markdownAlternates.exactByteMatches} explicit responses matched content negotiation byte for byte.` : "This crawl does not include the focused representation checks, so it cannot say whether HTML pages publish stable Markdown alternatives.",
|
|
7372
|
+
action: "Use the focused agent-readiness report when you need route coverage, content negotiation, byte stability, or extraction-quality evidence.",
|
|
7373
|
+
evidence: agentDiscovery ? {
|
|
7374
|
+
eligibleHtmlPages: agentDiscovery.markdownAlternates.eligibleHtmlPages,
|
|
7375
|
+
advertisedPages: agentDiscovery.markdownAlternates.advertisedPages,
|
|
7376
|
+
exactByteMatches: agentDiscovery.markdownAlternates.exactByteMatches,
|
|
7377
|
+
stableResponses: agentDiscovery.markdownAlternates.stableResponses
|
|
7378
|
+
} : void 0
|
|
7379
|
+
}),
|
|
7380
|
+
check({
|
|
7381
|
+
id: "agent-skills-discovery",
|
|
7382
|
+
section: "agent-content",
|
|
7383
|
+
maxScore: 0,
|
|
7384
|
+
score: 0,
|
|
7385
|
+
evaluated: Boolean(agentDiscovery),
|
|
7386
|
+
title: agentDiscovery ? "Agent Skills discovery was checked" : "Agent Skills discovery was not evaluated",
|
|
7387
|
+
plainEnglish: agentDiscovery ? `${agentDiscovery.agentSkills.skills.length} published skill${agentDiscovery.agentSkills.skills.length === 1 ? "" : "s"} were found. ${agentDiscovery.agentSkills.skills.filter((skill) => skill.digestMatches).length} matched their declared SHA-256 digest.` : "This crawl checked common legacy descriptor paths but did not validate the Agent Skills discovery index or its file digests.",
|
|
7388
|
+
action: "Publish Agent Skills only for real reusable capabilities. Generate the index digest from the exact deployed SKILL.md bytes.",
|
|
7389
|
+
evidence: agentDiscovery?.agentSkills
|
|
7390
|
+
}),
|
|
7391
|
+
check({
|
|
7392
|
+
id: "agent-content-profile",
|
|
7393
|
+
section: "agent-content",
|
|
7394
|
+
maxScore: 0,
|
|
7395
|
+
score: 0,
|
|
7396
|
+
evaluated: Boolean(agentDiscovery),
|
|
7397
|
+
title: agentDiscovery ? "The content profile was selected explicitly" : "No focused agent-readiness profile was selected",
|
|
7398
|
+
plainEnglish: agentDiscovery ? "Document access, representations, discovery, identity, and extraction quality were evaluated. API, application, and commerce checks were marked not applicable rather than failed." : "A normal crawl does not assume that a content site also exposes a public API, remote agent endpoint, or checkout.",
|
|
7399
|
+
action: "Run a different profile only when that public capability really exists.",
|
|
7400
|
+
evidence: agentDiscovery?.profileApplicability
|
|
7401
|
+
})
|
|
7402
|
+
]),
|
|
6279
7403
|
section("crawl-completeness", "Crawl completeness", [check({
|
|
6280
7404
|
id: "crawl-depth",
|
|
6281
7405
|
section: "crawl-completeness",
|
|
@@ -6326,117 +7450,6 @@ function aiReadiness(report) {
|
|
|
6326
7450
|
};
|
|
6327
7451
|
}
|
|
6328
7452
|
//#endregion
|
|
6329
|
-
//#region packages/core/dist/analyze/crawler/entity-readiness.js
|
|
6330
|
-
function unique$3(values) {
|
|
6331
|
-
return [...new Set(values.filter((value) => Boolean(value)))];
|
|
6332
|
-
}
|
|
6333
|
-
function pct$2(part, total) {
|
|
6334
|
-
return total ? Math.round(part / total * 100) : 0;
|
|
6335
|
-
}
|
|
6336
|
-
function entityReadiness(report) {
|
|
6337
|
-
const pages = report.pages.filter((page) => page.indexable && page.status < 400);
|
|
6338
|
-
const total = pages.length || 1;
|
|
6339
|
-
const schemaTypes = {};
|
|
6340
|
-
for (const page of pages) for (const type of page.schemaTypes ?? []) schemaTypes[type] = (schemaTypes[type] ?? 0) + 1;
|
|
6341
|
-
const entitySchemaPages = pages.filter((page) => page.schemaTypes?.some((type) => /^(Organization|LocalBusiness|Person|Product|WebSite)$/i.test(type)));
|
|
6342
|
-
const sameAsEvidence = pages.flatMap((page) => page.schemaSameAsEvidence ?? []).filter((evidence) => evidence.subjectTypes.some((type) => /^(Organization|LocalBusiness|Person)$/i.test(type)));
|
|
6343
|
-
const siteSameAsEvidence = sameAsEvidence.filter((evidence) => evidence.subjectTypes.some((type) => /^(Organization|LocalBusiness)$/i.test(type)));
|
|
6344
|
-
const sameAs = unique$3(sameAsEvidence.map((evidence) => evidence.url));
|
|
6345
|
-
const sameAsByType = {};
|
|
6346
|
-
for (const evidence of sameAsEvidence) for (const type of evidence.subjectTypes) {
|
|
6347
|
-
if (!/^(Organization|LocalBusiness|Person)$/i.test(type)) continue;
|
|
6348
|
-
sameAsByType[type] = unique$3([...sameAsByType[type] ?? [], evidence.url]);
|
|
6349
|
-
}
|
|
6350
|
-
const socialProfiles = unique$3(pages.flatMap((page) => page.socialProfileLinks ?? []));
|
|
6351
|
-
const authors = unique$3(pages.map((page) => page.author));
|
|
6352
|
-
const authorPages = pages.filter((page) => page.author || page.geo?.hasAuthor);
|
|
6353
|
-
const datePages = pages.filter((page) => page.hasDate || page.geo?.hasDate);
|
|
6354
|
-
const titlePages = pages.filter((page) => page.title && page.h1);
|
|
6355
|
-
const checks = [
|
|
6356
|
-
{
|
|
6357
|
-
coverage: pct$2(entitySchemaPages.length, total),
|
|
6358
|
-
check: {
|
|
6359
|
-
id: "entity-schema",
|
|
6360
|
-
title: "Entity schema is present",
|
|
6361
|
-
plainEnglish: `${pct$2(entitySchemaPages.length, total)}% of evaluated indexable pages include Organization, LocalBusiness, Person, Product, or WebSite schema.`,
|
|
6362
|
-
action: "Add accurate entity schema to the homepage and key product, author, organization, and local pages.",
|
|
6363
|
-
evidence: { schemaTypes },
|
|
6364
|
-
urls: pages.filter((page) => !entitySchemaPages.includes(page)).slice(0, 10).map((page) => page.finalUrl)
|
|
6365
|
-
}
|
|
6366
|
-
},
|
|
6367
|
-
{
|
|
6368
|
-
coverage: siteSameAsEvidence.length ? 100 : 0,
|
|
6369
|
-
check: {
|
|
6370
|
-
id: "same-as",
|
|
6371
|
-
title: "Site entity profiles are connected",
|
|
6372
|
-
plainEnglish: siteSameAsEvidence.length ? "The crawl found sameAs links attached to Organization or LocalBusiness structured data." : sameAs.length ? "The crawl found Person sameAs links, but no Organization or LocalBusiness profile evidence for the site entity." : socialProfiles.length ? "The crawl found social-domain links, but they are not enough to prove official profile ownership." : "The crawl did not find entity-scoped sameAs evidence.",
|
|
6373
|
-
action: "Connect official profiles with sameAs on Organization, Person, or LocalBusiness schema. Use only profiles you control.",
|
|
6374
|
-
evidence: {
|
|
6375
|
-
sameAs,
|
|
6376
|
-
siteSameAs: unique$3(siteSameAsEvidence.map((evidence) => evidence.url)),
|
|
6377
|
-
sameAsEvidence,
|
|
6378
|
-
unclassifiedSocialLinks: socialProfiles
|
|
6379
|
-
}
|
|
6380
|
-
}
|
|
6381
|
-
},
|
|
6382
|
-
{
|
|
6383
|
-
coverage: Math.round((pct$2(authorPages.length, total) + pct$2(datePages.length, total)) / 2),
|
|
6384
|
-
check: {
|
|
6385
|
-
id: "authority-freshness",
|
|
6386
|
-
title: "Author and date signals observed",
|
|
6387
|
-
plainEnglish: `${pct$2(authorPages.length, total)}% of evaluated indexable pages expose an author signal and ${pct$2(datePages.length, total)}% expose a date signal.`,
|
|
6388
|
-
action: "For editorial content, identify authors and show accurate published or modified dates when that information is useful to readers.",
|
|
6389
|
-
evidence: {
|
|
6390
|
-
authorCoverage: pct$2(authorPages.length, total),
|
|
6391
|
-
dateCoverage: pct$2(datePages.length, total),
|
|
6392
|
-
authors
|
|
6393
|
-
}
|
|
6394
|
-
}
|
|
6395
|
-
},
|
|
6396
|
-
{
|
|
6397
|
-
coverage: pct$2(titlePages.length, total),
|
|
6398
|
-
check: {
|
|
6399
|
-
id: "entity-naming",
|
|
6400
|
-
title: "Title and H1 coverage observed",
|
|
6401
|
-
plainEnglish: `${pct$2(titlePages.length, total)}% of evaluated indexable pages have both a title and H1.`,
|
|
6402
|
-
action: "Review pages missing either signal. Keep brand, product, and person names accurate wherever they appear."
|
|
6403
|
-
}
|
|
6404
|
-
}
|
|
6405
|
-
].map(({ coverage, check }) => ({
|
|
6406
|
-
...check,
|
|
6407
|
-
status: "info",
|
|
6408
|
-
evidence: {
|
|
6409
|
-
...check.evidence,
|
|
6410
|
-
observedCoveragePercent: coverage
|
|
6411
|
-
}
|
|
6412
|
-
}));
|
|
6413
|
-
const dataStatus = report.status === "completed" && pages.length > 0 ? "complete" : "partial";
|
|
6414
|
-
return {
|
|
6415
|
-
reportId: report.id,
|
|
6416
|
-
url: report.config.url,
|
|
6417
|
-
generatedAt: report.generatedAt,
|
|
6418
|
-
dataStatus,
|
|
6419
|
-
evaluatedPages: pages.length,
|
|
6420
|
-
crawlPages: report.pages.length,
|
|
6421
|
-
assessment: "evidence-only",
|
|
6422
|
-
headline: dataStatus === "partial" ? "Entity evidence is incomplete; treat these findings as scoped to the evaluated pages, not the whole site." : "Entity signals are reported as observations, not a ranking, visibility, or machine-understanding score.",
|
|
6423
|
-
caveats: [
|
|
6424
|
-
...report.caveats,
|
|
6425
|
-
"This report deliberately has no aggregate entity score. Add schema, authors, dates, and sameAs only where they accurately fit the page and a documented use case.",
|
|
6426
|
-
...dataStatus === "partial" ? [`Entity coverage is based on ${pages.length} evaluated indexable page${pages.length === 1 ? "" : "s"} from a ${report.status} crawl.`] : [],
|
|
6427
|
-
...socialProfiles.length && !sameAs.length ? ["Social-domain links are unclassified navigation evidence, not proof that the linked profiles are official."] : []
|
|
6428
|
-
],
|
|
6429
|
-
checks,
|
|
6430
|
-
entities: {
|
|
6431
|
-
schemaTypes,
|
|
6432
|
-
sameAs,
|
|
6433
|
-
sameAsByType,
|
|
6434
|
-
socialProfiles,
|
|
6435
|
-
authors
|
|
6436
|
-
}
|
|
6437
|
-
};
|
|
6438
|
-
}
|
|
6439
|
-
//#endregion
|
|
6440
7453
|
//#region packages/core/dist/analyze/crawler/geo-gaps.js
|
|
6441
7454
|
const ELIGIBILITY_RULES = /* @__PURE__ */ new Set([
|
|
6442
7455
|
"connection_error",
|
|
@@ -6504,7 +7517,7 @@ function partialReasons(report) {
|
|
|
6504
7517
|
if (report.requestEvidenceStatus === "partial") reasons.push("request-evidence-partial");
|
|
6505
7518
|
if (report.requestEvidenceStatus === "unavailable") reasons.push("request-evidence-unavailable");
|
|
6506
7519
|
if (report.summary.pageLimitReached) reasons.push("page-limit-reached");
|
|
6507
|
-
if (report.summary.
|
|
7520
|
+
if (report.summary.skippedUrlsByImpact.coverageAffecting > 0) reasons.push("coverage-affecting-skips");
|
|
6508
7521
|
if (report.summary.failedRequests > 0) reasons.push("failed-requests");
|
|
6509
7522
|
if (report.summary.extractionFailures > 0) reasons.push("extraction-failures");
|
|
6510
7523
|
return reasons;
|
|
@@ -6532,6 +7545,8 @@ function geoGapsReport(report, filters = {}) {
|
|
|
6532
7545
|
queuedUrls: report.summary.queuedUrls,
|
|
6533
7546
|
crawledUrls: report.summary.crawledUrls,
|
|
6534
7547
|
skippedUrls: report.summary.skippedUrls,
|
|
7548
|
+
coverageAffectingSkippedUrls: report.summary.skippedUrlsByImpact.coverageAffecting,
|
|
7549
|
+
nonImpactingSkippedUrls: report.summary.skippedUrlsByImpact.nonImpacting,
|
|
6535
7550
|
failedRequests: report.summary.failedRequests,
|
|
6536
7551
|
extractionFailures: report.summary.extractionFailures,
|
|
6537
7552
|
partialReasons: reasons
|
|
@@ -6594,8 +7609,56 @@ function candidatePages(report, options = {}) {
|
|
|
6594
7609
|
}
|
|
6595
7610
|
function auditLlmsTxt(report) {
|
|
6596
7611
|
const llmsTxt = report.ai?.llmsTxt;
|
|
7612
|
+
const validation = report.agentDiscovery?.llmsTxt;
|
|
6597
7613
|
const pages = candidatePages(report);
|
|
6598
7614
|
const issues = [];
|
|
7615
|
+
if (validation?.exists && !/^\s*(?:text\/plain|text\/markdown)\b/iu.test(validation.contentType ?? "")) issues.push({
|
|
7616
|
+
id: "llms-content-type",
|
|
7617
|
+
severity: "medium",
|
|
7618
|
+
title: "llms.txt uses an unexpected content type",
|
|
7619
|
+
plainEnglish: `The file returned ${validation.contentType ?? "no content type"} instead of plain text or Markdown.`,
|
|
7620
|
+
action: "Serve the deterministic text file as text/plain or text/markdown so consumers do not mistake it for HTML.",
|
|
7621
|
+
evidence: { contentType: validation.contentType }
|
|
7622
|
+
});
|
|
7623
|
+
if (validation?.exists && validation.headingCount === 0) issues.push({
|
|
7624
|
+
id: "llms-structure",
|
|
7625
|
+
severity: "medium",
|
|
7626
|
+
title: "llms.txt has no useful heading structure",
|
|
7627
|
+
plainEnglish: "The fetched file did not contain a level-one or level-two Markdown heading.",
|
|
7628
|
+
action: "Start with one clear title, then group a short list of useful entry points under descriptive headings."
|
|
7629
|
+
});
|
|
7630
|
+
if (validation?.duplicateLinks.length) issues.push({
|
|
7631
|
+
id: "llms-duplicate-links",
|
|
7632
|
+
severity: "low",
|
|
7633
|
+
title: "llms.txt repeats the same link",
|
|
7634
|
+
plainEnglish: `${validation.duplicateLinks.length} URL${validation.duplicateLinks.length === 1 ? " appears" : "s appear"} more than once.`,
|
|
7635
|
+
action: "Remove duplicate entries so the file remains short and deliberate.",
|
|
7636
|
+
evidence: { urls: validation.duplicateLinks }
|
|
7637
|
+
});
|
|
7638
|
+
const brokenLinks = validation?.links.filter((link) => !link.status || link.status < 200 || link.status >= 400 || link.error) ?? [];
|
|
7639
|
+
if (brokenLinks.length) issues.push({
|
|
7640
|
+
id: "llms-broken-links",
|
|
7641
|
+
severity: "medium",
|
|
7642
|
+
title: "llms.txt links did not resolve",
|
|
7643
|
+
plainEnglish: `${brokenLinks.length} declared link${brokenLinks.length === 1 ? "" : "s"} did not return a usable response during this audit.`,
|
|
7644
|
+
action: "Update or remove stale entries, then rerun the focused agent-readiness report.",
|
|
7645
|
+
evidence: { links: brokenLinks }
|
|
7646
|
+
});
|
|
7647
|
+
if (validation?.missingCrawlRoutes.length) issues.push({
|
|
7648
|
+
id: "llms-outside-crawl",
|
|
7649
|
+
severity: "low",
|
|
7650
|
+
title: "llms.txt links fall outside the crawl inventory",
|
|
7651
|
+
plainEnglish: `${validation.missingCrawlRoutes.length} same-origin route${validation.missingCrawlRoutes.length === 1 ? " was" : "s were"} not present in the retained crawl.`,
|
|
7652
|
+
action: "Confirm those routes are intentional and indexable, or remove stale declarations.",
|
|
7653
|
+
evidence: { urls: validation.missingCrawlRoutes }
|
|
7654
|
+
});
|
|
7655
|
+
if (validation?.repeatedHashStable === false) issues.push({
|
|
7656
|
+
id: "llms-unstable-body",
|
|
7657
|
+
severity: "medium",
|
|
7658
|
+
title: "llms.txt changed between repeated requests",
|
|
7659
|
+
plainEnglish: "Two requests in the same audit returned different SHA-256 digests.",
|
|
7660
|
+
action: "Generate the file during the build and remove timestamps, random ordering, or runtime rewriting."
|
|
7661
|
+
});
|
|
6599
7662
|
if (pages.length < 3) issues.push({
|
|
6600
7663
|
id: "thin-llms-inventory",
|
|
6601
7664
|
severity: "low",
|
|
@@ -6613,7 +7676,7 @@ function auditLlmsTxt(report) {
|
|
|
6613
7676
|
optional: true,
|
|
6614
7677
|
googleSearchImpact: "none",
|
|
6615
7678
|
guidanceUrl: "https://developers.google.com/search/updates#clarifying-guidance-on-llms-txt-files",
|
|
6616
|
-
headline: llmsTxt?.exists ? "An optional llms.txt file is present. It has no Google Search visibility impact." : "No llms.txt file was found. This is not an SEO issue and requires no action.",
|
|
7679
|
+
headline: llmsTxt?.exists ? validation ? issues.length ? `The optional llms.txt file was validated and ${issues.length} content or link issue${issues.length === 1 ? " needs" : "s need"} review. It has no Google Search visibility impact.` : `The optional llms.txt file was validated with ${validation.links.length} resolving link${validation.links.length === 1 ? "" : "s"} and a stable body. It has no Google Search visibility impact.` : "An optional llms.txt file is present, but its body was not validated in this crawl. It has no Google Search visibility impact." : "No llms.txt file was found. This is not an SEO issue and requires no action.",
|
|
6617
7680
|
issues,
|
|
6618
7681
|
recommendedPages: pages.slice(0, 25).map((page) => ({
|
|
6619
7682
|
url: page.finalUrl,
|
|
@@ -6712,8 +7775,6 @@ const SCORECARD_CHECK_ORDER = [
|
|
|
6712
7775
|
"entity-identity",
|
|
6713
7776
|
"answerable-content"
|
|
6714
7777
|
];
|
|
6715
|
-
const ENTITY_SCHEMA_PATTERN = /^(Organization|LocalBusiness|Person|Product|WebSite)$/i;
|
|
6716
|
-
const SITE_ENTITY_TYPE_PATTERN = /^(Organization|LocalBusiness)$/i;
|
|
6717
7778
|
function indexablePages(report) {
|
|
6718
7779
|
return report.pages.filter((page) => page.indexable && page.status < 400);
|
|
6719
7780
|
}
|
|
@@ -6921,8 +7982,8 @@ function entityIdentityCheck(report) {
|
|
|
6921
7982
|
finding: "Entity identity could not be evaluated because no indexable 2xx pages were available.",
|
|
6922
7983
|
verification
|
|
6923
7984
|
};
|
|
6924
|
-
const hasEntitySchema = Object.keys(entity.entities.schemaTypes).some(
|
|
6925
|
-
const siteSameAs = Object.entries(entity.entities.sameAsByType).filter(([type]) =>
|
|
7985
|
+
const hasEntitySchema = Object.keys(entity.entities.schemaTypes).some(isEntityEvidenceSchemaType);
|
|
7986
|
+
const siteSameAs = Object.entries(entity.entities.sameAsByType).filter(([type]) => isSiteIdentitySchemaType(type)).flatMap(([, urls]) => urls);
|
|
6926
7987
|
const uniqueSiteSameAs = [...new Set(siteSameAs)].sort();
|
|
6927
7988
|
const status = uniqueSiteSameAs.length ? "pass" : hasEntitySchema ? "warn" : "fail";
|
|
6928
7989
|
return {
|
|
@@ -6935,7 +7996,7 @@ function entityIdentityCheck(report) {
|
|
|
6935
7996
|
sameAs: entity.entities.sameAs,
|
|
6936
7997
|
socialProfiles: entity.entities.socialProfiles
|
|
6937
7998
|
},
|
|
6938
|
-
finding: status === "pass" ? "The crawl found sameAs links attached to Organization or LocalBusiness structured data for the site entity." : status === "warn" ? "Entity schema is present, but no Organization or LocalBusiness sameAs evidence connects the site entity to official profiles." : "No Organization, LocalBusiness, Person, Product, or WebSite schema was found on the evaluated pages.",
|
|
7999
|
+
finding: status === "pass" ? "The crawl found sameAs links attached to Organization or LocalBusiness structured data for the site entity." : status === "warn" ? "Entity schema is present, but no Organization or LocalBusiness sameAs evidence connects the site entity to official profiles." : "No Organization, LocalBusiness, Person, Product, SoftwareApplication, or WebSite schema was found on the evaluated pages.",
|
|
6939
8000
|
verification
|
|
6940
8001
|
};
|
|
6941
8002
|
}
|
|
@@ -7428,7 +8489,7 @@ const RULE_DEFINITIONS = [
|
|
|
7428
8489
|
whyItMatters: "External links are part of the page experience and source trail. Broken references make content feel stale and less trustworthy.",
|
|
7429
8490
|
howToFix: "Replace the link with a working source, update the URL, or remove the reference if it no longer supports the page.",
|
|
7430
8491
|
impactIfIgnored: "Readers and agents following citations hit dead ends, which weakens trust in the content.",
|
|
7431
|
-
howToVerify: "Re-run the crawl with external checks enabled and confirm the
|
|
8492
|
+
howToVerify: "Re-run the crawl with external checks enabled and confirm the URL no longer has a confirmed-broken externalLinkChecks result."
|
|
7432
8493
|
},
|
|
7433
8494
|
{
|
|
7434
8495
|
id: "orphan_page",
|
|
@@ -8086,11 +9147,13 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
8086
9147
|
if (page.hasHsts === false) issues.push(issue("hsts_missing", page, void 0, { header: "strict-transport-security" }));
|
|
8087
9148
|
}
|
|
8088
9149
|
for (const link of page.externalLinkChecks ?? []) {
|
|
8089
|
-
if (
|
|
9150
|
+
if (link.state !== "confirmed-broken") continue;
|
|
8090
9151
|
issues.push(issue("broken_external_link", page, link.url, {
|
|
8091
9152
|
url: link.url,
|
|
8092
9153
|
status: link.status,
|
|
8093
|
-
error: link.error
|
|
9154
|
+
error: link.error,
|
|
9155
|
+
state: link.state,
|
|
9156
|
+
attempts: link.attempts
|
|
8094
9157
|
}));
|
|
8095
9158
|
}
|
|
8096
9159
|
if (page.indexable && (page.internalInlinkCount ?? 0) === 0 && !sameUrl(page.url, opts.startUrl)) issues.push(issue("orphan_page", page, "No internal inlinks", { internalInlinkCount: page.internalInlinkCount ?? 0 }));
|
|
@@ -8225,6 +9288,80 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
8225
9288
|
return issues;
|
|
8226
9289
|
}
|
|
8227
9290
|
//#endregion
|
|
9291
|
+
//#region packages/core/dist/analyze/crawler/crawl-skip-reasons.js
|
|
9292
|
+
const CRAWL_SKIP_REASONS = [
|
|
9293
|
+
"asset-url",
|
|
9294
|
+
"configured-exclusion",
|
|
9295
|
+
"duplicate-normalization",
|
|
9296
|
+
"invalid-url",
|
|
9297
|
+
"off-origin",
|
|
9298
|
+
"origin-backpressure",
|
|
9299
|
+
"queue-safety-limit",
|
|
9300
|
+
"robots-disallowed",
|
|
9301
|
+
"robots-uncertain",
|
|
9302
|
+
"legacy-unclassified"
|
|
9303
|
+
];
|
|
9304
|
+
const SKIP_IMPACT = {
|
|
9305
|
+
"asset-url": "non-impacting",
|
|
9306
|
+
"configured-exclusion": "non-impacting",
|
|
9307
|
+
"duplicate-normalization": "non-impacting",
|
|
9308
|
+
"invalid-url": "non-impacting",
|
|
9309
|
+
"off-origin": "non-impacting",
|
|
9310
|
+
"origin-backpressure": "coverage-affecting",
|
|
9311
|
+
"queue-safety-limit": "coverage-affecting",
|
|
9312
|
+
"robots-disallowed": "coverage-affecting",
|
|
9313
|
+
"robots-uncertain": "coverage-affecting",
|
|
9314
|
+
"legacy-unclassified": "coverage-affecting"
|
|
9315
|
+
};
|
|
9316
|
+
const REASON_ORDER = new Map(CRAWL_SKIP_REASONS.map((reason, index) => [reason, index]));
|
|
9317
|
+
function isCrawlSkipReason(value) {
|
|
9318
|
+
return typeof value === "string" && CRAWL_SKIP_REASONS.includes(value);
|
|
9319
|
+
}
|
|
9320
|
+
function validCount$1(value) {
|
|
9321
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : 0;
|
|
9322
|
+
}
|
|
9323
|
+
function crawlSkipImpact(reason) {
|
|
9324
|
+
return SKIP_IMPACT[reason];
|
|
9325
|
+
}
|
|
9326
|
+
function normalizeCrawlSkipReasonCounts(input) {
|
|
9327
|
+
const counts = /* @__PURE__ */ new Map();
|
|
9328
|
+
if (Array.isArray(input.skipReasons)) for (const item of input.skipReasons) {
|
|
9329
|
+
if (!item || typeof item !== "object") continue;
|
|
9330
|
+
const { reason, count } = item;
|
|
9331
|
+
if (!isCrawlSkipReason(reason)) continue;
|
|
9332
|
+
const normalizedCount = validCount$1(count);
|
|
9333
|
+
if (!normalizedCount) continue;
|
|
9334
|
+
counts.set(reason, (counts.get(reason) ?? 0) + normalizedCount);
|
|
9335
|
+
}
|
|
9336
|
+
const declaredTotal = validCount$1(input.skippedUrls);
|
|
9337
|
+
const classifiedTotal = [...counts.values()].reduce((total, count) => total + count, 0);
|
|
9338
|
+
if (declaredTotal > classifiedTotal) counts.set("legacy-unclassified", (counts.get("legacy-unclassified") ?? 0) + declaredTotal - classifiedTotal);
|
|
9339
|
+
const skipReasons = [...counts.entries()].map(([reason, count]) => ({
|
|
9340
|
+
reason,
|
|
9341
|
+
impact: crawlSkipImpact(reason),
|
|
9342
|
+
count
|
|
9343
|
+
})).sort((left, right) => (REASON_ORDER.get(left.reason) ?? Number.MAX_SAFE_INTEGER) - (REASON_ORDER.get(right.reason) ?? Number.MAX_SAFE_INTEGER));
|
|
9344
|
+
const skippedUrlsByImpact = skipReasons.reduce((totals, item) => {
|
|
9345
|
+
if (item.impact === "coverage-affecting") totals.coverageAffecting += item.count;
|
|
9346
|
+
else totals.nonImpacting += item.count;
|
|
9347
|
+
return totals;
|
|
9348
|
+
}, {
|
|
9349
|
+
coverageAffecting: 0,
|
|
9350
|
+
nonImpacting: 0
|
|
9351
|
+
});
|
|
9352
|
+
return {
|
|
9353
|
+
skippedUrls: skippedUrlsByImpact.coverageAffecting + skippedUrlsByImpact.nonImpacting,
|
|
9354
|
+
skipReasons,
|
|
9355
|
+
skippedUrlsByImpact
|
|
9356
|
+
};
|
|
9357
|
+
}
|
|
9358
|
+
function crawlSkipReasonCountsFromRecord(input = {}) {
|
|
9359
|
+
return normalizeCrawlSkipReasonCounts({ skipReasons: Object.entries(input).map(([reason, count]) => ({
|
|
9360
|
+
reason,
|
|
9361
|
+
count
|
|
9362
|
+
})) }).skipReasons;
|
|
9363
|
+
}
|
|
9364
|
+
//#endregion
|
|
8228
9365
|
//#region packages/core/dist/analyze/crawler/report.js
|
|
8229
9366
|
const SENSITIVE_FIELD_NAMES = /* @__PURE__ */ new Set([
|
|
8230
9367
|
"authorization",
|
|
@@ -8263,6 +9400,7 @@ function normalizeCrawlConfig(input) {
|
|
|
8263
9400
|
respectRobots: input.respectRobots ?? true,
|
|
8264
9401
|
useSitemap: input.useSitemap ?? true,
|
|
8265
9402
|
checkExternal: input.checkExternal ?? true,
|
|
9403
|
+
checkAgentDiscovery: input.checkAgentDiscovery ?? false,
|
|
8266
9404
|
js: normalizeJavaScriptRenderingMode(input.js),
|
|
8267
9405
|
refresh: input.refresh ?? false,
|
|
8268
9406
|
fetchRate: normalizeFetchRate(input)
|
|
@@ -8382,12 +9520,13 @@ function summarizeCrawlReport(input) {
|
|
|
8382
9520
|
}
|
|
8383
9521
|
}
|
|
8384
9522
|
for (const issue of input.issues) byCategory[issue.category] = (byCategory[issue.category] ?? 0) + 1;
|
|
9523
|
+
const runStats = crawlRunStats(input.pages, input.stats);
|
|
8385
9524
|
return {
|
|
8386
9525
|
totalPages: input.pages.length,
|
|
8387
9526
|
indexablePages: input.pages.filter((page) => page.indexable).length,
|
|
8388
9527
|
nonIndexablePages: input.pages.filter((page) => !page.indexable).length,
|
|
8389
9528
|
statusErrors: input.pages.filter((page) => page.status === 0 || page.status >= 400).length,
|
|
8390
|
-
...
|
|
9529
|
+
...runStats,
|
|
8391
9530
|
attemptedRequests: input.requestEvidenceStatus === "unavailable" ? 0 : requests.length,
|
|
8392
9531
|
responseRequests: requests.filter((request) => request.outcome === "response").length,
|
|
8393
9532
|
failedRequests: failedRequests.length,
|
|
@@ -8412,11 +9551,17 @@ function crawlRunStats(pages, stats = {}) {
|
|
|
8412
9551
|
for (const url of page.sampleExternalLinks ?? []) discovered.add(url);
|
|
8413
9552
|
observedInternalLinks += page.outgoingInternalCount;
|
|
8414
9553
|
}
|
|
9554
|
+
const skips = normalizeCrawlSkipReasonCounts({
|
|
9555
|
+
skippedUrls: stats.skippedUrls,
|
|
9556
|
+
skipReasons: stats.skipReasons ?? crawlSkipReasonCountsFromRecord(stats.skipReasonCounts)
|
|
9557
|
+
});
|
|
8415
9558
|
return {
|
|
8416
9559
|
discoveredUrls: stats.discoveredUrls ?? discovered.size,
|
|
8417
9560
|
queuedUrls: stats.queuedUrls ?? pages.length,
|
|
8418
9561
|
crawledUrls: stats.crawledUrls ?? pages.length,
|
|
8419
|
-
skippedUrls:
|
|
9562
|
+
skippedUrls: skips.skippedUrls,
|
|
9563
|
+
skipReasons: skips.skipReasons,
|
|
9564
|
+
skippedUrlsByImpact: skips.skippedUrlsByImpact,
|
|
8420
9565
|
failedUrls: stats.failedUrls ?? pages.filter((page) => page.status === 0 || page.status >= 400).length,
|
|
8421
9566
|
observedInternalLinks: stats.observedInternalLinks ?? observedInternalLinks,
|
|
8422
9567
|
pageLimitReached: stats.pageLimitReached ?? false
|
|
@@ -8506,6 +9651,7 @@ function createCrawlReport(input) {
|
|
|
8506
9651
|
...input.ai ? { ai: sanitizeTenantValue(input.ai) } : {},
|
|
8507
9652
|
...input.sitemapDiscovery ? { sitemapDiscovery: sanitizeTenantValue(input.sitemapDiscovery) } : {},
|
|
8508
9653
|
...input.externalLinkVerification ? { externalLinkVerification: sanitizeTenantValue(input.externalLinkVerification) } : {},
|
|
9654
|
+
...input.agentDiscovery ? { agentDiscovery: sanitizeTenantValue(input.agentDiscovery) } : {},
|
|
8509
9655
|
warnings: sanitizeMessages(input.warnings).sort(compareText$13),
|
|
8510
9656
|
caveats: sanitizeMessages(input.caveats).sort(compareText$13)
|
|
8511
9657
|
};
|
|
@@ -8557,7 +9703,7 @@ function storageEnvelope(report) {
|
|
|
8557
9703
|
if (report.site) tenant.site = report.site;
|
|
8558
9704
|
return {
|
|
8559
9705
|
kind: "seo.crawl_report",
|
|
8560
|
-
version:
|
|
9706
|
+
version: 6,
|
|
8561
9707
|
savedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8562
9708
|
...tenant.projectId || tenant.site ? { tenant } : {},
|
|
8563
9709
|
report
|
|
@@ -8593,7 +9739,7 @@ function saveCrawlReportToSqlite(report) {
|
|
|
8593
9739
|
totalPages: report.summary.totalPages,
|
|
8594
9740
|
issueCount: report.issues.length,
|
|
8595
9741
|
createdAt: report.generatedAt,
|
|
8596
|
-
storageVersion:
|
|
9742
|
+
storageVersion: 6
|
|
8597
9743
|
};
|
|
8598
9744
|
}
|
|
8599
9745
|
function listCrawlReportsFromSqlite(input = {}) {
|
|
@@ -8676,6 +9822,15 @@ function cancellationRace(signal) {
|
|
|
8676
9822
|
//#endregion
|
|
8677
9823
|
//#region packages/core/dist/analyze/crawler/external-link-checks.js
|
|
8678
9824
|
const EXTERNAL_LINK_SELECTION_LIMIT = 200;
|
|
9825
|
+
const EXTERNAL_LINK_STATES = [
|
|
9826
|
+
"available",
|
|
9827
|
+
"confirmed-broken",
|
|
9828
|
+
"transient",
|
|
9829
|
+
"provider-blocked",
|
|
9830
|
+
"rate-limited",
|
|
9831
|
+
"method-rejected",
|
|
9832
|
+
"unavailable"
|
|
9833
|
+
];
|
|
8679
9834
|
function stableHash(value) {
|
|
8680
9835
|
let hash = 2166136261;
|
|
8681
9836
|
for (const character of value) hash = Math.imul(hash ^ (character.codePointAt(0) ?? 0), 16777619);
|
|
@@ -8745,32 +9900,73 @@ function selectCandidateUrls(candidates, limit) {
|
|
|
8745
9900
|
}
|
|
8746
9901
|
return [...selected];
|
|
8747
9902
|
}
|
|
8748
|
-
|
|
9903
|
+
function isAvailableStatus(status) {
|
|
9904
|
+
return status !== void 0 && status >= 200 && status < 400;
|
|
9905
|
+
}
|
|
9906
|
+
function isBrokenStatus(status) {
|
|
9907
|
+
return status === 404 || status === 410;
|
|
9908
|
+
}
|
|
9909
|
+
function isMethodRejectedStatus(status) {
|
|
9910
|
+
return status === 405 || status === 501;
|
|
9911
|
+
}
|
|
9912
|
+
function isProviderBlockedStatus(status) {
|
|
9913
|
+
return status === 401 || status === 403 || status === 451;
|
|
9914
|
+
}
|
|
9915
|
+
function isTransientStatus(status) {
|
|
9916
|
+
return status === 408 || status === 425 || status !== void 0 && status >= 500;
|
|
9917
|
+
}
|
|
9918
|
+
function stateForAttempts(attempts) {
|
|
9919
|
+
const last = attempts.at(-1);
|
|
9920
|
+
const statuses = attempts.map((attempt) => attempt.status).filter((status) => status !== void 0);
|
|
9921
|
+
if (isAvailableStatus(last?.status)) return isMethodRejectedStatus(attempts[0]?.status) ? "method-rejected" : attempts.length > 1 ? "transient" : "available";
|
|
9922
|
+
if (attempts.length >= 2 && isBrokenStatus(last?.status) && attempts.at(-2)?.status === last?.status) return "confirmed-broken";
|
|
9923
|
+
if (last?.status === 429) return "rate-limited";
|
|
9924
|
+
if (isProviderBlockedStatus(last?.status)) return "provider-blocked";
|
|
9925
|
+
if (isMethodRejectedStatus(last?.status)) return "method-rejected";
|
|
9926
|
+
if (isTransientStatus(last?.status) || statuses.some(isBrokenStatus) || statuses.length > 0 && attempts.some((attempt) => attempt.error)) return "transient";
|
|
9927
|
+
return "unavailable";
|
|
9928
|
+
}
|
|
9929
|
+
async function requestExternalLink(url, method, timeoutMs, fetch, signal) {
|
|
8749
9930
|
const controller = abortController({
|
|
8750
9931
|
timeoutMs,
|
|
8751
9932
|
signal
|
|
8752
9933
|
});
|
|
8753
9934
|
try {
|
|
8754
9935
|
const response = await fetch(url, {
|
|
8755
|
-
method
|
|
9936
|
+
method,
|
|
8756
9937
|
redirect: "follow",
|
|
8757
9938
|
signal: controller.signal
|
|
8758
9939
|
});
|
|
8759
9940
|
await response.body?.cancel().catch(() => void 0);
|
|
8760
9941
|
return {
|
|
8761
|
-
|
|
9942
|
+
method,
|
|
8762
9943
|
status: response.status
|
|
8763
9944
|
};
|
|
8764
9945
|
} catch (error) {
|
|
8765
9946
|
return {
|
|
8766
|
-
|
|
8767
|
-
status: 0,
|
|
9947
|
+
method,
|
|
8768
9948
|
error: error instanceof Error ? error.message : String(error)
|
|
8769
9949
|
};
|
|
8770
9950
|
} finally {
|
|
8771
9951
|
controller.cleanup();
|
|
8772
9952
|
}
|
|
8773
9953
|
}
|
|
9954
|
+
async function checkExternalLink(url, timeoutMs, fetch, signal) {
|
|
9955
|
+
const attempts = [];
|
|
9956
|
+
attempts.push(await requestExternalLink(url, "HEAD", timeoutMs, fetch, signal));
|
|
9957
|
+
const head = attempts[0];
|
|
9958
|
+
if (!signal?.aborted && !isAvailableStatus(head?.status) && head?.status !== 429 && !isProviderBlockedStatus(head?.status)) attempts.push(await requestExternalLink(url, "GET", timeoutMs, fetch, signal));
|
|
9959
|
+
const last = attempts.at(-1);
|
|
9960
|
+
if (!signal?.aborted && isBrokenStatus(last?.status) && attempts.at(-2)?.status !== last?.status) attempts.push(await requestExternalLink(url, "GET", timeoutMs, fetch, signal));
|
|
9961
|
+
const finalAttempt = attempts.at(-1);
|
|
9962
|
+
return {
|
|
9963
|
+
url,
|
|
9964
|
+
...finalAttempt?.status !== void 0 ? { status: finalAttempt.status } : {},
|
|
9965
|
+
...finalAttempt?.error ? { error: finalAttempt.error } : {},
|
|
9966
|
+
state: stateForAttempts(attempts),
|
|
9967
|
+
attempts
|
|
9968
|
+
};
|
|
9969
|
+
}
|
|
8774
9970
|
async function verifyExternalLinks(input) {
|
|
8775
9971
|
const { candidates, discoveredLinkOccurrences, retainedLinkOccurrences } = collectCandidates(input.pages);
|
|
8776
9972
|
const urls = input.signal?.aborted ? [] : selectCandidateUrls(candidates, EXTERNAL_LINK_SELECTION_LIMIT);
|
|
@@ -8785,16 +9981,22 @@ async function verifyExternalLinks(input) {
|
|
|
8785
9981
|
const pageChecks = (page.sampleExternalLinks ?? []).map((url) => checks.get(url)).filter((value) => Boolean(value));
|
|
8786
9982
|
if (pageChecks.length) page.externalLinkChecks = pageChecks;
|
|
8787
9983
|
}
|
|
8788
|
-
const fetchedUrls = [...checks.values()].filter((check) =>
|
|
8789
|
-
const failedUrls = checks.
|
|
9984
|
+
const fetchedUrls = [...checks.values()].filter((check) => check.attempts?.some((attempt) => attempt.status !== void 0)).length;
|
|
9985
|
+
const failedUrls = [...checks.values()].filter((check) => check.state === "unavailable").length;
|
|
9986
|
+
const unresolvedMethodRejectedUrls = [...checks.values()].filter((check) => check.state === "method-rejected" && !isAvailableStatus(check.status)).length;
|
|
8790
9987
|
const deferredUrls = candidates.length - checks.size;
|
|
9988
|
+
const outcomes = Object.fromEntries(EXTERNAL_LINK_STATES.map((state) => [state, [...checks.values()].filter((check) => check.state === state).length]));
|
|
8791
9989
|
const warnings = [];
|
|
8792
9990
|
if (discoveredLinkOccurrences > retainedLinkOccurrences) warnings.push(`Retained ${retainedLinkOccurrences} sampled external link occurrence${retainedLinkOccurrences === 1 ? "" : "s"} from ${discoveredLinkOccurrences} observed occurrence${discoveredLinkOccurrences === 1 ? "" : "s"}.`);
|
|
8793
9991
|
if (candidates.length > urls.length) warnings.push(`Selected ${urls.length} of ${candidates.length} retained external URLs using the ${EXTERNAL_LINK_SELECTION_LIMIT}-URL verification limit.`);
|
|
8794
9992
|
if (checks.size < urls.length) warnings.push(`Stopped before checking ${urls.length - checks.size} selected external URL${urls.length - checks.size === 1 ? "" : "s"}.`);
|
|
8795
|
-
if (failedUrls) warnings.push(`${failedUrls} external URL
|
|
9993
|
+
if (failedUrls) warnings.push(`${failedUrls} external URL${failedUrls === 1 ? "" : "s"} could not be reached after bounded verification.`);
|
|
9994
|
+
if (outcomes.transient) warnings.push(`${outcomes.transient} external URL${outcomes.transient === 1 ? "" : "s"} returned inconsistent or temporary responses. Recheck ${outcomes.transient === 1 ? "it" : "them"} before changing the source page.`);
|
|
9995
|
+
if (outcomes["provider-blocked"]) warnings.push(`${outcomes["provider-blocked"]} external URL${outcomes["provider-blocked"] === 1 ? "" : "s"} blocked automated verification. Open ${outcomes["provider-blocked"] === 1 ? "it" : "them"} manually before changing the source page.`);
|
|
9996
|
+
if (outcomes["rate-limited"]) warnings.push(`${outcomes["rate-limited"]} external URL${outcomes["rate-limited"] === 1 ? "" : "s"} rate-limited verification. Recheck ${outcomes["rate-limited"] === 1 ? "it" : "them"} later.`);
|
|
9997
|
+
if (unresolvedMethodRejectedUrls) warnings.push(`${unresolvedMethodRejectedUrls} external URL${unresolvedMethodRejectedUrls === 1 ? "" : "s"} rejected both verification methods. Open ${unresolvedMethodRejectedUrls === 1 ? "it" : "them"} manually before changing the source page.`);
|
|
8796
9998
|
return {
|
|
8797
|
-
dataStatus: discoveredLinkOccurrences > retainedLinkOccurrences || deferredUrls > 0 || failedUrls > 0 ? "partial" : "complete",
|
|
9999
|
+
dataStatus: discoveredLinkOccurrences > retainedLinkOccurrences || deferredUrls > 0 || failedUrls > 0 || outcomes.transient > 0 || outcomes["provider-blocked"] > 0 || outcomes["rate-limited"] > 0 || unresolvedMethodRejectedUrls > 0 ? "partial" : "complete",
|
|
8798
10000
|
discoveredLinkOccurrences,
|
|
8799
10001
|
retainedUrls: candidates.length,
|
|
8800
10002
|
selectedUrls: urls.length,
|
|
@@ -8802,6 +10004,7 @@ async function verifyExternalLinks(input) {
|
|
|
8802
10004
|
failedUrls,
|
|
8803
10005
|
deferredUrls,
|
|
8804
10006
|
limit: EXTERNAL_LINK_SELECTION_LIMIT,
|
|
10007
|
+
outcomes,
|
|
8805
10008
|
warnings
|
|
8806
10009
|
};
|
|
8807
10010
|
}
|
|
@@ -9402,6 +10605,7 @@ async function crawlOne(url, opts = {}) {
|
|
|
9402
10605
|
sampleExternalLinks: uniqueExternalLinks.slice(0, 25),
|
|
9403
10606
|
internalAnchorSamples: anchorSamples(extracted.links, true),
|
|
9404
10607
|
externalAnchorSamples: anchorSamples(extracted.links, false),
|
|
10608
|
+
markdownAlternates: extracted.markdownAlternates,
|
|
9405
10609
|
schemaTypes: extracted.schemaTypes,
|
|
9406
10610
|
structuredDataFormats: extracted.structuredDataFormats ?? [],
|
|
9407
10611
|
googleRichResults: richResults.assessments,
|
|
@@ -9997,6 +11201,7 @@ function resolveCrawlSiteDependencies(dependencies = {}) {
|
|
|
9997
11201
|
queryPagesTopQueriesBatch: injectedMode === "batch" ? dependencies.queryPagesTopQueriesBatch : injectedMode ? void 0 : queryPagesTopQueriesBatch,
|
|
9998
11202
|
fetchLandingPageValues: dependencies.fetchLandingPageValues ?? fetchLandingPageValues,
|
|
9999
11203
|
landingValueForUrl: dependencies.landingValueForUrl ?? landingValueForUrl,
|
|
11204
|
+
collectAgentDiscovery: dependencies.collectAgentDiscovery ?? collectAgentDiscovery,
|
|
10000
11205
|
now: dependencies.now ?? (() => /* @__PURE__ */ new Date())
|
|
10001
11206
|
};
|
|
10002
11207
|
}
|
|
@@ -10322,6 +11527,11 @@ async function crawlSite(input, dependencies) {
|
|
|
10322
11527
|
const followLinks = config.mode === "site";
|
|
10323
11528
|
let queuedUrls = 0;
|
|
10324
11529
|
let skippedUrls = 0;
|
|
11530
|
+
const skipReasonCounts = {};
|
|
11531
|
+
const recordSkip = (reason) => {
|
|
11532
|
+
skippedUrls += 1;
|
|
11533
|
+
skipReasonCounts[reason] = (skipReasonCounts[reason] ?? 0) + 1;
|
|
11534
|
+
};
|
|
10325
11535
|
let robotsDeferredUrls = 0;
|
|
10326
11536
|
let originBackpressureSkippedUrls = 0;
|
|
10327
11537
|
let queueSafetySkippedUrls = 0;
|
|
@@ -10388,7 +11598,7 @@ async function crawlSite(input, dependencies) {
|
|
|
10388
11598
|
const enqueue = (value, depth) => {
|
|
10389
11599
|
const normalized = normalizeUrl$1(value, config.url);
|
|
10390
11600
|
if (!normalized) {
|
|
10391
|
-
|
|
11601
|
+
recordSkip("invalid-url");
|
|
10392
11602
|
emitStatus("url_skipped", {
|
|
10393
11603
|
url: value,
|
|
10394
11604
|
depth,
|
|
@@ -10405,7 +11615,7 @@ async function crawlSite(input, dependencies) {
|
|
|
10405
11615
|
return;
|
|
10406
11616
|
}
|
|
10407
11617
|
if (!sameOrigin(normalized, origin)) {
|
|
10408
|
-
|
|
11618
|
+
recordSkip("off-origin");
|
|
10409
11619
|
emitStatus("url_skipped", {
|
|
10410
11620
|
url: normalized,
|
|
10411
11621
|
depth,
|
|
@@ -10414,7 +11624,7 @@ async function crawlSite(input, dependencies) {
|
|
|
10414
11624
|
return;
|
|
10415
11625
|
}
|
|
10416
11626
|
if (isLikelyAsset(normalized)) {
|
|
10417
|
-
|
|
11627
|
+
recordSkip("asset-url");
|
|
10418
11628
|
emitStatus("url_skipped", {
|
|
10419
11629
|
url: normalized,
|
|
10420
11630
|
depth,
|
|
@@ -10423,7 +11633,7 @@ async function crawlSite(input, dependencies) {
|
|
|
10423
11633
|
return;
|
|
10424
11634
|
}
|
|
10425
11635
|
if (!passesFilters(normalized, config.include, config.exclude)) {
|
|
10426
|
-
|
|
11636
|
+
recordSkip("configured-exclusion");
|
|
10427
11637
|
emitStatus("url_skipped", {
|
|
10428
11638
|
url: normalized,
|
|
10429
11639
|
depth,
|
|
@@ -10432,7 +11642,7 @@ async function crawlSite(input, dependencies) {
|
|
|
10432
11642
|
return;
|
|
10433
11643
|
}
|
|
10434
11644
|
if (queue.length + visited.size + inFlight.size >= config.maxPages * 5) {
|
|
10435
|
-
|
|
11645
|
+
recordSkip("queue-safety-limit");
|
|
10436
11646
|
queueSafetySkippedUrls += 1;
|
|
10437
11647
|
emitStatus("url_skipped", {
|
|
10438
11648
|
url: normalized,
|
|
@@ -10541,7 +11751,7 @@ async function crawlSite(input, dependencies) {
|
|
|
10541
11751
|
});
|
|
10542
11752
|
requests.push(request);
|
|
10543
11753
|
if (request.outcome === "skipped") {
|
|
10544
|
-
|
|
11754
|
+
recordSkip(request.reason === "robots-disallowed" ? "robots-disallowed" : request.reason === "robots-deferred" ? "robots-uncertain" : "origin-backpressure");
|
|
10545
11755
|
if (request.reason === "robots-deferred") {
|
|
10546
11756
|
robotsDeferredUrls += 1;
|
|
10547
11757
|
warnings.push(`${task.url}: crawl deferred because robots.txt availability is unknown.`);
|
|
@@ -10646,7 +11856,7 @@ async function crawlSite(input, dependencies) {
|
|
|
10646
11856
|
page.declaredIndexability = "robots-blocked";
|
|
10647
11857
|
storeDocument([]);
|
|
10648
11858
|
warnings.push(`${page.url}: skipped because robots.txt disallows it`);
|
|
10649
|
-
|
|
11859
|
+
recordSkip("robots-disallowed");
|
|
10650
11860
|
emitStatus("page_skipped", {
|
|
10651
11861
|
url: page.url,
|
|
10652
11862
|
depth: task.depth,
|
|
@@ -10733,6 +11943,14 @@ async function crawlSite(input, dependencies) {
|
|
|
10733
11943
|
message: "Finished external link checks."
|
|
10734
11944
|
});
|
|
10735
11945
|
}
|
|
11946
|
+
const agentDiscovery = !cancelled && config.checkAgentDiscovery ? await deps.collectAgentDiscovery({
|
|
11947
|
+
startUrl: config.url,
|
|
11948
|
+
pages,
|
|
11949
|
+
timeoutMs: config.timeoutMs,
|
|
11950
|
+
fetch: deps.fetch,
|
|
11951
|
+
signal,
|
|
11952
|
+
concurrency: Math.min(config.concurrency, 4)
|
|
11953
|
+
}) : void 0;
|
|
10736
11954
|
for (const page of pages) {
|
|
10737
11955
|
if (!page.geo) continue;
|
|
10738
11956
|
page.geo = {
|
|
@@ -10759,6 +11977,7 @@ async function crawlSite(input, dependencies) {
|
|
|
10759
11977
|
},
|
|
10760
11978
|
...sitemapDiscovery ? { sitemapDiscovery } : {},
|
|
10761
11979
|
...externalLinkVerification ? { externalLinkVerification } : {},
|
|
11980
|
+
...agentDiscovery ? { agentDiscovery } : {},
|
|
10762
11981
|
status: reportStatus,
|
|
10763
11982
|
warnings,
|
|
10764
11983
|
caveats: crawlCaveats({
|
|
@@ -10773,6 +11992,7 @@ async function crawlSite(input, dependencies) {
|
|
|
10773
11992
|
queuedUrls,
|
|
10774
11993
|
crawledUrls: pages.length,
|
|
10775
11994
|
skippedUrls,
|
|
11995
|
+
skipReasonCounts,
|
|
10776
11996
|
failedUrls,
|
|
10777
11997
|
observedInternalLinks,
|
|
10778
11998
|
pageLimitReached
|
|
@@ -10967,7 +12187,7 @@ function completenessReasons(report) {
|
|
|
10967
12187
|
if (report.status !== "completed") reasons.push(`report-status-${report.status}`);
|
|
10968
12188
|
if (report.summary.pageLimitReached) reasons.push("crawl-page-limit-reached");
|
|
10969
12189
|
if (report.requestEvidenceStatus !== "available") reasons.push(`request-evidence-${report.requestEvidenceStatus}`);
|
|
10970
|
-
if (report.summary.
|
|
12190
|
+
if (report.summary.skippedUrlsByImpact.coverageAffecting > 0) reasons.push("coverage-affecting-urls-skipped");
|
|
10971
12191
|
if (report.summary.failedRequests > 0) reasons.push("requests-failed");
|
|
10972
12192
|
if (report.summary.abortedRequests > 0) reasons.push("requests-aborted");
|
|
10973
12193
|
if (report.summary.extractionFailures > 0) reasons.push("document-extraction-failed");
|
|
@@ -10987,6 +12207,8 @@ function snapshotCompleteness(report) {
|
|
|
10987
12207
|
requestEvidenceStatus: report.requestEvidenceStatus,
|
|
10988
12208
|
pageLimitReached: report.summary.pageLimitReached,
|
|
10989
12209
|
skippedUrls: report.summary.skippedUrls,
|
|
12210
|
+
skipReasons: report.summary.skipReasons,
|
|
12211
|
+
skippedUrlsByImpact: report.summary.skippedUrlsByImpact,
|
|
10990
12212
|
failedRequests: report.summary.failedRequests,
|
|
10991
12213
|
abortedRequests: report.summary.abortedRequests,
|
|
10992
12214
|
extractionFailures: report.summary.extractionFailures,
|
|
@@ -11619,6 +12841,169 @@ function explainOkfValidation(validation) {
|
|
|
11619
12841
|
};
|
|
11620
12842
|
}
|
|
11621
12843
|
//#endregion
|
|
12844
|
+
//#region packages/core/dist/analyze/crawler/agent-discovery-schema.js
|
|
12845
|
+
const agentRepresentationResponseSchema = z.object({
|
|
12846
|
+
requestedUrl: z.string().url(),
|
|
12847
|
+
finalUrl: z.string().url().optional(),
|
|
12848
|
+
status: z.number().int().optional(),
|
|
12849
|
+
contentType: z.string().optional(),
|
|
12850
|
+
bytes: z.number().int().nonnegative().optional(),
|
|
12851
|
+
sha256: z.string().optional(),
|
|
12852
|
+
canonicalUrl: z.string().url().optional(),
|
|
12853
|
+
alternateUrl: z.string().url().optional(),
|
|
12854
|
+
varyAccept: z.boolean(),
|
|
12855
|
+
markdownTokens: z.number().int().nonnegative().optional(),
|
|
12856
|
+
contentSignal: z.string().optional(),
|
|
12857
|
+
error: z.string().optional()
|
|
12858
|
+
});
|
|
12859
|
+
const agentDiscoverySchema = z.object({
|
|
12860
|
+
profile: z.literal("content"),
|
|
12861
|
+
profileApplicability: z.record(z.enum([
|
|
12862
|
+
"content",
|
|
12863
|
+
"api",
|
|
12864
|
+
"application",
|
|
12865
|
+
"commerce"
|
|
12866
|
+
]), z.object({
|
|
12867
|
+
status: z.enum(["evaluated", "notApplicable"]),
|
|
12868
|
+
reason: z.string()
|
|
12869
|
+
})),
|
|
12870
|
+
dataStatus: z.enum([
|
|
12871
|
+
"complete",
|
|
12872
|
+
"partial",
|
|
12873
|
+
"unavailable"
|
|
12874
|
+
]),
|
|
12875
|
+
markdownAlternates: z.object({
|
|
12876
|
+
eligibleHtmlPages: z.number().int().nonnegative(),
|
|
12877
|
+
advertisedPages: z.number().int().nonnegative(),
|
|
12878
|
+
evaluatedPages: z.number().int().nonnegative(),
|
|
12879
|
+
exactByteMatches: z.number().int().nonnegative(),
|
|
12880
|
+
stableResponses: z.number().int().nonnegative(),
|
|
12881
|
+
pages: z.array(z.object({
|
|
12882
|
+
htmlUrl: z.string().url(),
|
|
12883
|
+
advertisedUrls: z.array(z.string().url()),
|
|
12884
|
+
htmlAlternateUnique: z.boolean(),
|
|
12885
|
+
httpAlternateUrls: z.array(z.string().url()),
|
|
12886
|
+
explicit: agentRepresentationResponseSchema.optional(),
|
|
12887
|
+
negotiated: agentRepresentationResponseSchema.optional(),
|
|
12888
|
+
repeated: agentRepresentationResponseSchema.optional(),
|
|
12889
|
+
explicitMatchesNegotiated: z.boolean().nullable(),
|
|
12890
|
+
repeatedHashStable: z.boolean().nullable(),
|
|
12891
|
+
markdownCanonicalMatchesHtml: z.boolean().nullable(),
|
|
12892
|
+
quality: z.object({
|
|
12893
|
+
frontmatterTitle: z.boolean(),
|
|
12894
|
+
h1Count: z.number().int().nonnegative(),
|
|
12895
|
+
codeFenceBalanced: z.boolean(),
|
|
12896
|
+
tableRows: z.number().int().nonnegative(),
|
|
12897
|
+
links: z.number().int().nonnegative(),
|
|
12898
|
+
wordCount: z.number().int().nonnegative(),
|
|
12899
|
+
rawHtmlTags: z.number().int().nonnegative(),
|
|
12900
|
+
rawSvgTags: z.number().int().nonnegative(),
|
|
12901
|
+
rawScriptTags: z.number().int().nonnegative(),
|
|
12902
|
+
rawStyleTags: z.number().int().nonnegative(),
|
|
12903
|
+
suspiciousConcatenations: z.number().int().nonnegative(),
|
|
12904
|
+
repeatedLines: z.number().int().nonnegative(),
|
|
12905
|
+
sourceWordCount: z.number().int().nonnegative(),
|
|
12906
|
+
wordRetentionRatio: z.number().nonnegative().nullable(),
|
|
12907
|
+
introductoryCopyRetained: z.boolean().nullable(),
|
|
12908
|
+
navigationOnly: z.boolean()
|
|
12909
|
+
}).optional()
|
|
12910
|
+
}))
|
|
12911
|
+
}),
|
|
12912
|
+
contentNegotiation: z.object({
|
|
12913
|
+
qZeroHonoured: z.boolean().nullable(),
|
|
12914
|
+
qZeroStatus: z.number().int().optional(),
|
|
12915
|
+
qZeroContentType: z.string().optional(),
|
|
12916
|
+
error: z.string().optional()
|
|
12917
|
+
}),
|
|
12918
|
+
routeManifest: z.object({
|
|
12919
|
+
url: z.string().url(),
|
|
12920
|
+
status: z.number().int().optional(),
|
|
12921
|
+
valid: z.boolean(),
|
|
12922
|
+
declaredHtmlRoutes: z.array(z.string()),
|
|
12923
|
+
declaredMarkdownRoutes: z.array(z.string()),
|
|
12924
|
+
missingHtmlRoutes: z.array(z.string()),
|
|
12925
|
+
missingMarkdownRoutes: z.array(z.string()),
|
|
12926
|
+
orphanMarkdownRoutes: z.array(z.string()),
|
|
12927
|
+
error: z.string().optional()
|
|
12928
|
+
}),
|
|
12929
|
+
agentSkills: z.object({
|
|
12930
|
+
indexUrl: z.string().url(),
|
|
12931
|
+
status: z.number().int().optional(),
|
|
12932
|
+
contentType: z.string().optional(),
|
|
12933
|
+
cors: z.string().optional(),
|
|
12934
|
+
validIndex: z.boolean(),
|
|
12935
|
+
skills: z.array(z.object({
|
|
12936
|
+
name: z.string().optional(),
|
|
12937
|
+
url: z.string().url().optional(),
|
|
12938
|
+
status: z.number().int().optional(),
|
|
12939
|
+
contentType: z.string().optional(),
|
|
12940
|
+
declaredDigest: z.string().optional(),
|
|
12941
|
+
observedDigest: z.string().optional(),
|
|
12942
|
+
digestMatches: z.boolean().nullable(),
|
|
12943
|
+
frontmatterValid: z.boolean().nullable(),
|
|
12944
|
+
sameOrigin: z.boolean(),
|
|
12945
|
+
cors: z.string().optional(),
|
|
12946
|
+
error: z.string().optional()
|
|
12947
|
+
})),
|
|
12948
|
+
error: z.string().optional()
|
|
12949
|
+
}),
|
|
12950
|
+
llmsTxt: z.object({
|
|
12951
|
+
url: z.string().url(),
|
|
12952
|
+
exists: z.boolean(),
|
|
12953
|
+
status: z.number().int().optional(),
|
|
12954
|
+
contentType: z.string().optional(),
|
|
12955
|
+
bytes: z.number().int().nonnegative().optional(),
|
|
12956
|
+
sha256: z.string().optional(),
|
|
12957
|
+
repeatedHashStable: z.boolean().nullable(),
|
|
12958
|
+
headingCount: z.number().int().nonnegative(),
|
|
12959
|
+
totalParsedLinks: z.number().int().nonnegative(),
|
|
12960
|
+
linkLimitReached: z.boolean(),
|
|
12961
|
+
links: z.array(z.object({
|
|
12962
|
+
label: z.string(),
|
|
12963
|
+
url: z.string().url(),
|
|
12964
|
+
sameOrigin: z.boolean(),
|
|
12965
|
+
status: z.number().int().optional(),
|
|
12966
|
+
finalUrl: z.string().url().optional(),
|
|
12967
|
+
redirected: z.boolean(),
|
|
12968
|
+
indexableTarget: z.boolean().optional(),
|
|
12969
|
+
error: z.string().optional()
|
|
12970
|
+
})),
|
|
12971
|
+
invalidLinks: z.array(z.string()),
|
|
12972
|
+
duplicateLinks: z.array(z.string().url()),
|
|
12973
|
+
offSiteLinks: z.array(z.string().url()),
|
|
12974
|
+
redirectedLinks: z.array(z.string().url()),
|
|
12975
|
+
nonIndexableLinks: z.array(z.string().url()),
|
|
12976
|
+
missingCrawlRoutes: z.array(z.string().url()),
|
|
12977
|
+
oversized: z.boolean(),
|
|
12978
|
+
error: z.string().optional()
|
|
12979
|
+
}),
|
|
12980
|
+
contentSignals: z.object({
|
|
12981
|
+
htmlValues: z.array(z.string()),
|
|
12982
|
+
markdownValues: z.array(z.string()),
|
|
12983
|
+
missingHtmlPages: z.number().int().nonnegative(),
|
|
12984
|
+
missingMarkdownPages: z.number().int().nonnegative(),
|
|
12985
|
+
consistent: z.boolean().nullable()
|
|
12986
|
+
}),
|
|
12987
|
+
protocolVariants: z.object({
|
|
12988
|
+
http: z.object({
|
|
12989
|
+
url: z.string().url(),
|
|
12990
|
+
status: z.number().int().optional(),
|
|
12991
|
+
location: z.string().url().optional(),
|
|
12992
|
+
permanentRedirectToHttps: z.boolean().nullable(),
|
|
12993
|
+
error: z.string().optional()
|
|
12994
|
+
}),
|
|
12995
|
+
www: z.object({
|
|
12996
|
+
url: z.string().url(),
|
|
12997
|
+
status: z.number().int().optional(),
|
|
12998
|
+
location: z.string().url().optional(),
|
|
12999
|
+
redirectsToPreferredHost: z.boolean().nullable(),
|
|
13000
|
+
error: z.string().optional()
|
|
13001
|
+
}),
|
|
13002
|
+
hstsOnStartPage: z.boolean().nullable()
|
|
13003
|
+
}),
|
|
13004
|
+
warnings: z.array(z.string())
|
|
13005
|
+
});
|
|
13006
|
+
//#endregion
|
|
11622
13007
|
//#region packages/core/dist/analyze/crawler/schemas.js
|
|
11623
13008
|
const crawlerRuleSeveritySchema = z.enum([
|
|
11624
13009
|
"low",
|
|
@@ -12043,10 +13428,25 @@ const crawlPageSnapshotSchema = z.object({
|
|
|
12043
13428
|
href: z.string().url(),
|
|
12044
13429
|
text: z.string()
|
|
12045
13430
|
})).optional(),
|
|
13431
|
+
markdownAlternates: z.array(z.string().url()).optional(),
|
|
12046
13432
|
externalLinkChecks: z.array(z.object({
|
|
12047
13433
|
url: z.string().url(),
|
|
12048
13434
|
status: z.number().int().optional(),
|
|
12049
|
-
error: z.string().optional()
|
|
13435
|
+
error: z.string().optional(),
|
|
13436
|
+
state: z.enum([
|
|
13437
|
+
"available",
|
|
13438
|
+
"confirmed-broken",
|
|
13439
|
+
"transient",
|
|
13440
|
+
"provider-blocked",
|
|
13441
|
+
"rate-limited",
|
|
13442
|
+
"method-rejected",
|
|
13443
|
+
"unavailable"
|
|
13444
|
+
]).optional(),
|
|
13445
|
+
attempts: z.array(z.object({
|
|
13446
|
+
method: z.enum(["HEAD", "GET"]),
|
|
13447
|
+
status: z.number().int().optional(),
|
|
13448
|
+
error: z.string().optional()
|
|
13449
|
+
})).optional()
|
|
12050
13450
|
})).optional(),
|
|
12051
13451
|
schemaTypes: z.array(z.string()).optional(),
|
|
12052
13452
|
structuredDataFormats: z.array(z.enum([
|
|
@@ -12327,6 +13727,15 @@ const crawlExternalLinkVerificationSchema = z.object({
|
|
|
12327
13727
|
failedUrls: z.number().int().nonnegative(),
|
|
12328
13728
|
deferredUrls: z.number().int().nonnegative(),
|
|
12329
13729
|
limit: z.number().int().positive(),
|
|
13730
|
+
outcomes: z.record(z.enum([
|
|
13731
|
+
"available",
|
|
13732
|
+
"confirmed-broken",
|
|
13733
|
+
"transient",
|
|
13734
|
+
"provider-blocked",
|
|
13735
|
+
"rate-limited",
|
|
13736
|
+
"method-rejected",
|
|
13737
|
+
"unavailable"
|
|
13738
|
+
]), z.number().int().nonnegative()).optional(),
|
|
12330
13739
|
warnings: z.array(z.string())
|
|
12331
13740
|
});
|
|
12332
13741
|
const crawlConfigSchema = z.object({
|
|
@@ -12347,6 +13756,7 @@ const crawlConfigSchema = z.object({
|
|
|
12347
13756
|
respectRobots: z.boolean(),
|
|
12348
13757
|
useSitemap: z.boolean(),
|
|
12349
13758
|
checkExternal: z.boolean(),
|
|
13759
|
+
checkAgentDiscovery: z.boolean(),
|
|
12350
13760
|
js: z.union([z.enum([
|
|
12351
13761
|
"auto",
|
|
12352
13762
|
"on",
|
|
@@ -12377,6 +13787,15 @@ const crawlReportSummarySchema = z.object({
|
|
|
12377
13787
|
queuedUrls: z.number().int(),
|
|
12378
13788
|
crawledUrls: z.number().int(),
|
|
12379
13789
|
skippedUrls: z.number().int(),
|
|
13790
|
+
skipReasons: z.array(z.object({
|
|
13791
|
+
reason: z.enum(CRAWL_SKIP_REASONS),
|
|
13792
|
+
impact: z.enum(["coverage-affecting", "non-impacting"]),
|
|
13793
|
+
count: z.number().int().positive()
|
|
13794
|
+
})),
|
|
13795
|
+
skippedUrlsByImpact: z.object({
|
|
13796
|
+
coverageAffecting: z.number().int().nonnegative(),
|
|
13797
|
+
nonImpacting: z.number().int().nonnegative()
|
|
13798
|
+
}),
|
|
12380
13799
|
failedUrls: z.number().int(),
|
|
12381
13800
|
observedInternalLinks: z.number().int(),
|
|
12382
13801
|
pageLimitReached: z.boolean().default(false),
|
|
@@ -12459,6 +13878,7 @@ const crawlReportBaseSchema = z.object({
|
|
|
12459
13878
|
ai: crawlAiSignalsSchema.optional(),
|
|
12460
13879
|
sitemapDiscovery: crawlSitemapDiscoverySchema.optional(),
|
|
12461
13880
|
externalLinkVerification: crawlExternalLinkVerificationSchema.optional(),
|
|
13881
|
+
agentDiscovery: agentDiscoverySchema.optional(),
|
|
12462
13882
|
warnings: z.array(z.string()),
|
|
12463
13883
|
caveats: z.array(z.string())
|
|
12464
13884
|
});
|
|
@@ -18184,7 +19604,7 @@ function crawlInput(input) {
|
|
|
18184
19604
|
return {
|
|
18185
19605
|
pages,
|
|
18186
19606
|
evidence: sourceEvidence({
|
|
18187
|
-
completeness: report ? report.summary.pageLimitReached ? "truncated" : report.status === "
|
|
19607
|
+
completeness: report ? report.summary.pageLimitReached ? "truncated" : report.status === "partial" || report.summary.skippedUrlsByImpact.coverageAffecting > 0 ? "partial" : report.status === "completed" ? "complete" : "unknown" : "completeness" in input ? input.completeness : "unknown",
|
|
18188
19608
|
rowsReceived: pages.length,
|
|
18189
19609
|
validUrls: urls.length,
|
|
18190
19610
|
uniqueUrls: uniqueUrls.size,
|
|
@@ -23236,14 +24656,14 @@ function workflowReport(input) {
|
|
|
23236
24656
|
async function diagnosePropertyWorkflow(input) {
|
|
23237
24657
|
const narrative = await reportNarrative(input);
|
|
23238
24658
|
const skippedCount = narrative.diagnosis.skippedSections?.length ?? 0;
|
|
23239
|
-
const stepSummary = narrative.dataStatus === "complete" ? "Generated diagnosis, movement, change, and monitoring narrative." : skippedCount ? `Generated ${narrative.dataStatus === "unavailable" ? "an" : "a"} ${narrative.dataStatus} diagnosis; ${skippedCount} sections were unavailable.` : "Generated a partial diagnosis because one or more source datasets were incomplete.";
|
|
24659
|
+
const stepSummary = input.skipSearchData ? "Skipped provider-backed search diagnosis because no Search Console property was selected." : narrative.dataStatus === "complete" ? "Generated diagnosis, movement, change, and monitoring narrative." : skippedCount ? `Generated ${narrative.dataStatus === "unavailable" ? "an" : "a"} ${narrative.dataStatus} diagnosis; ${skippedCount} sections were unavailable.` : "Generated a partial diagnosis because one or more source datasets were incomplete.";
|
|
23240
24660
|
return workflowReport({
|
|
23241
24661
|
workflow: "diagnose-property",
|
|
23242
24662
|
site: input.site,
|
|
23243
24663
|
summary: narrative.headline,
|
|
23244
24664
|
steps: [{
|
|
23245
24665
|
tool: "seo_report_narrative",
|
|
23246
|
-
status: "completed",
|
|
24666
|
+
status: input.skipSearchData ? "skipped" : "completed",
|
|
23247
24667
|
summary: stepSummary
|
|
23248
24668
|
}],
|
|
23249
24669
|
actions: narrative.priorities.map((priority) => ({
|
|
@@ -24780,6 +26200,9 @@ function resolveSavedReportAlias(input) {
|
|
|
24780
26200
|
}
|
|
24781
26201
|
return loadCrawlReport(input.value);
|
|
24782
26202
|
}
|
|
26203
|
+
function assertExclusiveReportInput(url, reportId) {
|
|
26204
|
+
if (url && reportId) throw new SeoError("INVALID_INPUT", "Use either url or reportId, not both.");
|
|
26205
|
+
}
|
|
24783
26206
|
function registerCrawlerTools(server) {
|
|
24784
26207
|
server.registerTool("seo_crawl_site", {
|
|
24785
26208
|
description: "Crawl a site and run technical SEO checks. The compact result separates prioritised fixes from review observations. Set includePages/includeIssues for raw data.",
|
|
@@ -24896,6 +26319,7 @@ function registerCrawlerTools(server) {
|
|
|
24896
26319
|
}
|
|
24897
26320
|
}, async ({ url, reportId, site, maxPages, maxDepth, fetchIntervalCap, fetchIntervalMs, refresh, category, limit }) => {
|
|
24898
26321
|
try {
|
|
26322
|
+
assertExclusiveReportInput(url, reportId);
|
|
24899
26323
|
const report = url ? await crawlSite({
|
|
24900
26324
|
url,
|
|
24901
26325
|
site,
|
|
@@ -25022,6 +26446,40 @@ function registerCrawlerTools(server) {
|
|
|
25022
26446
|
return toolError(error);
|
|
25023
26447
|
}
|
|
25024
26448
|
});
|
|
26449
|
+
server.registerTool("seo_agent_readiness", {
|
|
26450
|
+
description: "Check a content site for stable Markdown alternatives, agent discovery, crawler access, and identity evidence without an aggregate score.",
|
|
26451
|
+
inputSchema: {
|
|
26452
|
+
url: z$1.string().url().optional(),
|
|
26453
|
+
reportId: z$1.string().optional(),
|
|
26454
|
+
site: z$1.string().optional(),
|
|
26455
|
+
maxPages: z$1.number().int().positive().optional(),
|
|
26456
|
+
fetchIntervalCap: z$1.number().int().positive().optional(),
|
|
26457
|
+
fetchIntervalMs: z$1.number().int().positive().optional(),
|
|
26458
|
+
refresh: z$1.boolean().optional()
|
|
26459
|
+
}
|
|
26460
|
+
}, async ({ url, reportId, site, maxPages, fetchIntervalCap, fetchIntervalMs, refresh }) => {
|
|
26461
|
+
try {
|
|
26462
|
+
assertExclusiveReportInput(url, reportId);
|
|
26463
|
+
const report = url ? await crawlSite({
|
|
26464
|
+
url,
|
|
26465
|
+
site,
|
|
26466
|
+
maxPages,
|
|
26467
|
+
refresh,
|
|
26468
|
+
checkAgentDiscovery: true,
|
|
26469
|
+
useSitemap: true,
|
|
26470
|
+
checkExternal: false,
|
|
26471
|
+
fetchRate: fetchRateInput({
|
|
26472
|
+
fetchIntervalCap,
|
|
26473
|
+
fetchIntervalMs
|
|
26474
|
+
})
|
|
26475
|
+
}) : reportId ? loadCrawlReport(reportId) : latestCrawlReport(site);
|
|
26476
|
+
if (!report) return toolError("No crawl report found. Pass url, reportId, or run seo_crawl_site with saveReport first.");
|
|
26477
|
+
const readiness = agentReadiness(report);
|
|
26478
|
+
return toolSuccess(readiness.headline, readiness);
|
|
26479
|
+
} catch (error) {
|
|
26480
|
+
return toolError(error);
|
|
26481
|
+
}
|
|
26482
|
+
});
|
|
25025
26483
|
server.registerTool("seo_ai_readiness", {
|
|
25026
26484
|
description: "Return AI-search technical evidence and optional observations without an aggregate score or citation prediction.",
|
|
25027
26485
|
inputSchema: {
|
|
@@ -26206,6 +27664,39 @@ const REPORT_DEPTH = {
|
|
|
26206
27664
|
}
|
|
26207
27665
|
]
|
|
26208
27666
|
},
|
|
27667
|
+
"agent-readiness": {
|
|
27668
|
+
readOrder: [
|
|
27669
|
+
"profile, profileApplicability, dataStatus, and crawl caveats",
|
|
27670
|
+
"summary and topActions",
|
|
27671
|
+
"HTML and Markdown representation checks with affected URLs",
|
|
27672
|
+
"Agent Skills, llms.txt, route manifest, crawler access, and identity evidence"
|
|
27673
|
+
],
|
|
27674
|
+
doNotClaim: [
|
|
27675
|
+
"A clean content profile is not proof of indexing, rankings, AI mentions, citations, or selection.",
|
|
27676
|
+
"API, application, and commerce checks marked notApplicable were outside this content-site run. They did not fail.",
|
|
27677
|
+
"An allowed crawler token does not prove that a service fetched or used the page.",
|
|
27678
|
+
"Optional discovery files are not Google ranking requirements."
|
|
27679
|
+
],
|
|
27680
|
+
verify: "Run the same content profile against the same public route scope after deployment and confirm each failed or warning check changed.",
|
|
27681
|
+
related: [
|
|
27682
|
+
{
|
|
27683
|
+
id: "ai-readiness",
|
|
27684
|
+
reason: "Checks AI search technical eligibility."
|
|
27685
|
+
},
|
|
27686
|
+
{
|
|
27687
|
+
id: "llms-txt-audit",
|
|
27688
|
+
reason: "Inspects llms.txt in more detail."
|
|
27689
|
+
},
|
|
27690
|
+
{
|
|
27691
|
+
id: "entity-readiness",
|
|
27692
|
+
reason: "Reviews identity evidence."
|
|
27693
|
+
},
|
|
27694
|
+
{
|
|
27695
|
+
id: "site-crawl",
|
|
27696
|
+
reason: "Creates a reusable crawl baseline."
|
|
27697
|
+
}
|
|
27698
|
+
]
|
|
27699
|
+
},
|
|
26209
27700
|
"ai-readiness": {
|
|
26210
27701
|
readOrder: [
|
|
26211
27702
|
"dataStatus and crawl caveats",
|
|
@@ -27575,6 +29066,13 @@ const REPORT_GUIDANCE = {
|
|
|
27575
29066
|
avoidWhen: ["You have not run or selected a crawl yet.", "You need an overview of the most important crawl findings first."],
|
|
27576
29067
|
outcome: "A focused list of affected URLs with the evidence for each match."
|
|
27577
29068
|
},
|
|
29069
|
+
"agent-readiness": {
|
|
29070
|
+
name: "AI agent readiness",
|
|
29071
|
+
description: "Check whether a content site gives agents clean, stable, machine-readable pages and discovery files.",
|
|
29072
|
+
useWhen: ["You want to test Markdown alternatives, content negotiation, Agent Skills, llms.txt, identity, and crawler access together.", "You need evidence that the agent-facing version of a content site matches its public HTML routes."],
|
|
29073
|
+
avoidWhen: ["You need an AI visibility, mention, citation, or ranking report.", "You are auditing a public API, interactive application, or commerce flow rather than a content site."],
|
|
29074
|
+
outcome: "An unscored content-site assessment with failed routes, evidence, and clear next actions."
|
|
29075
|
+
},
|
|
27578
29076
|
"ai-readiness": {
|
|
27579
29077
|
name: "AI search technical readiness",
|
|
27580
29078
|
description: "Check whether crawl, index, or snippet controls could block pages from Google AI search features.",
|
|
@@ -28348,6 +29846,7 @@ const reportGroups = [
|
|
|
28348
29846
|
register: registerCrawlerTools,
|
|
28349
29847
|
names: [
|
|
28350
29848
|
"seo_affected_urls",
|
|
29849
|
+
"seo_agent_readiness",
|
|
28351
29850
|
"seo_ai_readiness",
|
|
28352
29851
|
"seo_audit_urls",
|
|
28353
29852
|
"seo_compare_crawl_reports",
|