seo 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +9 -4
- package/README.md +2 -2
- package/dist/cli.js +79 -71
- package/dist/cli.js.map +1 -1
- package/dist/dist-CpTkpH4F.js +713 -0
- package/dist/dist-CpTkpH4F.js.map +1 -0
- package/dist/{dist-eiewVfiS.js → dist-DUfbYI4E.js} +1 -1
- package/dist/index.d.ts +2544 -41
- package/dist/index.js +1573 -180
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +1637 -164
- package/dist/mcp.js.map +1 -1
- package/dist/{package-BTOI9tde.js → package-uOaBUbds.js} +2 -2
- package/dist/{package-BTOI9tde.js.map → package-uOaBUbds.js.map} +1 -1
- package/evals/agent-readiness.json +58 -0
- package/package.json +2 -2
- package/skills/seo/SKILL.md +3 -2
- package/dist/dist-CTBhWxn6.js +0 -713
- package/dist/dist-CTBhWxn6.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { n as load, t as version } from "./package-
|
|
2
|
+
import { n as load, t as version } from "./package-uOaBUbds.js";
|
|
3
3
|
import { Headers, fetch as fetch$1 } from "undici";
|
|
4
4
|
import { chmodSync, createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
5
5
|
import { homedir } from "node:os";
|
|
@@ -857,10 +857,8 @@ async function createGoogleAccessTokenClient() {
|
|
|
857
857
|
}
|
|
858
858
|
async function authStatus() {
|
|
859
859
|
const status = getAuthModeStatus();
|
|
860
|
-
const tokens = await readTokens();
|
|
861
860
|
const serviceAccount = getServiceAccountStatus();
|
|
862
861
|
if (serviceAccount.configured || serviceAccount.error) return {
|
|
863
|
-
tokens,
|
|
864
862
|
configured: serviceAccount.configured,
|
|
865
863
|
sharedConfigured: status.sharedConfigured,
|
|
866
864
|
byoConfigured: status.byoConfigured,
|
|
@@ -868,6 +866,7 @@ async function authStatus() {
|
|
|
868
866
|
identity: serviceAccount.identity,
|
|
869
867
|
serviceAccount
|
|
870
868
|
};
|
|
869
|
+
const tokens = await readTokens();
|
|
871
870
|
return {
|
|
872
871
|
tokens,
|
|
873
872
|
configured: Boolean(getClientConfig(tokens?.client_source)),
|
|
@@ -2219,7 +2218,7 @@ function effectiveSnippetControl(input) {
|
|
|
2219
2218
|
}
|
|
2220
2219
|
//#endregion
|
|
2221
2220
|
//#region packages/core/src/extract/canonical.ts
|
|
2222
|
-
function headerValue$
|
|
2221
|
+
function headerValue$4(headers, name) {
|
|
2223
2222
|
const lower = name.toLowerCase();
|
|
2224
2223
|
return Object.entries(headers).find(([key]) => key.toLowerCase() === lower)?.[1];
|
|
2225
2224
|
}
|
|
@@ -2303,7 +2302,7 @@ function parameter(value, name) {
|
|
|
2303
2302
|
return match?.[1] ?? match?.[2] ?? match?.[3];
|
|
2304
2303
|
}
|
|
2305
2304
|
function headerCandidates(headers, baseUrl) {
|
|
2306
|
-
const link = headerValue$
|
|
2305
|
+
const link = headerValue$4(headers, "link");
|
|
2307
2306
|
if (!link) return [];
|
|
2308
2307
|
const candidates = [];
|
|
2309
2308
|
for (const item of splitLinkHeader(link)) {
|
|
@@ -2972,7 +2971,7 @@ function safeText$1(value) {
|
|
|
2972
2971
|
const trimmed = value?.replace(/\s+/g, " ").trim();
|
|
2973
2972
|
return trimmed ? trimmed : void 0;
|
|
2974
2973
|
}
|
|
2975
|
-
function headerValue$
|
|
2974
|
+
function headerValue$3(headers, name) {
|
|
2976
2975
|
const target = name.toLowerCase();
|
|
2977
2976
|
return Object.entries(headers).find(([key]) => key.toLowerCase() === target)?.[1];
|
|
2978
2977
|
}
|
|
@@ -3071,6 +3070,7 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
3071
3070
|
});
|
|
3072
3071
|
else invalidHreflangCount += 1;
|
|
3073
3072
|
}
|
|
3073
|
+
const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
|
|
3074
3074
|
const openGraph = Object.fromEntries($("meta[property^=\"og:\"]").toArray().map((element) => [$(element).attr("property") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
3075
3075
|
const twitter = Object.fromEntries($("meta[name^=\"twitter:\"]").toArray().map((element) => [$(element).attr("name") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
3076
3076
|
const structuredData = extractStructuredData($, fetchResult.finalUrl);
|
|
@@ -3118,13 +3118,14 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
3118
3118
|
title: safeText$1($("title").first().text()),
|
|
3119
3119
|
metaDescription: safeText$1($("meta[name=\"description\"]").attr("content")),
|
|
3120
3120
|
metaRobots,
|
|
3121
|
-
xRobotsTag: safeText$1(headerValue$
|
|
3121
|
+
xRobotsTag: safeText$1(headerValue$3(fetchResult.headers, "x-robots-tag")),
|
|
3122
3122
|
canonical: canonicalEvidence.selectedRaw,
|
|
3123
3123
|
canonicalEvidence,
|
|
3124
3124
|
lang: safeText$1($("html").attr("lang")),
|
|
3125
3125
|
hasViewport: Boolean($("meta[name=\"viewport\"]").attr("content")),
|
|
3126
3126
|
headings,
|
|
3127
3127
|
links,
|
|
3128
|
+
markdownAlternates,
|
|
3128
3129
|
hreflang,
|
|
3129
3130
|
jsonLd: structuredData.jsonLd,
|
|
3130
3131
|
invalidJsonLdCount: structuredData.invalidJsonLdSamples.length,
|
|
@@ -3929,7 +3930,7 @@ function safeText(value) {
|
|
|
3929
3930
|
const normalized = value?.replace(/\s+/g, " ").trim();
|
|
3930
3931
|
return normalized ? normalized.slice(0, MAX_TEXT_VALUE_LENGTH) : void 0;
|
|
3931
3932
|
}
|
|
3932
|
-
function headerValue$
|
|
3933
|
+
function headerValue$2(headers, name) {
|
|
3933
3934
|
const expected = name.toLowerCase();
|
|
3934
3935
|
return Object.entries(headers).find(([key]) => key.toLowerCase() === expected)?.[1];
|
|
3935
3936
|
}
|
|
@@ -3982,7 +3983,7 @@ function renderingDocumentSnapshot(result) {
|
|
|
3982
3983
|
const structuredData = extractStructuredData($, result.finalUrl);
|
|
3983
3984
|
const metaRobots = normalizedDirectiveValue([metaValue($, "robots")]);
|
|
3984
3985
|
const googlebotRobots = normalizedDirectiveValue([metaValue($, "googlebot")]);
|
|
3985
|
-
const httpRobots = normalizedDirectiveValue([headerValue$
|
|
3986
|
+
const httpRobots = normalizedDirectiveValue([headerValue$2(result.headers, "x-robots-tag")]);
|
|
3986
3987
|
const headings = $("h1, h2, h3, h4, h5, h6").toArray().map((element) => ({
|
|
3987
3988
|
level: Number.parseInt(element.tagName.slice(1), 10),
|
|
3988
3989
|
text: safeText($(element).text())
|
|
@@ -6627,24 +6628,1078 @@ function selectAffectedUrls(report, filters = {}) {
|
|
|
6627
6628
|
const limit = filters.limit ?? 100;
|
|
6628
6629
|
const selected = matches.slice(0, limit);
|
|
6629
6630
|
return {
|
|
6630
|
-
affectedUrls: selected,
|
|
6631
|
-
selection: {
|
|
6632
|
-
totalMatchedRows: matches.length,
|
|
6633
|
-
returnedRows: selected.length,
|
|
6634
|
-
limit,
|
|
6635
|
-
truncated: selected.length < matches.length
|
|
6631
|
+
affectedUrls: selected,
|
|
6632
|
+
selection: {
|
|
6633
|
+
totalMatchedRows: matches.length,
|
|
6634
|
+
returnedRows: selected.length,
|
|
6635
|
+
limit,
|
|
6636
|
+
truncated: selected.length < matches.length
|
|
6637
|
+
}
|
|
6638
|
+
};
|
|
6639
|
+
}
|
|
6640
|
+
function matchesFilters(issue, filters) {
|
|
6641
|
+
if (filters.ruleId && issue.ruleId !== filters.ruleId) return false;
|
|
6642
|
+
if (filters.category && issue.category !== filters.category) return false;
|
|
6643
|
+
if (filters.severity && issue.severity !== filters.severity) return false;
|
|
6644
|
+
return true;
|
|
6645
|
+
}
|
|
6646
|
+
//#endregion
|
|
6647
|
+
//#region packages/core/src/analyze/crawler/agent-discovery.ts
|
|
6648
|
+
const MAX_BODY_BYTES = 2e6;
|
|
6649
|
+
const MAX_LLMS_LINKS = 100;
|
|
6650
|
+
const MAX_CURATED_LLMS_BYTES = 1e5;
|
|
6651
|
+
const MAX_SKILLS = 25;
|
|
6652
|
+
function sha256(value) {
|
|
6653
|
+
return createHash("sha256").update(value).digest("hex");
|
|
6654
|
+
}
|
|
6655
|
+
function headerValue$1(headers, name) {
|
|
6656
|
+
if (!headers) return void 0;
|
|
6657
|
+
const target = name.toLowerCase();
|
|
6658
|
+
return Object.entries(headers).find(([key]) => key.toLowerCase() === target)?.[1];
|
|
6659
|
+
}
|
|
6660
|
+
function linkEntries(value, base) {
|
|
6661
|
+
if (!value) return [];
|
|
6662
|
+
return value.split(/,(?=\s*<)/u).map((entry) => {
|
|
6663
|
+
const match = entry.match(/^\s*<([^>]+)>\s*(.*)$/u);
|
|
6664
|
+
if (!match?.[1]) return void 0;
|
|
6665
|
+
try {
|
|
6666
|
+
const url = new URL(match[1], base).toString();
|
|
6667
|
+
const parameters = match[2] ?? "";
|
|
6668
|
+
const rel = parameters.match(/(?:^|;)\s*rel=(?:"([^"]+)"|([^;\s]+))/iu);
|
|
6669
|
+
const type = parameters.match(/(?:^|;)\s*type=(?:"([^"]+)"|([^;\s]+))/iu);
|
|
6670
|
+
return {
|
|
6671
|
+
url,
|
|
6672
|
+
rel: (rel?.[1] ?? rel?.[2] ?? "").toLowerCase().split(/\s+/u),
|
|
6673
|
+
type: (type?.[1] ?? type?.[2] ?? "").toLowerCase()
|
|
6674
|
+
};
|
|
6675
|
+
} catch {
|
|
6676
|
+
return;
|
|
6677
|
+
}
|
|
6678
|
+
}).filter((entry) => Boolean(entry));
|
|
6679
|
+
}
|
|
6680
|
+
function normalizedDocumentUrl(value) {
|
|
6681
|
+
const url = new URL(value);
|
|
6682
|
+
url.hash = "";
|
|
6683
|
+
if (url.pathname !== "/") url.pathname = url.pathname.replace(/\/$/u, "");
|
|
6684
|
+
return url.toString();
|
|
6685
|
+
}
|
|
6686
|
+
function sameDocument(left, right) {
|
|
6687
|
+
if (!left || !right) return null;
|
|
6688
|
+
try {
|
|
6689
|
+
return normalizedDocumentUrl(left) === normalizedDocumentUrl(right);
|
|
6690
|
+
} catch {
|
|
6691
|
+
return false;
|
|
6692
|
+
}
|
|
6693
|
+
}
|
|
6694
|
+
function bodyWordCount(value) {
|
|
6695
|
+
return value.replace(/^---[\s\S]*?---\s*/u, "").replace(/```[\s\S]*?```/gu, " ").replace(/[#>*_`[\]()|:-]/gu, " ").split(/\s+/u).filter(Boolean).length;
|
|
6696
|
+
}
|
|
6697
|
+
function responseIsNoindex(response, body) {
|
|
6698
|
+
if ((response.headers.get("x-robots-tag") ?? "").toLowerCase().includes("noindex")) return true;
|
|
6699
|
+
return [...body.matchAll(/<meta\s+[^>]*>/giu)].some((match) => {
|
|
6700
|
+
const tag = match[0];
|
|
6701
|
+
const name = tag.match(/\bname\s*=\s*["']([^"']+)["']/iu)?.[1];
|
|
6702
|
+
const content = tag.match(/\bcontent\s*=\s*["']([^"']+)["']/iu)?.[1];
|
|
6703
|
+
return name?.toLowerCase() === "robots" && /\bnoindex\b/iu.test(content ?? "");
|
|
6704
|
+
});
|
|
6705
|
+
}
|
|
6706
|
+
function markdownQuality(markdown, page) {
|
|
6707
|
+
const introProbe = (page.contentSample?.replace(/\s+/gu, " ").trim())?.split(" ").slice(0, 10).join(" ");
|
|
6708
|
+
const normalizedMarkdown = markdown.replace(/\s+/gu, " ");
|
|
6709
|
+
const codeFences = markdown.match(/^```/gmu)?.length ?? 0;
|
|
6710
|
+
const repeatedLines = [...markdown.matchAll(/^(.{12,})$/gmu)].map((match) => match[1]?.trim()).filter((line) => Boolean(line)).filter((line, index, lines) => lines.indexOf(line) !== index).length;
|
|
6711
|
+
const wordCount = bodyWordCount(markdown);
|
|
6712
|
+
const sourceWordCount = page.wordCount ?? 0;
|
|
6713
|
+
return {
|
|
6714
|
+
frontmatterTitle: /^---[\s\S]*?^title:\s*.+$/mu.test(markdown),
|
|
6715
|
+
h1Count: markdown.match(/^#\s+\S.+$/gmu)?.length ?? 0,
|
|
6716
|
+
codeFenceBalanced: codeFences % 2 === 0,
|
|
6717
|
+
tableRows: markdown.match(/^\s*\|.*\|\s*$/gmu)?.length ?? 0,
|
|
6718
|
+
links: markdown.match(/\[[^\]]+\]\([^)]+\)/gu)?.length ?? 0,
|
|
6719
|
+
wordCount,
|
|
6720
|
+
rawHtmlTags: markdown.match(/<(?:div|span|section|article|nav|button)\b/giu)?.length ?? 0,
|
|
6721
|
+
rawSvgTags: markdown.match(/<svg\b/giu)?.length ?? 0,
|
|
6722
|
+
rawScriptTags: markdown.match(/<script\b/giu)?.length ?? 0,
|
|
6723
|
+
rawStyleTags: markdown.match(/<style\b/giu)?.length ?? 0,
|
|
6724
|
+
suspiciousConcatenations: markdown.match(/\b[a-z]{4,}[A-Z][A-Za-z]{3,}\b/gu)?.length ?? 0,
|
|
6725
|
+
repeatedLines,
|
|
6726
|
+
sourceWordCount,
|
|
6727
|
+
wordRetentionRatio: sourceWordCount > 0 ? Math.round(wordCount / sourceWordCount * 1e3) / 1e3 : null,
|
|
6728
|
+
introductoryCopyRetained: introProbe ? normalizedMarkdown.includes(introProbe) : null,
|
|
6729
|
+
navigationOnly: bodyWordCount(markdown) < 25
|
|
6730
|
+
};
|
|
6731
|
+
}
|
|
6732
|
+
function combinedSignal(timeoutMs, signal) {
|
|
6733
|
+
const controller = new AbortController();
|
|
6734
|
+
const abort = () => controller.abort();
|
|
6735
|
+
const timer = setTimeout(abort, timeoutMs);
|
|
6736
|
+
if (signal?.aborted) controller.abort();
|
|
6737
|
+
else signal?.addEventListener("abort", abort, { once: true });
|
|
6738
|
+
return {
|
|
6739
|
+
signal: controller.signal,
|
|
6740
|
+
cleanup: () => {
|
|
6741
|
+
clearTimeout(timer);
|
|
6742
|
+
signal?.removeEventListener("abort", abort);
|
|
6743
|
+
}
|
|
6744
|
+
};
|
|
6745
|
+
}
|
|
6746
|
+
async function readBoundedText(response) {
|
|
6747
|
+
if (!response.body) return "";
|
|
6748
|
+
const reader = response.body.getReader();
|
|
6749
|
+
const chunks = [];
|
|
6750
|
+
let size = 0;
|
|
6751
|
+
try {
|
|
6752
|
+
while (true) {
|
|
6753
|
+
const result = await reader.read();
|
|
6754
|
+
if (result.done) break;
|
|
6755
|
+
size += result.value.byteLength;
|
|
6756
|
+
if (size > MAX_BODY_BYTES) throw new Error(`Response exceeds ${MAX_BODY_BYTES} bytes.`);
|
|
6757
|
+
chunks.push(result.value);
|
|
6758
|
+
}
|
|
6759
|
+
} finally {
|
|
6760
|
+
await reader.cancel().catch(() => void 0);
|
|
6761
|
+
}
|
|
6762
|
+
return Buffer.concat(chunks).toString("utf8");
|
|
6763
|
+
}
|
|
6764
|
+
async function fetchRepresentation(input) {
|
|
6765
|
+
const controller = combinedSignal(input.timeoutMs, input.signal);
|
|
6766
|
+
try {
|
|
6767
|
+
const response = await input.fetch(input.url, {
|
|
6768
|
+
profile: "bot",
|
|
6769
|
+
redirect: "follow",
|
|
6770
|
+
headers: { accept: input.accept },
|
|
6771
|
+
signal: controller.signal
|
|
6772
|
+
});
|
|
6773
|
+
const body = await readBoundedText(response);
|
|
6774
|
+
const link = linkEntries(response.headers.get("link") ?? void 0, input.url);
|
|
6775
|
+
const tokens = Number(response.headers.get("x-markdown-tokens"));
|
|
6776
|
+
return {
|
|
6777
|
+
observation: {
|
|
6778
|
+
requestedUrl: input.url,
|
|
6779
|
+
finalUrl: response.url || input.url,
|
|
6780
|
+
status: response.status,
|
|
6781
|
+
contentType: response.headers.get("content-type") ?? void 0,
|
|
6782
|
+
bytes: Buffer.byteLength(body),
|
|
6783
|
+
sha256: sha256(body),
|
|
6784
|
+
canonicalUrl: link.find((entry) => entry.rel.includes("canonical"))?.url,
|
|
6785
|
+
alternateUrl: link.find((entry) => entry.rel.includes("alternate") && entry.type === "text/markdown")?.url,
|
|
6786
|
+
varyAccept: (response.headers.get("vary") ?? "").split(",").some((value) => value.trim().toLowerCase() === "accept"),
|
|
6787
|
+
...Number.isInteger(tokens) && tokens >= 0 ? { markdownTokens: tokens } : {},
|
|
6788
|
+
contentSignal: response.headers.get("content-signal") ?? void 0
|
|
6789
|
+
},
|
|
6790
|
+
body
|
|
6791
|
+
};
|
|
6792
|
+
} catch (error) {
|
|
6793
|
+
return { observation: {
|
|
6794
|
+
requestedUrl: input.url,
|
|
6795
|
+
varyAccept: false,
|
|
6796
|
+
error: error instanceof Error ? error.message : String(error)
|
|
6797
|
+
} };
|
|
6798
|
+
} finally {
|
|
6799
|
+
controller.cleanup();
|
|
6800
|
+
}
|
|
6801
|
+
}
|
|
6802
|
+
async function inspectMarkdownPage(input) {
|
|
6803
|
+
const advertisedUrls = [...new Set(input.page.markdownAlternates ?? [])].sort();
|
|
6804
|
+
const httpAlternateUrls = linkEntries(headerValue$1(input.page.responseHeaders, "link"), input.page.finalUrl).filter((entry) => entry.rel.includes("alternate") && entry.type === "text/markdown").map((entry) => entry.url).sort();
|
|
6805
|
+
const observation = {
|
|
6806
|
+
htmlUrl: input.page.finalUrl,
|
|
6807
|
+
advertisedUrls,
|
|
6808
|
+
htmlAlternateUnique: advertisedUrls.length === 1,
|
|
6809
|
+
httpAlternateUrls,
|
|
6810
|
+
explicitMatchesNegotiated: null,
|
|
6811
|
+
repeatedHashStable: null,
|
|
6812
|
+
markdownCanonicalMatchesHtml: null
|
|
6813
|
+
};
|
|
6814
|
+
if (advertisedUrls.length !== 1) return observation;
|
|
6815
|
+
const explicitUrl = advertisedUrls[0];
|
|
6816
|
+
const [explicit, negotiated, repeated] = await Promise.all([
|
|
6817
|
+
fetchRepresentation({
|
|
6818
|
+
url: explicitUrl,
|
|
6819
|
+
accept: "text/markdown",
|
|
6820
|
+
timeoutMs: input.timeoutMs,
|
|
6821
|
+
fetch: input.fetch,
|
|
6822
|
+
signal: input.signal
|
|
6823
|
+
}),
|
|
6824
|
+
fetchRepresentation({
|
|
6825
|
+
url: input.page.finalUrl,
|
|
6826
|
+
accept: "text/markdown",
|
|
6827
|
+
timeoutMs: input.timeoutMs,
|
|
6828
|
+
fetch: input.fetch,
|
|
6829
|
+
signal: input.signal
|
|
6830
|
+
}),
|
|
6831
|
+
fetchRepresentation({
|
|
6832
|
+
url: explicitUrl,
|
|
6833
|
+
accept: "text/markdown",
|
|
6834
|
+
timeoutMs: input.timeoutMs,
|
|
6835
|
+
fetch: input.fetch,
|
|
6836
|
+
signal: input.signal
|
|
6837
|
+
})
|
|
6838
|
+
]);
|
|
6839
|
+
observation.explicit = explicit.observation;
|
|
6840
|
+
observation.negotiated = negotiated.observation;
|
|
6841
|
+
observation.repeated = repeated.observation;
|
|
6842
|
+
observation.explicitMatchesNegotiated = explicit.observation.sha256 && negotiated.observation.sha256 ? explicit.observation.sha256 === negotiated.observation.sha256 : null;
|
|
6843
|
+
observation.repeatedHashStable = explicit.observation.sha256 && repeated.observation.sha256 ? explicit.observation.sha256 === repeated.observation.sha256 : null;
|
|
6844
|
+
observation.markdownCanonicalMatchesHtml = sameDocument(explicit.observation.canonicalUrl, input.page.finalUrl);
|
|
6845
|
+
if (explicit.body) observation.quality = markdownQuality(explicit.body, input.page);
|
|
6846
|
+
return observation;
|
|
6847
|
+
}
|
|
6848
|
+
async function fetchText(input) {
|
|
6849
|
+
const controller = combinedSignal(input.timeoutMs, input.signal);
|
|
6850
|
+
try {
|
|
6851
|
+
const response = await input.fetch(input.url, {
|
|
6852
|
+
profile: "bot",
|
|
6853
|
+
redirect: input.redirect ?? "follow",
|
|
6854
|
+
headers: input.accept ? { accept: input.accept } : void 0,
|
|
6855
|
+
signal: controller.signal
|
|
6856
|
+
});
|
|
6857
|
+
return {
|
|
6858
|
+
response,
|
|
6859
|
+
body: await readBoundedText(response)
|
|
6860
|
+
};
|
|
6861
|
+
} finally {
|
|
6862
|
+
controller.cleanup();
|
|
6863
|
+
}
|
|
6864
|
+
}
|
|
6865
|
+
function safeError(error) {
|
|
6866
|
+
return error instanceof Error ? error.message : String(error);
|
|
6867
|
+
}
|
|
6868
|
+
async function inspectAgentSkills(input) {
|
|
6869
|
+
const indexUrl = new URL("/.well-known/agent-skills/index.json", input.origin).toString();
|
|
6870
|
+
try {
|
|
6871
|
+
const { response, body } = await fetchText({
|
|
6872
|
+
url: indexUrl,
|
|
6873
|
+
timeoutMs: input.timeoutMs,
|
|
6874
|
+
fetch: input.fetch,
|
|
6875
|
+
signal: input.signal,
|
|
6876
|
+
accept: "application/json"
|
|
6877
|
+
});
|
|
6878
|
+
const contentType = response.headers.get("content-type") ?? void 0;
|
|
6879
|
+
const cors = response.headers.get("access-control-allow-origin") ?? void 0;
|
|
6880
|
+
if (response.status < 200 || response.status >= 300) return {
|
|
6881
|
+
indexUrl,
|
|
6882
|
+
status: response.status,
|
|
6883
|
+
contentType,
|
|
6884
|
+
validIndex: false,
|
|
6885
|
+
skills: []
|
|
6886
|
+
};
|
|
6887
|
+
const parsed = JSON.parse(body);
|
|
6888
|
+
const entries = Array.isArray(parsed.skills) ? parsed.skills.slice(0, MAX_SKILLS) : [];
|
|
6889
|
+
const skills = await Promise.all(entries.map(async (entry) => {
|
|
6890
|
+
const name = typeof entry.name === "string" ? entry.name : void 0;
|
|
6891
|
+
const declaredDigest = typeof entry.digest === "string" ? entry.digest : void 0;
|
|
6892
|
+
const relativeUrl = typeof entry.url === "string" ? entry.url : void 0;
|
|
6893
|
+
if (!relativeUrl) return {
|
|
6894
|
+
name,
|
|
6895
|
+
declaredDigest,
|
|
6896
|
+
digestMatches: null,
|
|
6897
|
+
frontmatterValid: null,
|
|
6898
|
+
sameOrigin: false,
|
|
6899
|
+
error: "Skill URL is missing."
|
|
6900
|
+
};
|
|
6901
|
+
const url = new URL(relativeUrl, indexUrl).toString();
|
|
6902
|
+
const sameOrigin = new URL(url).origin === new URL(input.origin).origin;
|
|
6903
|
+
if (!sameOrigin) return {
|
|
6904
|
+
name,
|
|
6905
|
+
url,
|
|
6906
|
+
declaredDigest,
|
|
6907
|
+
digestMatches: null,
|
|
6908
|
+
frontmatterValid: null,
|
|
6909
|
+
sameOrigin,
|
|
6910
|
+
error: "Skill URL is off origin."
|
|
6911
|
+
};
|
|
6912
|
+
try {
|
|
6913
|
+
const skill = await fetchText({
|
|
6914
|
+
url,
|
|
6915
|
+
timeoutMs: input.timeoutMs,
|
|
6916
|
+
fetch: input.fetch,
|
|
6917
|
+
signal: input.signal,
|
|
6918
|
+
accept: "text/markdown,text/plain;q=0.9"
|
|
6919
|
+
});
|
|
6920
|
+
const observedDigest = `sha256:${sha256(skill.body)}`;
|
|
6921
|
+
const frontmatterValid = /^---\s*\n[\s\S]*?^name:\s*\S.+$[\s\S]*?^description:\s*\S.+$[\s\S]*?^---\s*$/mu.test(skill.body);
|
|
6922
|
+
return {
|
|
6923
|
+
name,
|
|
6924
|
+
url,
|
|
6925
|
+
status: skill.response.status,
|
|
6926
|
+
contentType: skill.response.headers.get("content-type") ?? void 0,
|
|
6927
|
+
cors: skill.response.headers.get("access-control-allow-origin") ?? void 0,
|
|
6928
|
+
declaredDigest,
|
|
6929
|
+
observedDigest,
|
|
6930
|
+
digestMatches: declaredDigest ? declaredDigest === observedDigest : null,
|
|
6931
|
+
frontmatterValid,
|
|
6932
|
+
sameOrigin
|
|
6933
|
+
};
|
|
6934
|
+
} catch (error) {
|
|
6935
|
+
return {
|
|
6936
|
+
name,
|
|
6937
|
+
url,
|
|
6938
|
+
declaredDigest,
|
|
6939
|
+
digestMatches: null,
|
|
6940
|
+
frontmatterValid: null,
|
|
6941
|
+
sameOrigin,
|
|
6942
|
+
error: safeError(error)
|
|
6943
|
+
};
|
|
6944
|
+
}
|
|
6945
|
+
}));
|
|
6946
|
+
return {
|
|
6947
|
+
indexUrl,
|
|
6948
|
+
status: response.status,
|
|
6949
|
+
contentType,
|
|
6950
|
+
cors,
|
|
6951
|
+
validIndex: typeof parsed.$schema === "string" && Array.isArray(parsed.skills),
|
|
6952
|
+
skills
|
|
6953
|
+
};
|
|
6954
|
+
} catch (error) {
|
|
6955
|
+
return {
|
|
6956
|
+
indexUrl,
|
|
6957
|
+
validIndex: false,
|
|
6958
|
+
skills: [],
|
|
6959
|
+
error: safeError(error)
|
|
6960
|
+
};
|
|
6961
|
+
}
|
|
6962
|
+
}
|
|
6963
|
+
function markdownLinks(value, base) {
|
|
6964
|
+
const links = [];
|
|
6965
|
+
const invalidLinks = [];
|
|
6966
|
+
for (const match of value.matchAll(/\[([^\u005d]+)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/gu)) {
|
|
6967
|
+
if (!match[1] || !match[2]) continue;
|
|
6968
|
+
try {
|
|
6969
|
+
links.push({
|
|
6970
|
+
label: match[1],
|
|
6971
|
+
url: new URL(match[2], base).toString()
|
|
6972
|
+
});
|
|
6973
|
+
} catch {
|
|
6974
|
+
invalidLinks.push(match[2]);
|
|
6975
|
+
}
|
|
6976
|
+
}
|
|
6977
|
+
return {
|
|
6978
|
+
links,
|
|
6979
|
+
invalidLinks
|
|
6980
|
+
};
|
|
6981
|
+
}
|
|
6982
|
+
async function inspectLlmsTxt(input) {
|
|
6983
|
+
const url = new URL("/llms.txt", input.origin).toString();
|
|
6984
|
+
try {
|
|
6985
|
+
const [first, second] = await Promise.all([fetchText({
|
|
6986
|
+
url,
|
|
6987
|
+
timeoutMs: input.timeoutMs,
|
|
6988
|
+
fetch: input.fetch,
|
|
6989
|
+
signal: input.signal,
|
|
6990
|
+
accept: "text/plain,text/markdown;q=0.9"
|
|
6991
|
+
}), fetchText({
|
|
6992
|
+
url,
|
|
6993
|
+
timeoutMs: input.timeoutMs,
|
|
6994
|
+
fetch: input.fetch,
|
|
6995
|
+
signal: input.signal,
|
|
6996
|
+
accept: "text/plain,text/markdown;q=0.9"
|
|
6997
|
+
})]);
|
|
6998
|
+
const exists = first.response.status >= 200 && first.response.status < 300;
|
|
6999
|
+
const parsed = markdownLinks(first.body, url);
|
|
7000
|
+
const links = parsed.links.slice(0, MAX_LLMS_LINKS);
|
|
7001
|
+
const linkLimitReached = parsed.links.length > links.length;
|
|
7002
|
+
const counts = /* @__PURE__ */ new Map();
|
|
7003
|
+
for (const link of links) counts.set(link.url, (counts.get(link.url) ?? 0) + 1);
|
|
7004
|
+
const duplicateLinks = [...counts].filter(([, count]) => count > 1).map(([link]) => link).sort();
|
|
7005
|
+
const crawlRoutes = new Set(input.pages.map((page) => normalizedDocumentUrl(page.finalUrl)));
|
|
7006
|
+
const linkObservations = await Promise.all(links.map(async (link) => {
|
|
7007
|
+
const sameOrigin = new URL(link.url).origin === new URL(input.origin).origin;
|
|
7008
|
+
try {
|
|
7009
|
+
const target = await fetchText({
|
|
7010
|
+
url: link.url,
|
|
7011
|
+
timeoutMs: input.timeoutMs,
|
|
7012
|
+
fetch: input.fetch,
|
|
7013
|
+
signal: input.signal
|
|
7014
|
+
});
|
|
7015
|
+
const indexableTarget = sameOrigin && target.response.status >= 200 && target.response.status < 300 ? !responseIsNoindex(target.response, target.body) : void 0;
|
|
7016
|
+
const finalUrl = target.response.url || void 0;
|
|
7017
|
+
return {
|
|
7018
|
+
...link,
|
|
7019
|
+
sameOrigin,
|
|
7020
|
+
status: target.response.status,
|
|
7021
|
+
finalUrl,
|
|
7022
|
+
redirected: target.response.redirected,
|
|
7023
|
+
...indexableTarget === void 0 ? {} : { indexableTarget }
|
|
7024
|
+
};
|
|
7025
|
+
} catch (error) {
|
|
7026
|
+
return {
|
|
7027
|
+
...link,
|
|
7028
|
+
sameOrigin,
|
|
7029
|
+
redirected: false,
|
|
7030
|
+
error: safeError(error)
|
|
7031
|
+
};
|
|
7032
|
+
}
|
|
7033
|
+
}));
|
|
7034
|
+
const linkedCrawlRoutes = new Set(linkObservations.filter((link) => link.sameOrigin && !new URL(link.url).pathname.startsWith("/.well-known/") && !new URL(link.url).pathname.endsWith(".md")).map((link) => normalizedDocumentUrl(link.url)));
|
|
7035
|
+
return {
|
|
7036
|
+
url,
|
|
7037
|
+
exists,
|
|
7038
|
+
status: first.response.status,
|
|
7039
|
+
contentType: first.response.headers.get("content-type") ?? void 0,
|
|
7040
|
+
bytes: Buffer.byteLength(first.body),
|
|
7041
|
+
sha256: sha256(first.body),
|
|
7042
|
+
repeatedHashStable: sha256(first.body) === sha256(second.body),
|
|
7043
|
+
headingCount: first.body.match(/^#{1,2}\s+\S/gmu)?.length ?? 0,
|
|
7044
|
+
totalParsedLinks: parsed.links.length,
|
|
7045
|
+
linkLimitReached,
|
|
7046
|
+
links: linkObservations,
|
|
7047
|
+
invalidLinks: parsed.invalidLinks.sort(),
|
|
7048
|
+
duplicateLinks,
|
|
7049
|
+
offSiteLinks: linkObservations.filter((link) => !link.sameOrigin).map((link) => link.url).sort(),
|
|
7050
|
+
redirectedLinks: linkObservations.filter((link) => link.redirected).map((link) => link.url).sort(),
|
|
7051
|
+
nonIndexableLinks: linkObservations.filter((link) => link.indexableTarget === false).map((link) => link.url).sort(),
|
|
7052
|
+
missingCrawlRoutes: [...linkedCrawlRoutes].filter((route) => !crawlRoutes.has(route)).sort(),
|
|
7053
|
+
oversized: Buffer.byteLength(first.body) > MAX_CURATED_LLMS_BYTES
|
|
7054
|
+
};
|
|
7055
|
+
} catch (error) {
|
|
7056
|
+
return {
|
|
7057
|
+
url,
|
|
7058
|
+
exists: false,
|
|
7059
|
+
repeatedHashStable: null,
|
|
7060
|
+
headingCount: 0,
|
|
7061
|
+
totalParsedLinks: 0,
|
|
7062
|
+
linkLimitReached: false,
|
|
7063
|
+
links: [],
|
|
7064
|
+
invalidLinks: [],
|
|
7065
|
+
duplicateLinks: [],
|
|
7066
|
+
offSiteLinks: [],
|
|
7067
|
+
redirectedLinks: [],
|
|
7068
|
+
nonIndexableLinks: [],
|
|
7069
|
+
missingCrawlRoutes: [],
|
|
7070
|
+
oversized: false,
|
|
7071
|
+
error: safeError(error)
|
|
7072
|
+
};
|
|
7073
|
+
}
|
|
7074
|
+
}
|
|
7075
|
+
async function inspectRouteManifest(input) {
|
|
7076
|
+
const url = new URL("/agent-routes.json", input.origin).toString();
|
|
7077
|
+
try {
|
|
7078
|
+
const result = await fetchText({
|
|
7079
|
+
url,
|
|
7080
|
+
timeoutMs: input.timeoutMs,
|
|
7081
|
+
fetch: input.fetch,
|
|
7082
|
+
signal: input.signal,
|
|
7083
|
+
accept: "application/json"
|
|
7084
|
+
});
|
|
7085
|
+
const parsed = JSON.parse(result.body);
|
|
7086
|
+
const declared = Array.isArray(parsed.pages) ? parsed.pages : [];
|
|
7087
|
+
const declaredHtmlRoutes = declared.map((page) => page.htmlPath).filter((path) => typeof path === "string").sort();
|
|
7088
|
+
const declaredMarkdownRoutes = declared.map((page) => page.markdownPath).filter((path) => typeof path === "string").sort();
|
|
7089
|
+
const crawledHtmlRoutes = new Set(input.pages.map((page) => new URL(page.finalUrl).pathname.replace(/\/$/u, "") || "/"));
|
|
7090
|
+
const observedMarkdownRoutes = new Set(input.observations.flatMap((page) => page.advertisedUrls.map((value) => new URL(value).pathname)));
|
|
7091
|
+
return {
|
|
7092
|
+
url,
|
|
7093
|
+
status: result.response.status,
|
|
7094
|
+
valid: Array.isArray(parsed.pages),
|
|
7095
|
+
declaredHtmlRoutes,
|
|
7096
|
+
declaredMarkdownRoutes,
|
|
7097
|
+
missingHtmlRoutes: declaredHtmlRoutes.filter((path) => !crawledHtmlRoutes.has(path.replace(/\/$/u, "") || "/")).sort(),
|
|
7098
|
+
missingMarkdownRoutes: declaredMarkdownRoutes.filter((path) => !observedMarkdownRoutes.has(path)).sort(),
|
|
7099
|
+
orphanMarkdownRoutes: [...observedMarkdownRoutes].filter((path) => !declaredMarkdownRoutes.includes(path)).sort()
|
|
7100
|
+
};
|
|
7101
|
+
} catch (error) {
|
|
7102
|
+
return {
|
|
7103
|
+
url,
|
|
7104
|
+
valid: false,
|
|
7105
|
+
declaredHtmlRoutes: [],
|
|
7106
|
+
declaredMarkdownRoutes: [],
|
|
7107
|
+
missingHtmlRoutes: [],
|
|
7108
|
+
missingMarkdownRoutes: [],
|
|
7109
|
+
orphanMarkdownRoutes: [],
|
|
7110
|
+
error: safeError(error)
|
|
7111
|
+
};
|
|
7112
|
+
}
|
|
7113
|
+
}
|
|
7114
|
+
async function inspectProtocolVariant(input) {
|
|
7115
|
+
try {
|
|
7116
|
+
const result = await fetchText({
|
|
7117
|
+
url: input.url,
|
|
7118
|
+
timeoutMs: input.timeoutMs,
|
|
7119
|
+
fetch: input.fetch,
|
|
7120
|
+
signal: input.signal,
|
|
7121
|
+
redirect: "manual"
|
|
7122
|
+
});
|
|
7123
|
+
return {
|
|
7124
|
+
status: result.response.status,
|
|
7125
|
+
location: result.response.headers.get("location") ?? void 0
|
|
7126
|
+
};
|
|
7127
|
+
} catch (error) {
|
|
7128
|
+
return { error: safeError(error) };
|
|
7129
|
+
}
|
|
7130
|
+
}
|
|
7131
|
+
async function collectAgentDiscovery(input) {
|
|
7132
|
+
const origin = new URL(input.startUrl).origin;
|
|
7133
|
+
const pages = input.pages.filter((page) => page.status >= 200 && page.status < 300 && /^\s*(?:text\/html|application\/xhtml\+xml)\b/iu.test(page.contentType ?? "")).sort((left, right) => left.finalUrl.localeCompare(right.finalUrl));
|
|
7134
|
+
const queue = new PQueue({ concurrency: input.concurrency ?? 4 });
|
|
7135
|
+
const observations = await Promise.all(pages.map((page) => queue.add(() => inspectMarkdownPage({
|
|
7136
|
+
page,
|
|
7137
|
+
timeoutMs: input.timeoutMs,
|
|
7138
|
+
fetch: input.fetch,
|
|
7139
|
+
signal: input.signal
|
|
7140
|
+
}))));
|
|
7141
|
+
const [agentSkills, llmsTxt, routeManifest] = await Promise.all([
|
|
7142
|
+
inspectAgentSkills({
|
|
7143
|
+
...input,
|
|
7144
|
+
origin
|
|
7145
|
+
}),
|
|
7146
|
+
inspectLlmsTxt({
|
|
7147
|
+
...input,
|
|
7148
|
+
origin,
|
|
7149
|
+
pages
|
|
7150
|
+
}),
|
|
7151
|
+
inspectRouteManifest({
|
|
7152
|
+
...input,
|
|
7153
|
+
origin,
|
|
7154
|
+
pages,
|
|
7155
|
+
observations
|
|
7156
|
+
})
|
|
7157
|
+
]);
|
|
7158
|
+
const qZero = await fetchRepresentation({
|
|
7159
|
+
url: input.startUrl,
|
|
7160
|
+
accept: "text/markdown;q=0,text/html;q=1",
|
|
7161
|
+
timeoutMs: input.timeoutMs,
|
|
7162
|
+
fetch: input.fetch,
|
|
7163
|
+
signal: input.signal
|
|
7164
|
+
});
|
|
7165
|
+
const start = new URL(input.startUrl);
|
|
7166
|
+
const httpUrl = new URL(start);
|
|
7167
|
+
httpUrl.protocol = "http:";
|
|
7168
|
+
const wwwUrl = new URL(start);
|
|
7169
|
+
wwwUrl.hostname = start.hostname.startsWith("www.") ? start.hostname.slice(4) : `www.${start.hostname}`;
|
|
7170
|
+
const [http, www] = await Promise.all([inspectProtocolVariant({
|
|
7171
|
+
...input,
|
|
7172
|
+
url: httpUrl.toString()
|
|
7173
|
+
}), inspectProtocolVariant({
|
|
7174
|
+
...input,
|
|
7175
|
+
url: wwwUrl.toString()
|
|
7176
|
+
})]);
|
|
7177
|
+
const httpLocation = http.location ? new URL(http.location, httpUrl).toString() : void 0;
|
|
7178
|
+
const wwwLocation = www.location ? new URL(www.location, wwwUrl).toString() : void 0;
|
|
7179
|
+
const qZeroContentType = qZero.observation.contentType;
|
|
7180
|
+
const advertisedPages = observations.filter((observation) => observation.htmlAlternateUnique).length;
|
|
7181
|
+
const evaluatedPages = observations.filter((observation) => observation.explicit?.status !== void 0).length;
|
|
7182
|
+
const warnings = [...pages.length !== input.pages.length ? [`Agent representations were evaluated for ${pages.length} successful HTML pages from ${input.pages.length} retained documents.`] : [], ...routeManifest.valid ? [] : ["No valid public agent route manifest was available for orphan checks."]].sort();
|
|
7183
|
+
const htmlContentSignals = pages.map((page) => headerValue$1(page.responseHeaders, "content-signal")).filter((value) => Boolean(value));
|
|
7184
|
+
const markdownContentSignals = observations.map((observation) => observation.explicit?.contentSignal).filter((value) => Boolean(value));
|
|
7185
|
+
const contentSignalValues = [.../* @__PURE__ */ new Set([...htmlContentSignals, ...markdownContentSignals])].sort();
|
|
7186
|
+
return {
|
|
7187
|
+
profile: "content",
|
|
7188
|
+
profileApplicability: {
|
|
7189
|
+
content: {
|
|
7190
|
+
status: "evaluated",
|
|
7191
|
+
reason: "This site publishes documents, so access, representations, discovery, identity, and extraction quality were evaluated."
|
|
7192
|
+
},
|
|
7193
|
+
api: {
|
|
7194
|
+
status: "notApplicable",
|
|
7195
|
+
reason: "No public API surface was selected for this content-site run."
|
|
7196
|
+
},
|
|
7197
|
+
application: {
|
|
7198
|
+
status: "notApplicable",
|
|
7199
|
+
reason: "A local MCP product does not make this website a public remote agent endpoint."
|
|
7200
|
+
},
|
|
7201
|
+
commerce: {
|
|
7202
|
+
status: "notApplicable",
|
|
7203
|
+
reason: "No product checkout or transactional surface was selected."
|
|
7204
|
+
}
|
|
7205
|
+
},
|
|
7206
|
+
dataStatus: pages.length === 0 ? "unavailable" : evaluatedPages === pages.length ? "complete" : "partial",
|
|
7207
|
+
markdownAlternates: {
|
|
7208
|
+
eligibleHtmlPages: pages.length,
|
|
7209
|
+
advertisedPages,
|
|
7210
|
+
evaluatedPages,
|
|
7211
|
+
exactByteMatches: observations.filter((observation) => observation.explicitMatchesNegotiated).length,
|
|
7212
|
+
stableResponses: observations.filter((observation) => observation.repeatedHashStable).length,
|
|
7213
|
+
pages: observations
|
|
7214
|
+
},
|
|
7215
|
+
contentNegotiation: {
|
|
7216
|
+
qZeroHonoured: qZeroContentType !== void 0 ? !/^\s*text\/markdown\b/iu.test(qZeroContentType) : null,
|
|
7217
|
+
qZeroStatus: qZero.observation.status,
|
|
7218
|
+
qZeroContentType,
|
|
7219
|
+
error: qZero.observation.error
|
|
7220
|
+
},
|
|
7221
|
+
routeManifest,
|
|
7222
|
+
agentSkills,
|
|
7223
|
+
llmsTxt,
|
|
7224
|
+
contentSignals: {
|
|
7225
|
+
htmlValues: [...new Set(htmlContentSignals)].sort(),
|
|
7226
|
+
markdownValues: [...new Set(markdownContentSignals)].sort(),
|
|
7227
|
+
missingHtmlPages: pages.length - htmlContentSignals.length,
|
|
7228
|
+
missingMarkdownPages: observations.length - markdownContentSignals.length,
|
|
7229
|
+
consistent: pages.length > 0 && observations.length > 0 ? contentSignalValues.length === 1 && htmlContentSignals.length === pages.length && markdownContentSignals.length === observations.length : null
|
|
7230
|
+
},
|
|
7231
|
+
protocolVariants: {
|
|
7232
|
+
http: {
|
|
7233
|
+
url: httpUrl.toString(),
|
|
7234
|
+
status: http.status,
|
|
7235
|
+
location: httpLocation,
|
|
7236
|
+
permanentRedirectToHttps: http.status !== void 0 ? [301, 308].includes(http.status) && Boolean(httpLocation?.startsWith("https://")) : null,
|
|
7237
|
+
error: http.error
|
|
7238
|
+
},
|
|
7239
|
+
www: {
|
|
7240
|
+
url: wwwUrl.toString(),
|
|
7241
|
+
status: www.status,
|
|
7242
|
+
location: wwwLocation,
|
|
7243
|
+
redirectsToPreferredHost: www.status !== void 0 ? [301, 308].includes(www.status) && Boolean(wwwLocation && new URL(wwwLocation).hostname === start.hostname && new URL(wwwLocation).protocol === "https:") : null,
|
|
7244
|
+
error: www.error
|
|
7245
|
+
},
|
|
7246
|
+
hstsOnStartPage: pages.find((page) => normalizedDocumentUrl(page.finalUrl) === normalizedDocumentUrl(input.startUrl))?.hasHsts ?? null
|
|
7247
|
+
},
|
|
7248
|
+
warnings
|
|
7249
|
+
};
|
|
7250
|
+
}
|
|
7251
|
+
//#endregion
|
|
7252
|
+
//#region packages/core/src/analyze/crawler/agent-readiness.ts
|
|
7253
|
+
function check$1(section, input) {
|
|
7254
|
+
return {
|
|
7255
|
+
section,
|
|
7256
|
+
...input
|
|
7257
|
+
};
|
|
7258
|
+
}
|
|
7259
|
+
function section$1(id, title, checks) {
|
|
7260
|
+
return {
|
|
7261
|
+
id,
|
|
7262
|
+
title,
|
|
7263
|
+
checks
|
|
7264
|
+
};
|
|
7265
|
+
}
|
|
7266
|
+
function failedMarkdownUrls(discovery) {
|
|
7267
|
+
return discovery.markdownAlternates.pages.filter((page) => !page.htmlAlternateUnique || page.httpAlternateUrls.length !== 1 || page.explicit?.status !== 200 || !/^\s*text\/markdown\b/iu.test(page.explicit.contentType ?? "") || page.explicitMatchesNegotiated !== true || page.repeatedHashStable !== true || page.markdownCanonicalMatchesHtml !== true || page.explicit?.varyAccept !== true || page.negotiated?.varyAccept !== true).map((page) => page.htmlUrl);
|
|
7268
|
+
}
|
|
7269
|
+
function qualityFailureUrls(discovery) {
|
|
7270
|
+
return discovery.markdownAlternates.pages.filter((page) => {
|
|
7271
|
+
const quality = page.quality;
|
|
7272
|
+
if (!quality) return true;
|
|
7273
|
+
return !quality.frontmatterTitle || quality.h1Count !== 1 || !quality.codeFenceBalanced || quality.rawHtmlTags > 0 || quality.rawSvgTags > 0 || quality.rawScriptTags > 0 || quality.rawStyleTags > 0 || quality.navigationOnly || quality.repeatedLines > 0 || quality.wordRetentionRatio !== null && quality.wordRetentionRatio < .4 || quality.introductoryCopyRetained === false;
|
|
7274
|
+
}).map((page) => page.htmlUrl);
|
|
7275
|
+
}
|
|
7276
|
+
function profileChecks(discovery) {
|
|
7277
|
+
return [
|
|
7278
|
+
"api",
|
|
7279
|
+
"application",
|
|
7280
|
+
"commerce"
|
|
7281
|
+
].map((profile) => check$1("profile", {
|
|
7282
|
+
id: `profile-${profile}`,
|
|
7283
|
+
status: "notApplicable",
|
|
7284
|
+
title: `${profile[0]?.toUpperCase()}${profile.slice(1)} checks are not applicable`,
|
|
7285
|
+
plainEnglish: discovery.profileApplicability[profile].reason,
|
|
7286
|
+
action: "Choose this profile only when the site exposes that capability publicly.",
|
|
7287
|
+
evidence: {
|
|
7288
|
+
profile,
|
|
7289
|
+
selectedProfile: discovery.profile
|
|
7290
|
+
}
|
|
7291
|
+
}));
|
|
7292
|
+
}
|
|
7293
|
+
function summariseChecks(checks) {
|
|
7294
|
+
const count = (status) => checks.filter((item) => item.status === status).length;
|
|
7295
|
+
return {
|
|
7296
|
+
checks: checks.length,
|
|
7297
|
+
passed: count("pass"),
|
|
7298
|
+
warnings: count("warning"),
|
|
7299
|
+
failed: count("fail"),
|
|
7300
|
+
unknown: count("unknown"),
|
|
7301
|
+
information: count("info"),
|
|
7302
|
+
notApplicable: count("notApplicable")
|
|
7303
|
+
};
|
|
7304
|
+
}
|
|
7305
|
+
function markdownChecks(discovery) {
|
|
7306
|
+
const markdown = discovery.markdownAlternates;
|
|
7307
|
+
const failedUrls = failedMarkdownUrls(discovery);
|
|
7308
|
+
const qualityUrls = qualityFailureUrls(discovery);
|
|
7309
|
+
const coverageComplete = markdown.eligibleHtmlPages > 0 && markdown.advertisedPages === markdown.eligibleHtmlPages && markdown.evaluatedPages === markdown.eligibleHtmlPages;
|
|
7310
|
+
const bytesMatch = markdown.eligibleHtmlPages > 0 && markdown.exactByteMatches === markdown.eligibleHtmlPages;
|
|
7311
|
+
const stable = markdown.eligibleHtmlPages > 0 && markdown.stableResponses === markdown.eligibleHtmlPages;
|
|
7312
|
+
const tokenHeaders = markdown.pages.filter((page) => page.explicit?.bytes !== void 0 && page.explicit?.markdownTokens !== void 0 && page.explicit.markdownTokens === Math.ceil(page.explicit.bytes / 4) && page.explicit.markdownTokens === page.negotiated?.markdownTokens).length;
|
|
7313
|
+
return [
|
|
7314
|
+
check$1("representations", {
|
|
7315
|
+
id: "markdown-coverage",
|
|
7316
|
+
status: markdown.eligibleHtmlPages === 0 ? "unknown" : coverageComplete ? "pass" : "fail",
|
|
7317
|
+
title: coverageComplete ? "Every successful HTML page advertises one Markdown alternative" : "Markdown alternative coverage is incomplete",
|
|
7318
|
+
plainEnglish: `${markdown.advertisedPages} of ${markdown.eligibleHtmlPages} successful HTML pages advertised one Markdown alternative, and ${markdown.evaluatedPages} were fetched.`,
|
|
7319
|
+
action: coverageComplete ? "Keep route generation tied to the same deterministic page manifest." : "Add one text/markdown alternate for each public HTML page and make every advertised URL return successfully.",
|
|
7320
|
+
evidence: {
|
|
7321
|
+
eligibleHtmlPages: markdown.eligibleHtmlPages,
|
|
7322
|
+
advertisedPages: markdown.advertisedPages,
|
|
7323
|
+
evaluatedPages: markdown.evaluatedPages
|
|
7324
|
+
},
|
|
7325
|
+
urls: failedUrls.slice(0, 25)
|
|
7326
|
+
}),
|
|
7327
|
+
check$1("representations", {
|
|
7328
|
+
id: "markdown-token-estimates",
|
|
7329
|
+
status: markdown.evaluatedPages === 0 ? "unknown" : tokenHeaders === markdown.evaluatedPages ? "pass" : "warning",
|
|
7330
|
+
title: tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes the same token estimate" : "Some Markdown responses are missing a stable token estimate",
|
|
7331
|
+
plainEnglish: `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned matching X-Markdown-Tokens values on the explicit and negotiated responses. The value is a quick size estimate, not an exact model-specific token count.`,
|
|
7332
|
+
action: "Generate the estimate from the stored Markdown bytes and return the same value for explicit and negotiated responses.",
|
|
7333
|
+
urls: markdown.pages.filter((page) => page.explicit?.markdownTokens === void 0 || page.explicit.markdownTokens !== page.negotiated?.markdownTokens).map((page) => page.htmlUrl).slice(0, 25)
|
|
7334
|
+
}),
|
|
7335
|
+
check$1("representations", {
|
|
7336
|
+
id: "markdown-negotiation",
|
|
7337
|
+
status: markdown.eligibleHtmlPages === 0 ? "unknown" : bytesMatch && discovery.contentNegotiation.qZeroHonoured ? "pass" : "fail",
|
|
7338
|
+
title: bytesMatch && discovery.contentNegotiation.qZeroHonoured ? "Explicit and negotiated Markdown use the same bytes" : "Content negotiation does not match the explicit Markdown contract",
|
|
7339
|
+
plainEnglish: `${markdown.exactByteMatches} of ${markdown.eligibleHtmlPages} pages returned identical explicit and negotiated Markdown. A request that refuses Markdown ${discovery.contentNegotiation.qZeroHonoured ? "received HTML" : "did not produce confirmed HTML evidence"}.`,
|
|
7340
|
+
action: "Use one stored Markdown artifact for both routes, honour Accept q-values, and send Vary: Accept on negotiated responses.",
|
|
7341
|
+
evidence: {
|
|
7342
|
+
exactByteMatches: markdown.exactByteMatches,
|
|
7343
|
+
qZero: discovery.contentNegotiation
|
|
7344
|
+
},
|
|
7345
|
+
urls: failedUrls.slice(0, 25)
|
|
7346
|
+
}),
|
|
7347
|
+
check$1("representations", {
|
|
7348
|
+
id: "markdown-determinism",
|
|
7349
|
+
status: markdown.eligibleHtmlPages === 0 ? "unknown" : stable ? "pass" : "fail",
|
|
7350
|
+
title: stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
|
|
7351
|
+
plainEnglish: `${markdown.stableResponses} of ${markdown.eligibleHtmlPages} repeated explicit responses kept the same SHA-256 digest.`,
|
|
7352
|
+
action: "Generate Markdown during the build and remove runtime rewriting, timestamps, or random output.",
|
|
7353
|
+
evidence: { stableResponses: markdown.stableResponses },
|
|
7354
|
+
urls: failedUrls.slice(0, 25)
|
|
7355
|
+
}),
|
|
7356
|
+
check$1("representations", {
|
|
7357
|
+
id: "markdown-quality",
|
|
7358
|
+
status: markdown.evaluatedPages === 0 ? "unknown" : qualityUrls.length === 0 ? "pass" : "warning",
|
|
7359
|
+
title: qualityUrls.length === 0 ? "Markdown keeps the useful document structure cleanly" : "Some Markdown alternatives need an extraction review",
|
|
7360
|
+
plainEnglish: qualityUrls.length === 0 ? "The evaluated alternatives retained one H1, frontmatter titles, balanced code fences, useful copy, and no leaked SVG, script, style, or layout tags." : `${qualityUrls.length} Markdown alternative${qualityUrls.length === 1 ? "" : "s"} lost important structure, looked navigation-only, or leaked presentation markup.`,
|
|
7361
|
+
action: "Open the affected Markdown directly and fix the shared converter or component exclusion rules rather than patching generated files.",
|
|
7362
|
+
urls: qualityUrls.slice(0, 25)
|
|
7363
|
+
})
|
|
7364
|
+
];
|
|
7365
|
+
}
|
|
7366
|
+
function discoveryChecks(discovery) {
|
|
7367
|
+
const skills = discovery.agentSkills;
|
|
7368
|
+
const skillsValid = skills.validIndex && skills.skills.length > 0 && skills.skills.every((skill) => skill.sameOrigin && skill.status === 200 && /^\s*text\/markdown\b/iu.test(skill.contentType ?? "") && Boolean(skill.cors) && skill.digestMatches === true && skill.frontmatterValid === true) && Boolean(skills.cors);
|
|
7369
|
+
const llms = discovery.llmsTxt;
|
|
7370
|
+
const llmsBroken = llms.links.filter((link) => !link.status || link.status < 200 || link.status >= 400);
|
|
7371
|
+
const llmsValid = llms.exists && /^\s*(?:text\/plain|text\/markdown)\b/iu.test(llms.contentType ?? "") && llms.headingCount > 0 && llms.links.length > 0 && !llms.linkLimitReached && !llms.oversized && llms.invalidLinks.length === 0 && llms.duplicateLinks.length === 0 && llms.redirectedLinks.length === 0 && llms.nonIndexableLinks.length === 0 && llms.missingCrawlRoutes.length === 0 && llmsBroken.length === 0 && llms.repeatedHashStable === true;
|
|
7372
|
+
return [
|
|
7373
|
+
check$1("discovery", {
|
|
7374
|
+
id: "agent-skills",
|
|
7375
|
+
status: skillsValid ? "pass" : skills.status === 404 ? "info" : "warning",
|
|
7376
|
+
title: skillsValid ? "Agent Skills discovery is valid and digest verified" : "Agent Skills discovery is absent or needs attention",
|
|
7377
|
+
plainEnglish: skillsValid ? `${skills.skills.length} published skill${skills.skills.length === 1 ? "" : "s"} matched the declared SHA-256 digest, valid SKILL.md frontmatter, Markdown content type, and cross-origin access policy.` : "Agent Skills is optional for a content site, but a published index should resolve same-origin Markdown skill files, allow intended cross-origin clients, and verify their declared bytes.",
|
|
7378
|
+
action: "If you publish Agent Skills, keep the discovery index and SHA-256 digests generated from the exact deployed skill bytes.",
|
|
7379
|
+
evidence: skills
|
|
7380
|
+
}),
|
|
7381
|
+
check$1("discovery", {
|
|
7382
|
+
id: "llms-txt",
|
|
7383
|
+
status: llms.exists ? llmsValid ? "pass" : "warning" : "info",
|
|
7384
|
+
title: llms.exists ? llmsValid ? "llms.txt is short, stable, and its links resolve" : "llms.txt exists but needs a content or link review" : "llms.txt is not published",
|
|
7385
|
+
plainEnglish: llms.exists ? `${llms.links.length} declared links were checked. ${llms.invalidLinks.length} were malformed, ${llms.duplicateLinks.length} were duplicated, ${llms.redirectedLinks.length} redirected, ${llms.nonIndexableLinks.length} reached non-indexable pages, ${llmsBroken.length} did not resolve, and ${llms.missingCrawlRoutes.length} were missing from the crawl inventory. ${llms.offSiteLinks.length} linked to other sites.` : "llms.txt is optional and its absence is not a search ranking problem.",
|
|
7386
|
+
action: llms.exists ? "Keep the file curated, deterministic, and limited to useful entry points whose links still resolve." : "Add it only when an intended consumer uses it. Do not treat it as a Google ranking requirement.",
|
|
7387
|
+
evidence: {
|
|
7388
|
+
status: llms.status,
|
|
7389
|
+
contentType: llms.contentType,
|
|
7390
|
+
bytes: llms.bytes,
|
|
7391
|
+
oversized: llms.oversized,
|
|
7392
|
+
totalParsedLinks: llms.totalParsedLinks,
|
|
7393
|
+
linkLimitReached: llms.linkLimitReached,
|
|
7394
|
+
links: llms.links.length,
|
|
7395
|
+
invalidLinks: llms.invalidLinks,
|
|
7396
|
+
duplicateLinks: llms.duplicateLinks,
|
|
7397
|
+
offSiteLinks: llms.offSiteLinks,
|
|
7398
|
+
redirectedLinks: llms.redirectedLinks,
|
|
7399
|
+
nonIndexableLinks: llms.nonIndexableLinks,
|
|
7400
|
+
missingCrawlRoutes: llms.missingCrawlRoutes,
|
|
7401
|
+
repeatedHashStable: llms.repeatedHashStable
|
|
7402
|
+
},
|
|
7403
|
+
urls: [
|
|
7404
|
+
...llmsBroken.map((link) => link.url),
|
|
7405
|
+
...llms.redirectedLinks,
|
|
7406
|
+
...llms.nonIndexableLinks,
|
|
7407
|
+
...llms.missingCrawlRoutes
|
|
7408
|
+
].slice(0, 25)
|
|
7409
|
+
}),
|
|
7410
|
+
check$1("discovery", {
|
|
7411
|
+
id: "content-signals",
|
|
7412
|
+
status: discovery.contentSignals.consistent === null ? "unknown" : discovery.contentSignals.consistent ? "pass" : "warning",
|
|
7413
|
+
title: discovery.contentSignals.consistent ? "HTML and Markdown publish one explicit Content Signals policy" : "Content Signals are missing or inconsistent across representations",
|
|
7414
|
+
plainEnglish: discovery.contentSignals.consistent ? `The same ${discovery.contentSignals.htmlValues[0]} policy was observed on every evaluated HTML and Markdown response.` : `${discovery.contentSignals.missingHtmlPages} HTML responses and ${discovery.contentSignals.missingMarkdownPages} Markdown responses were missing the header, or the observed values differed.`,
|
|
7415
|
+
action: "Choose the site policy deliberately and return the same Content-Signal value on HTML and Markdown responses.",
|
|
7416
|
+
evidence: discovery.contentSignals
|
|
7417
|
+
}),
|
|
7418
|
+
check$1("discovery", {
|
|
7419
|
+
id: "route-manifest",
|
|
7420
|
+
status: !discovery.routeManifest.valid ? "info" : discovery.routeManifest.missingHtmlRoutes.length === 0 && discovery.routeManifest.missingMarkdownRoutes.length === 0 && discovery.routeManifest.orphanMarkdownRoutes.length === 0 ? "pass" : "warning",
|
|
7421
|
+
title: discovery.routeManifest.valid ? "The public route manifest agrees with the crawl" : "No public route manifest was available",
|
|
7422
|
+
plainEnglish: discovery.routeManifest.valid ? `The manifest declared ${discovery.routeManifest.declaredHtmlRoutes.length} HTML routes and ${discovery.routeManifest.declaredMarkdownRoutes.length} Markdown routes. Missing or orphan routes are listed in the evidence.` : "A public route manifest is optional. Without one, this crawl cannot prove that no orphan Markdown files were deployed.",
|
|
7423
|
+
action: "Generate one deterministic route inventory from the build when exact HTML and Markdown parity matters.",
|
|
7424
|
+
evidence: discovery.routeManifest
|
|
7425
|
+
})
|
|
7426
|
+
];
|
|
7427
|
+
}
|
|
7428
|
+
function accessChecks(report, discovery) {
|
|
7429
|
+
const bots = report.ai?.robotsTxt?.botAccess ?? [];
|
|
7430
|
+
const blocked = bots.filter((bot) => bot.allowed === false);
|
|
7431
|
+
const robotsKnown = report.ai?.robotsTxt?.availability === "available" || report.ai?.robotsTxt?.availability === "absent";
|
|
7432
|
+
const protocol = discovery.protocolVariants;
|
|
7433
|
+
return [check$1("access", {
|
|
7434
|
+
id: "crawler-access",
|
|
7435
|
+
status: !robotsKnown ? "unknown" : blocked.length > 0 ? "warning" : "pass",
|
|
7436
|
+
title: blocked.length ? "robots.txt blocks selected crawler tokens" : "No selected crawler token is blocked at the start URL",
|
|
7437
|
+
plainEnglish: robotsKnown ? `${blocked.length} of ${bots.length} selected crawler tokens were blocked by the start URL robots.txt policy.` : "The robots.txt response was not stable enough to make a crawler-access claim.",
|
|
7438
|
+
action: "Keep blocks intentional. A crawler token being allowed here does not prove that the service will crawl, index, or cite the site.",
|
|
7439
|
+
evidence: {
|
|
7440
|
+
availability: report.ai?.robotsTxt?.availability,
|
|
7441
|
+
blocked,
|
|
7442
|
+
botAccess: bots
|
|
7443
|
+
}
|
|
7444
|
+
}), check$1("access", {
|
|
7445
|
+
id: "protocol-canonicalization",
|
|
7446
|
+
status: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "pass" : protocol.http.permanentRedirectToHttps === null ? "unknown" : "warning",
|
|
7447
|
+
title: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "HTTP redirects permanently to HTTPS and HTTPS sends HSTS" : "Protocol canonicalization needs attention or could not be proven",
|
|
7448
|
+
plainEnglish: `The HTTP variant ${protocol.http.permanentRedirectToHttps === true ? "redirected permanently to HTTPS" : "did not produce a confirmed permanent HTTPS redirect"}. The HTTPS start page ${protocol.hstsOnStartPage ? "sent HSTS" : "did not provide confirmed HSTS evidence"}.`,
|
|
7449
|
+
action: "Use one permanent HTTP-to-HTTPS redirect and send HSTS only after HTTPS is stable across the site.",
|
|
7450
|
+
evidence: protocol
|
|
7451
|
+
})];
|
|
7452
|
+
}
|
|
7453
|
+
function identityChecks(report) {
|
|
7454
|
+
const types = new Set(report.pages.flatMap((page) => page.schemaTypes ?? []));
|
|
7455
|
+
const hasSoftware = types.has("SoftwareApplication");
|
|
7456
|
+
const hasWebsite = types.has("WebSite");
|
|
7457
|
+
const hasCreator = types.has("Person") || types.has("Organization");
|
|
7458
|
+
const pageTypes = [
|
|
7459
|
+
"WebPage",
|
|
7460
|
+
"TechArticle",
|
|
7461
|
+
"CollectionPage"
|
|
7462
|
+
].filter((type) => types.has(type));
|
|
7463
|
+
const officialProfiles = [...new Set(report.pages.flatMap((page) => [...page.schemaSameAs ?? [], ...page.socialProfileLinks ?? []]))];
|
|
7464
|
+
return [check$1("identity", {
|
|
7465
|
+
id: "identity-graph",
|
|
7466
|
+
status: hasSoftware && hasWebsite && hasCreator && pageTypes.length > 0 ? "pass" : "warning",
|
|
7467
|
+
title: hasSoftware && hasWebsite && hasCreator && pageTypes.length > 0 ? "Structured identity covers the software, website, creator, and pages" : "The structured identity graph is missing a useful entity layer",
|
|
7468
|
+
plainEnglish: `Software identity: ${hasSoftware ? "present" : "missing"}. Website identity: ${hasWebsite ? "present" : "missing"}. Creator or publisher identity: ${hasCreator ? "present" : "missing"}. Page-level types: ${pageTypes.length ? pageTypes.join(", ") : "missing"}.`,
|
|
7469
|
+
action: "Connect truthful SoftwareApplication, WebSite, creator or publisher, and page nodes. Use sameAs only for official profiles that identify the same entity.",
|
|
7470
|
+
evidence: {
|
|
7471
|
+
schemaTypes: [...types].sort(),
|
|
7472
|
+
officialProfiles: officialProfiles.sort()
|
|
7473
|
+
}
|
|
7474
|
+
})];
|
|
7475
|
+
}
|
|
7476
|
+
function agentReadiness(report) {
|
|
7477
|
+
const discovery = report.agentDiscovery;
|
|
7478
|
+
if (!discovery) {
|
|
7479
|
+
const profiles = {
|
|
7480
|
+
content: {
|
|
7481
|
+
status: "evaluated",
|
|
7482
|
+
reason: "The content profile was selected but discovery evidence was not collected."
|
|
7483
|
+
},
|
|
7484
|
+
api: {
|
|
7485
|
+
status: "notApplicable",
|
|
7486
|
+
reason: "Not selected."
|
|
7487
|
+
},
|
|
7488
|
+
application: {
|
|
7489
|
+
status: "notApplicable",
|
|
7490
|
+
reason: "Not selected."
|
|
7491
|
+
},
|
|
7492
|
+
commerce: {
|
|
7493
|
+
status: "notApplicable",
|
|
7494
|
+
reason: "Not selected."
|
|
7495
|
+
}
|
|
7496
|
+
};
|
|
7497
|
+
const missing = check$1("collection", {
|
|
7498
|
+
id: "agent-discovery-evidence",
|
|
7499
|
+
status: "unknown",
|
|
7500
|
+
title: "Agent discovery evidence was not collected",
|
|
7501
|
+
plainEnglish: "This saved crawl predates the focused content-profile checks or was run without them.",
|
|
7502
|
+
action: "Run the agent-readiness report with a URL or start a fresh crawl with agent discovery enabled."
|
|
7503
|
+
});
|
|
7504
|
+
const scopeChecks = profileChecks({
|
|
7505
|
+
profile: "content",
|
|
7506
|
+
profileApplicability: profiles
|
|
7507
|
+
});
|
|
7508
|
+
const checks = [missing, ...scopeChecks];
|
|
7509
|
+
return {
|
|
7510
|
+
reportId: report.id,
|
|
7511
|
+
url: report.config.url,
|
|
7512
|
+
generatedAt: report.generatedAt,
|
|
7513
|
+
profile: "content",
|
|
7514
|
+
dataStatus: "unavailable",
|
|
7515
|
+
assessment: "evidence-only",
|
|
7516
|
+
headline: "No agent-discovery evidence is stored in this crawl. Run a fresh focused check.",
|
|
7517
|
+
profileApplicability: profiles,
|
|
7518
|
+
summary: summariseChecks(checks),
|
|
7519
|
+
sections: [section$1("collection", "Evidence collection", [missing]), section$1("profile", "Profile scope", scopeChecks)],
|
|
7520
|
+
checks,
|
|
7521
|
+
topActions: [missing],
|
|
7522
|
+
caveats: ["Not evaluated does not mean absent or broken.", "This report does not measure rankings, AI mentions, citations, or selection."]
|
|
7523
|
+
};
|
|
7524
|
+
}
|
|
7525
|
+
const sections = [
|
|
7526
|
+
section$1("profile", "Profile scope", profileChecks(discovery)),
|
|
7527
|
+
section$1("access", "Crawler and protocol access", accessChecks(report, discovery)),
|
|
7528
|
+
section$1("representations", "HTML and Markdown representations", markdownChecks(discovery)),
|
|
7529
|
+
section$1("discovery", "Agent discovery files", discoveryChecks(discovery)),
|
|
7530
|
+
section$1("identity", "Identity evidence", identityChecks(report))
|
|
7531
|
+
];
|
|
7532
|
+
const checks = sections.flatMap((item) => item.checks);
|
|
7533
|
+
const actionable = checks.filter((item) => [
|
|
7534
|
+
"fail",
|
|
7535
|
+
"warning",
|
|
7536
|
+
"unknown"
|
|
7537
|
+
].includes(item.status));
|
|
7538
|
+
const rank = {
|
|
7539
|
+
fail: 0,
|
|
7540
|
+
warning: 1,
|
|
7541
|
+
unknown: 2,
|
|
7542
|
+
pass: 3,
|
|
7543
|
+
info: 4,
|
|
7544
|
+
notApplicable: 5
|
|
7545
|
+
};
|
|
7546
|
+
const topActions = [...actionable].sort((left, right) => rank[left.status] - rank[right.status]).slice(0, 10);
|
|
7547
|
+
const summary = summariseChecks(checks);
|
|
7548
|
+
return {
|
|
7549
|
+
reportId: report.id,
|
|
7550
|
+
url: report.config.url,
|
|
7551
|
+
generatedAt: report.generatedAt,
|
|
7552
|
+
profile: discovery.profile,
|
|
7553
|
+
dataStatus: discovery.dataStatus,
|
|
7554
|
+
assessment: "evidence-only",
|
|
7555
|
+
headline: `${summary.passed} checks passed, ${summary.warnings} need review, and ${summary.failed} failed for the content profile.`,
|
|
7556
|
+
profileApplicability: discovery.profileApplicability,
|
|
7557
|
+
summary,
|
|
7558
|
+
sections,
|
|
7559
|
+
checks,
|
|
7560
|
+
topActions,
|
|
7561
|
+
caveats: [
|
|
7562
|
+
...report.caveats,
|
|
7563
|
+
...discovery.warnings,
|
|
7564
|
+
"A clean content profile does not prove indexing, rankings, AI mentions, citations, or selection.",
|
|
7565
|
+
"A crawler token being allowed does not prove that a service fetched or used the page.",
|
|
7566
|
+
"API, application, and commerce checks were not applicable to this content-site run."
|
|
7567
|
+
]
|
|
7568
|
+
};
|
|
7569
|
+
}
|
|
7570
|
+
//#endregion
|
|
7571
|
+
//#region packages/core/src/analyze/crawler/entity-readiness.ts
|
|
7572
|
+
const ENTITY_EVIDENCE_SCHEMA_TYPES = /* @__PURE__ */ new Set([
|
|
7573
|
+
"organization",
|
|
7574
|
+
"localbusiness",
|
|
7575
|
+
"person",
|
|
7576
|
+
"product",
|
|
7577
|
+
"softwareapplication",
|
|
7578
|
+
"website"
|
|
7579
|
+
]);
|
|
7580
|
+
const SITE_IDENTITY_SCHEMA_TYPES = /* @__PURE__ */ new Set(["organization", "localbusiness"]);
|
|
7581
|
+
const PROFILE_IDENTITY_SCHEMA_TYPES = /* @__PURE__ */ new Set([...SITE_IDENTITY_SCHEMA_TYPES, "person"]);
|
|
7582
|
+
function isEntityEvidenceSchemaType(type) {
|
|
7583
|
+
return ENTITY_EVIDENCE_SCHEMA_TYPES.has(type.toLowerCase());
|
|
7584
|
+
}
|
|
7585
|
+
function isSiteIdentitySchemaType(type) {
|
|
7586
|
+
return SITE_IDENTITY_SCHEMA_TYPES.has(type.toLowerCase());
|
|
7587
|
+
}
|
|
7588
|
+
function isProfileIdentitySchemaType(type) {
|
|
7589
|
+
return PROFILE_IDENTITY_SCHEMA_TYPES.has(type.toLowerCase());
|
|
7590
|
+
}
|
|
7591
|
+
function unique$3(values) {
|
|
7592
|
+
return [...new Set(values.filter((value) => Boolean(value)))];
|
|
7593
|
+
}
|
|
7594
|
+
function pct$3(part, total) {
|
|
7595
|
+
return total ? Math.round(part / total * 100) : 0;
|
|
7596
|
+
}
|
|
7597
|
+
function entityReadiness(report) {
|
|
7598
|
+
const pages = report.pages.filter((page) => page.indexable && page.status < 400);
|
|
7599
|
+
const total = pages.length || 1;
|
|
7600
|
+
const schemaTypes = {};
|
|
7601
|
+
for (const page of pages) for (const type of page.schemaTypes ?? []) schemaTypes[type] = (schemaTypes[type] ?? 0) + 1;
|
|
7602
|
+
const entitySchemaPages = pages.filter((page) => page.schemaTypes?.some(isEntityEvidenceSchemaType));
|
|
7603
|
+
const sameAsEvidence = pages.flatMap((page) => page.schemaSameAsEvidence ?? []).filter((evidence) => evidence.subjectTypes.some(isProfileIdentitySchemaType));
|
|
7604
|
+
const siteSameAsEvidence = sameAsEvidence.filter((evidence) => evidence.subjectTypes.some(isSiteIdentitySchemaType));
|
|
7605
|
+
const sameAs = unique$3(sameAsEvidence.map((evidence) => evidence.url));
|
|
7606
|
+
const sameAsByType = {};
|
|
7607
|
+
for (const evidence of sameAsEvidence) for (const type of evidence.subjectTypes) {
|
|
7608
|
+
if (!isProfileIdentitySchemaType(type)) continue;
|
|
7609
|
+
sameAsByType[type] = unique$3([...sameAsByType[type] ?? [], evidence.url]);
|
|
7610
|
+
}
|
|
7611
|
+
const socialProfiles = unique$3(pages.flatMap((page) => page.socialProfileLinks ?? []));
|
|
7612
|
+
const authors = unique$3(pages.map((page) => page.author));
|
|
7613
|
+
const authorPages = pages.filter((page) => page.author || page.geo?.hasAuthor);
|
|
7614
|
+
const datePages = pages.filter((page) => page.hasDate || page.geo?.hasDate);
|
|
7615
|
+
const titlePages = pages.filter((page) => page.title && page.h1);
|
|
7616
|
+
const checks = [
|
|
7617
|
+
{
|
|
7618
|
+
coverage: pct$3(entitySchemaPages.length, total),
|
|
7619
|
+
check: {
|
|
7620
|
+
id: "entity-schema",
|
|
7621
|
+
title: "Entity schema is present",
|
|
7622
|
+
plainEnglish: `${pct$3(entitySchemaPages.length, total)}% of evaluated indexable pages include Organization, LocalBusiness, Person, Product, SoftwareApplication, or WebSite schema.`,
|
|
7623
|
+
action: "Add accurate schema only where it describes the software, website, organization, person, product, or local business shown on the page.",
|
|
7624
|
+
evidence: { schemaTypes },
|
|
7625
|
+
urls: pages.filter((page) => !entitySchemaPages.includes(page)).slice(0, 10).map((page) => page.finalUrl)
|
|
7626
|
+
}
|
|
7627
|
+
},
|
|
7628
|
+
{
|
|
7629
|
+
coverage: siteSameAsEvidence.length ? 100 : 0,
|
|
7630
|
+
check: {
|
|
7631
|
+
id: "same-as",
|
|
7632
|
+
title: "Site entity profiles are connected",
|
|
7633
|
+
plainEnglish: siteSameAsEvidence.length ? "The crawl found sameAs links attached to Organization or LocalBusiness structured data." : sameAs.length ? "The crawl found Person sameAs links, but no Organization or LocalBusiness profile evidence for the site entity." : socialProfiles.length ? "The crawl found social-domain links, but they are not enough to prove official profile ownership." : "The crawl did not find entity-scoped sameAs evidence.",
|
|
7634
|
+
action: "Connect official profiles with sameAs on Organization, Person, or LocalBusiness schema. Use only profiles you control.",
|
|
7635
|
+
evidence: {
|
|
7636
|
+
sameAs,
|
|
7637
|
+
siteSameAs: unique$3(siteSameAsEvidence.map((evidence) => evidence.url)),
|
|
7638
|
+
sameAsEvidence,
|
|
7639
|
+
unclassifiedSocialLinks: socialProfiles
|
|
7640
|
+
}
|
|
7641
|
+
}
|
|
7642
|
+
},
|
|
7643
|
+
{
|
|
7644
|
+
coverage: Math.round((pct$3(authorPages.length, total) + pct$3(datePages.length, total)) / 2),
|
|
7645
|
+
check: {
|
|
7646
|
+
id: "authority-freshness",
|
|
7647
|
+
title: "Author and date signals observed",
|
|
7648
|
+
plainEnglish: `${pct$3(authorPages.length, total)}% of evaluated indexable pages expose an author signal and ${pct$3(datePages.length, total)}% expose a date signal.`,
|
|
7649
|
+
action: "For editorial content, identify authors and show accurate published or modified dates when that information is useful to readers.",
|
|
7650
|
+
evidence: {
|
|
7651
|
+
authorCoverage: pct$3(authorPages.length, total),
|
|
7652
|
+
dateCoverage: pct$3(datePages.length, total),
|
|
7653
|
+
authors
|
|
7654
|
+
}
|
|
7655
|
+
}
|
|
7656
|
+
},
|
|
7657
|
+
{
|
|
7658
|
+
coverage: pct$3(titlePages.length, total),
|
|
7659
|
+
check: {
|
|
7660
|
+
id: "entity-naming",
|
|
7661
|
+
title: "Title and H1 coverage observed",
|
|
7662
|
+
plainEnglish: `${pct$3(titlePages.length, total)}% of evaluated indexable pages have both a title and H1.`,
|
|
7663
|
+
action: "Review pages missing either signal. Keep brand, product, and person names accurate wherever they appear."
|
|
7664
|
+
}
|
|
7665
|
+
}
|
|
7666
|
+
].map(({ coverage, check }) => ({
|
|
7667
|
+
...check,
|
|
7668
|
+
status: "info",
|
|
7669
|
+
evidence: {
|
|
7670
|
+
...check.evidence,
|
|
7671
|
+
observedCoveragePercent: coverage
|
|
7672
|
+
}
|
|
7673
|
+
}));
|
|
7674
|
+
const dataStatus = report.status === "completed" && pages.length > 0 ? "complete" : "partial";
|
|
7675
|
+
return {
|
|
7676
|
+
reportId: report.id,
|
|
7677
|
+
url: report.config.url,
|
|
7678
|
+
generatedAt: report.generatedAt,
|
|
7679
|
+
dataStatus,
|
|
7680
|
+
evaluatedPages: pages.length,
|
|
7681
|
+
crawlPages: report.pages.length,
|
|
7682
|
+
assessment: "evidence-only",
|
|
7683
|
+
headline: dataStatus === "partial" ? "Entity evidence is incomplete; treat these findings as scoped to the evaluated pages, not the whole site." : "Entity signals are reported as observations, not a ranking, visibility, or machine-understanding score.",
|
|
7684
|
+
caveats: [
|
|
7685
|
+
...report.caveats,
|
|
7686
|
+
"This report deliberately has no aggregate entity score. Add schema, authors, dates, and sameAs only where they accurately fit the page and a documented use case.",
|
|
7687
|
+
...dataStatus === "partial" ? [`Entity coverage is based on ${pages.length} evaluated indexable page${pages.length === 1 ? "" : "s"} from a ${report.status} crawl.`] : [],
|
|
7688
|
+
...socialProfiles.length && !sameAs.length ? ["Social-domain links are unclassified navigation evidence, not proof that the linked profiles are official."] : []
|
|
7689
|
+
],
|
|
7690
|
+
checks,
|
|
7691
|
+
entities: {
|
|
7692
|
+
schemaTypes,
|
|
7693
|
+
sameAs,
|
|
7694
|
+
sameAsByType,
|
|
7695
|
+
socialProfiles,
|
|
7696
|
+
authors
|
|
6636
7697
|
}
|
|
6637
7698
|
};
|
|
6638
7699
|
}
|
|
6639
|
-
function matchesFilters(issue, filters) {
|
|
6640
|
-
if (filters.ruleId && issue.ruleId !== filters.ruleId) return false;
|
|
6641
|
-
if (filters.category && issue.category !== filters.category) return false;
|
|
6642
|
-
if (filters.severity && issue.severity !== filters.severity) return false;
|
|
6643
|
-
return true;
|
|
6644
|
-
}
|
|
6645
7700
|
//#endregion
|
|
6646
7701
|
//#region packages/core/src/analyze/crawler/ai-readiness.ts
|
|
6647
|
-
function pct$
|
|
7702
|
+
function pct$2(part, total) {
|
|
6648
7703
|
if (total <= 0) return 0;
|
|
6649
7704
|
return Math.round(part / total * 100);
|
|
6650
7705
|
}
|
|
@@ -6681,6 +7736,7 @@ function sampleUrls$1(pages, predicate) {
|
|
|
6681
7736
|
return pages.filter(predicate).slice(0, 10).map((page) => page.finalUrl);
|
|
6682
7737
|
}
|
|
6683
7738
|
function aiReadiness(report) {
|
|
7739
|
+
const agentDiscovery = report.agentDiscovery;
|
|
6684
7740
|
const pages = indexablePages$1(report);
|
|
6685
7741
|
const pageCount = pages.length;
|
|
6686
7742
|
const hasIndexablePages = pageCount > 0;
|
|
@@ -6704,7 +7760,7 @@ function aiReadiness(report) {
|
|
|
6704
7760
|
const titlePages = pages.filter((page) => page.title && page.h1);
|
|
6705
7761
|
const metaPages = pages.filter((page) => page.metaDescription);
|
|
6706
7762
|
const langPages = pages.filter((page) => page.lang);
|
|
6707
|
-
const entityPages = pages.filter((page) => page.schemaTypes?.some(
|
|
7763
|
+
const entityPages = pages.filter((page) => page.schemaTypes?.some(isEntityEvidenceSchemaType) || (page.schemaSameAs?.length ?? 0) > 0 || (page.socialProfileLinks?.length ?? 0) > 0);
|
|
6708
7764
|
const restrictedSnippets = pages.map((page) => ({
|
|
6709
7765
|
url: page.finalUrl,
|
|
6710
7766
|
control: effectiveSnippetControl({
|
|
@@ -6721,7 +7777,7 @@ function aiReadiness(report) {
|
|
|
6721
7777
|
section: "agent-access",
|
|
6722
7778
|
maxScore: 20,
|
|
6723
7779
|
evaluated: botAccessEvaluated,
|
|
6724
|
-
score: robotsUnavailable ? 0 : botAccess.length ? blockedBots.length ? scoreFromPercent(pct$
|
|
7780
|
+
score: robotsUnavailable ? 0 : botAccess.length ? blockedBots.length ? scoreFromPercent(pct$2(botAccess.length - blockedBots.length, botAccess.length), 20) : 20 : 8,
|
|
6725
7781
|
title: robotsUnavailable ? "robots.txt crawler-policy evidence is unavailable" : !hasBotAccessEvidence ? "robots.txt crawler-policy evidence was not collected" : blockedBots.length ? "robots.txt blocks selected crawler tokens at the start URL" : "robots.txt allows selected crawler tokens at the start URL",
|
|
6726
7782
|
plainEnglish: robotsUnavailable ? "The robots.txt request failed, so this report cannot say whether selected crawler tokens are allowed at the start URL." : !hasBotAccessEvidence ? "This crawl does not include per-bot robots.txt policy data for the start URL." : blockedBots.length ? `robots.txt policy blocks ${blockedBots.length} selected crawler ${blockedBots.length === 1 ? "token" : "tokens"} at the start URL.` : "robots.txt policy does not block the selected crawler tokens at the start URL. This does not verify actual or site-wide fetchability.",
|
|
6727
7783
|
action: robotsUnavailable ? "Restore a stable robots.txt response, then rerun this report before making crawler-policy claims." : !hasBotAccessEvidence ? "Rerun the crawl with top-level per-bot robots.txt evidence before making crawler-policy claims." : blockedBots.length ? "Only block AI crawlers intentionally. If discovery matters, remove accidental Disallow rules for the blocked user agents." : "Keep robots.txt explicit and intentional so humans and agents can see what is allowed.",
|
|
@@ -6764,10 +7820,11 @@ function aiReadiness(report) {
|
|
|
6764
7820
|
maxScore: 0,
|
|
6765
7821
|
score: 0,
|
|
6766
7822
|
title: "llms.txt is optional",
|
|
6767
|
-
plainEnglish: report.ai?.llmsTxt?.exists ? "An optional llms.txt file was found. Google says it has no positive or negative Search impact." : "No llms.txt file was found. Google says the file is not needed for Search and does not affect visibility.",
|
|
7823
|
+
plainEnglish: report.ai?.llmsTxt?.exists ? agentDiscovery?.llmsTxt ? `An optional llms.txt file was found and its body was checked. It declared ${agentDiscovery.llmsTxt.links.length} links, ${agentDiscovery.llmsTxt.duplicateLinks.length} duplicates, and ${agentDiscovery.llmsTxt.missingCrawlRoutes.length} links outside the crawled route inventory. Google says llms.txt has no positive or negative Search impact.` : "An optional llms.txt file was found. Its body was not validated in this crawl. Google says it has no positive or negative Search impact." : "No llms.txt file was found. Google says the file is not needed for Search and does not affect visibility.",
|
|
6768
7824
|
action: "No SEO action is required. Generate one only for a specific agent or service that consumes it.",
|
|
6769
7825
|
evidence: {
|
|
6770
7826
|
llmsTxt: report.ai?.llmsTxt,
|
|
7827
|
+
validation: agentDiscovery?.llmsTxt,
|
|
6771
7828
|
googleSearchImpact: "none",
|
|
6772
7829
|
guidanceUrl: "https://developers.google.com/search/updates#clarifying-guidance-on-llms-txt-files"
|
|
6773
7830
|
}
|
|
@@ -6790,7 +7847,7 @@ function aiReadiness(report) {
|
|
|
6790
7847
|
evaluated: hasIndexablePages,
|
|
6791
7848
|
score: 0,
|
|
6792
7849
|
title: "Short opening paragraphs are an unscored observation",
|
|
6793
|
-
plainEnglish: hasIndexablePages ? `${pct$
|
|
7850
|
+
plainEnglish: hasIndexablePages ? `${pct$2(answerablePages.length, pageCount)}% of indexable pages have at least one 25-word paragraph near the start. This does not establish content quality, citation likelihood, or Google AI eligibility.` : "No indexable 2xx pages were available, so opening-paragraph coverage was not evaluated.",
|
|
6794
7851
|
action: "Write for the reader and the page intent. Do not create artificial answer blocks or chunk content solely for AI Search.",
|
|
6795
7852
|
urls: sampleUrls$1(pages, (page) => !page.geo?.answerable),
|
|
6796
7853
|
evidence: {
|
|
@@ -6808,7 +7865,7 @@ function aiReadiness(report) {
|
|
|
6808
7865
|
evaluated: hasIndexablePages,
|
|
6809
7866
|
score: 0,
|
|
6810
7867
|
title: "Semantic HTML coverage observed",
|
|
6811
|
-
plainEnglish: hasIndexablePages ? `${pct$
|
|
7868
|
+
plainEnglish: hasIndexablePages ? `${pct$2(semanticPages.length, pageCount)}% of indexable pages use semantic structure.` : "No indexable 2xx pages were available, so semantic HTML coverage was not evaluated.",
|
|
6812
7869
|
action: "Use real headings, lists, tables, nav, main, article, and section elements instead of layout-only divs."
|
|
6813
7870
|
}),
|
|
6814
7871
|
check({
|
|
@@ -6818,7 +7875,7 @@ function aiReadiness(report) {
|
|
|
6818
7875
|
evaluated: hasIndexablePages,
|
|
6819
7876
|
score: 0,
|
|
6820
7877
|
title: "Title, H1, and description coverage observed",
|
|
6821
|
-
plainEnglish: hasIndexablePages ? `${pct$
|
|
7878
|
+
plainEnglish: hasIndexablePages ? `${pct$2(titlePages.length, pageCount)}% of indexable pages have title/H1 coverage and ${pct$2(metaPages.length, pageCount)}% have meta descriptions.` : "No indexable 2xx pages were available, so title, H1, and description coverage was not evaluated.",
|
|
6822
7879
|
action: "Write a unique title and descriptive H1. Add a useful meta description when a summary would help searchers understand the page."
|
|
6823
7880
|
}),
|
|
6824
7881
|
check({
|
|
@@ -6828,7 +7885,7 @@ function aiReadiness(report) {
|
|
|
6828
7885
|
evaluated: hasIndexablePages,
|
|
6829
7886
|
score: 0,
|
|
6830
7887
|
title: "HTML language declaration coverage observed",
|
|
6831
|
-
plainEnglish: hasIndexablePages ? `${pct$
|
|
7888
|
+
plainEnglish: hasIndexablePages ? `${pct$2(langPages.length, pageCount)}% of indexable pages declare a language.` : "No indexable 2xx pages were available, so language declaration coverage was not evaluated.",
|
|
6832
7889
|
action: "Set the html lang attribute so crawlers and assistive tech understand the page language."
|
|
6833
7890
|
})
|
|
6834
7891
|
]),
|
|
@@ -6839,7 +7896,7 @@ function aiReadiness(report) {
|
|
|
6839
7896
|
evaluated: hasIndexablePages,
|
|
6840
7897
|
score: 0,
|
|
6841
7898
|
title: "Entity-signal coverage observed",
|
|
6842
|
-
plainEnglish: hasIndexablePages ? `${pct$
|
|
7899
|
+
plainEnglish: hasIndexablePages ? `${pct$2(entityPages.length, pageCount)}% of indexable pages include entity signals such as Organization, Person, Product, sameAs, or social profile links.` : "No indexable 2xx pages were available, so entity-signal coverage was not evaluated.",
|
|
6843
7900
|
action: "Make the homepage and key pages explicit about the brand, product, people, and official profiles using accurate schema and links.",
|
|
6844
7901
|
urls: sampleUrls$1(pages, (page) => !entityPages.includes(page))
|
|
6845
7902
|
}), check({
|
|
@@ -6883,12 +7940,52 @@ function aiReadiness(report) {
|
|
|
6883
7940
|
section: "technical-ux",
|
|
6884
7941
|
maxScore: 12,
|
|
6885
7942
|
evaluated: report.pages.length > 0,
|
|
6886
|
-
score: scoreFromPercent(pct$
|
|
7943
|
+
score: scoreFromPercent(pct$2(pages.length, report.pages.length || 1), 12),
|
|
6887
7944
|
title: report.pages.length ? "Crawled-page indexability observed" : "Crawled-page indexability was not evaluated",
|
|
6888
|
-
plainEnglish: report.pages.length ? `${pct$
|
|
7945
|
+
plainEnglish: report.pages.length ? `${pct$2(pages.length, report.pages.length)}% of crawled pages are indexable 2xx pages.` : "No crawled pages were available, so status and indexability were not evaluated.",
|
|
6889
7946
|
action: "Fix 4xx/5xx pages, accidental noindex, and blocked internal URLs before expanding AI discovery work."
|
|
6890
7947
|
})
|
|
6891
7948
|
]),
|
|
7949
|
+
section("agent-content", "Agent content observations", [
|
|
7950
|
+
check({
|
|
7951
|
+
id: "markdown-alternates",
|
|
7952
|
+
section: "agent-content",
|
|
7953
|
+
maxScore: 0,
|
|
7954
|
+
score: 0,
|
|
7955
|
+
evaluated: Boolean(agentDiscovery),
|
|
7956
|
+
title: agentDiscovery ? "Markdown alternatives were checked" : "Markdown alternatives were not evaluated",
|
|
7957
|
+
plainEnglish: agentDiscovery ? `${agentDiscovery.markdownAlternates.advertisedPages} of ${agentDiscovery.markdownAlternates.eligibleHtmlPages} successful HTML pages advertised one Markdown alternative. ${agentDiscovery.markdownAlternates.exactByteMatches} explicit responses matched content negotiation byte for byte.` : "This crawl does not include the focused representation checks, so it cannot say whether HTML pages publish stable Markdown alternatives.",
|
|
7958
|
+
action: "Use the focused agent-readiness report when you need route coverage, content negotiation, byte stability, or extraction-quality evidence.",
|
|
7959
|
+
evidence: agentDiscovery ? {
|
|
7960
|
+
eligibleHtmlPages: agentDiscovery.markdownAlternates.eligibleHtmlPages,
|
|
7961
|
+
advertisedPages: agentDiscovery.markdownAlternates.advertisedPages,
|
|
7962
|
+
exactByteMatches: agentDiscovery.markdownAlternates.exactByteMatches,
|
|
7963
|
+
stableResponses: agentDiscovery.markdownAlternates.stableResponses
|
|
7964
|
+
} : void 0
|
|
7965
|
+
}),
|
|
7966
|
+
check({
|
|
7967
|
+
id: "agent-skills-discovery",
|
|
7968
|
+
section: "agent-content",
|
|
7969
|
+
maxScore: 0,
|
|
7970
|
+
score: 0,
|
|
7971
|
+
evaluated: Boolean(agentDiscovery),
|
|
7972
|
+
title: agentDiscovery ? "Agent Skills discovery was checked" : "Agent Skills discovery was not evaluated",
|
|
7973
|
+
plainEnglish: agentDiscovery ? `${agentDiscovery.agentSkills.skills.length} published skill${agentDiscovery.agentSkills.skills.length === 1 ? "" : "s"} were found. ${agentDiscovery.agentSkills.skills.filter((skill) => skill.digestMatches).length} matched their declared SHA-256 digest.` : "This crawl checked common legacy descriptor paths but did not validate the Agent Skills discovery index or its file digests.",
|
|
7974
|
+
action: "Publish Agent Skills only for real reusable capabilities. Generate the index digest from the exact deployed SKILL.md bytes.",
|
|
7975
|
+
evidence: agentDiscovery?.agentSkills
|
|
7976
|
+
}),
|
|
7977
|
+
check({
|
|
7978
|
+
id: "agent-content-profile",
|
|
7979
|
+
section: "agent-content",
|
|
7980
|
+
maxScore: 0,
|
|
7981
|
+
score: 0,
|
|
7982
|
+
evaluated: Boolean(agentDiscovery),
|
|
7983
|
+
title: agentDiscovery ? "The content profile was selected explicitly" : "No focused agent-readiness profile was selected",
|
|
7984
|
+
plainEnglish: agentDiscovery ? "Document access, representations, discovery, identity, and extraction quality were evaluated. API, application, and commerce checks were marked not applicable rather than failed." : "A normal crawl does not assume that a content site also exposes a public API, remote agent endpoint, or checkout.",
|
|
7985
|
+
action: "Run a different profile only when that public capability really exists.",
|
|
7986
|
+
evidence: agentDiscovery?.profileApplicability
|
|
7987
|
+
})
|
|
7988
|
+
]),
|
|
6892
7989
|
section("crawl-completeness", "Crawl completeness", [check({
|
|
6893
7990
|
id: "crawl-depth",
|
|
6894
7991
|
section: "crawl-completeness",
|
|
@@ -6939,117 +8036,6 @@ function aiReadiness(report) {
|
|
|
6939
8036
|
};
|
|
6940
8037
|
}
|
|
6941
8038
|
//#endregion
|
|
6942
|
-
//#region packages/core/src/analyze/crawler/entity-readiness.ts
|
|
6943
|
-
function unique$3(values) {
|
|
6944
|
-
return [...new Set(values.filter((value) => Boolean(value)))];
|
|
6945
|
-
}
|
|
6946
|
-
function pct$2(part, total) {
|
|
6947
|
-
return total ? Math.round(part / total * 100) : 0;
|
|
6948
|
-
}
|
|
6949
|
-
function entityReadiness(report) {
|
|
6950
|
-
const pages = report.pages.filter((page) => page.indexable && page.status < 400);
|
|
6951
|
-
const total = pages.length || 1;
|
|
6952
|
-
const schemaTypes = {};
|
|
6953
|
-
for (const page of pages) for (const type of page.schemaTypes ?? []) schemaTypes[type] = (schemaTypes[type] ?? 0) + 1;
|
|
6954
|
-
const entitySchemaPages = pages.filter((page) => page.schemaTypes?.some((type) => /^(Organization|LocalBusiness|Person|Product|WebSite)$/i.test(type)));
|
|
6955
|
-
const sameAsEvidence = pages.flatMap((page) => page.schemaSameAsEvidence ?? []).filter((evidence) => evidence.subjectTypes.some((type) => /^(Organization|LocalBusiness|Person)$/i.test(type)));
|
|
6956
|
-
const siteSameAsEvidence = sameAsEvidence.filter((evidence) => evidence.subjectTypes.some((type) => /^(Organization|LocalBusiness)$/i.test(type)));
|
|
6957
|
-
const sameAs = unique$3(sameAsEvidence.map((evidence) => evidence.url));
|
|
6958
|
-
const sameAsByType = {};
|
|
6959
|
-
for (const evidence of sameAsEvidence) for (const type of evidence.subjectTypes) {
|
|
6960
|
-
if (!/^(Organization|LocalBusiness|Person)$/i.test(type)) continue;
|
|
6961
|
-
sameAsByType[type] = unique$3([...sameAsByType[type] ?? [], evidence.url]);
|
|
6962
|
-
}
|
|
6963
|
-
const socialProfiles = unique$3(pages.flatMap((page) => page.socialProfileLinks ?? []));
|
|
6964
|
-
const authors = unique$3(pages.map((page) => page.author));
|
|
6965
|
-
const authorPages = pages.filter((page) => page.author || page.geo?.hasAuthor);
|
|
6966
|
-
const datePages = pages.filter((page) => page.hasDate || page.geo?.hasDate);
|
|
6967
|
-
const titlePages = pages.filter((page) => page.title && page.h1);
|
|
6968
|
-
const checks = [
|
|
6969
|
-
{
|
|
6970
|
-
coverage: pct$2(entitySchemaPages.length, total),
|
|
6971
|
-
check: {
|
|
6972
|
-
id: "entity-schema",
|
|
6973
|
-
title: "Entity schema is present",
|
|
6974
|
-
plainEnglish: `${pct$2(entitySchemaPages.length, total)}% of evaluated indexable pages include Organization, LocalBusiness, Person, Product, or WebSite schema.`,
|
|
6975
|
-
action: "Add accurate entity schema to the homepage and key product, author, organization, and local pages.",
|
|
6976
|
-
evidence: { schemaTypes },
|
|
6977
|
-
urls: pages.filter((page) => !entitySchemaPages.includes(page)).slice(0, 10).map((page) => page.finalUrl)
|
|
6978
|
-
}
|
|
6979
|
-
},
|
|
6980
|
-
{
|
|
6981
|
-
coverage: siteSameAsEvidence.length ? 100 : 0,
|
|
6982
|
-
check: {
|
|
6983
|
-
id: "same-as",
|
|
6984
|
-
title: "Site entity profiles are connected",
|
|
6985
|
-
plainEnglish: siteSameAsEvidence.length ? "The crawl found sameAs links attached to Organization or LocalBusiness structured data." : sameAs.length ? "The crawl found Person sameAs links, but no Organization or LocalBusiness profile evidence for the site entity." : socialProfiles.length ? "The crawl found social-domain links, but they are not enough to prove official profile ownership." : "The crawl did not find entity-scoped sameAs evidence.",
|
|
6986
|
-
action: "Connect official profiles with sameAs on Organization, Person, or LocalBusiness schema. Use only profiles you control.",
|
|
6987
|
-
evidence: {
|
|
6988
|
-
sameAs,
|
|
6989
|
-
siteSameAs: unique$3(siteSameAsEvidence.map((evidence) => evidence.url)),
|
|
6990
|
-
sameAsEvidence,
|
|
6991
|
-
unclassifiedSocialLinks: socialProfiles
|
|
6992
|
-
}
|
|
6993
|
-
}
|
|
6994
|
-
},
|
|
6995
|
-
{
|
|
6996
|
-
coverage: Math.round((pct$2(authorPages.length, total) + pct$2(datePages.length, total)) / 2),
|
|
6997
|
-
check: {
|
|
6998
|
-
id: "authority-freshness",
|
|
6999
|
-
title: "Author and date signals observed",
|
|
7000
|
-
plainEnglish: `${pct$2(authorPages.length, total)}% of evaluated indexable pages expose an author signal and ${pct$2(datePages.length, total)}% expose a date signal.`,
|
|
7001
|
-
action: "For editorial content, identify authors and show accurate published or modified dates when that information is useful to readers.",
|
|
7002
|
-
evidence: {
|
|
7003
|
-
authorCoverage: pct$2(authorPages.length, total),
|
|
7004
|
-
dateCoverage: pct$2(datePages.length, total),
|
|
7005
|
-
authors
|
|
7006
|
-
}
|
|
7007
|
-
}
|
|
7008
|
-
},
|
|
7009
|
-
{
|
|
7010
|
-
coverage: pct$2(titlePages.length, total),
|
|
7011
|
-
check: {
|
|
7012
|
-
id: "entity-naming",
|
|
7013
|
-
title: "Title and H1 coverage observed",
|
|
7014
|
-
plainEnglish: `${pct$2(titlePages.length, total)}% of evaluated indexable pages have both a title and H1.`,
|
|
7015
|
-
action: "Review pages missing either signal. Keep brand, product, and person names accurate wherever they appear."
|
|
7016
|
-
}
|
|
7017
|
-
}
|
|
7018
|
-
].map(({ coverage, check }) => ({
|
|
7019
|
-
...check,
|
|
7020
|
-
status: "info",
|
|
7021
|
-
evidence: {
|
|
7022
|
-
...check.evidence,
|
|
7023
|
-
observedCoveragePercent: coverage
|
|
7024
|
-
}
|
|
7025
|
-
}));
|
|
7026
|
-
const dataStatus = report.status === "completed" && pages.length > 0 ? "complete" : "partial";
|
|
7027
|
-
return {
|
|
7028
|
-
reportId: report.id,
|
|
7029
|
-
url: report.config.url,
|
|
7030
|
-
generatedAt: report.generatedAt,
|
|
7031
|
-
dataStatus,
|
|
7032
|
-
evaluatedPages: pages.length,
|
|
7033
|
-
crawlPages: report.pages.length,
|
|
7034
|
-
assessment: "evidence-only",
|
|
7035
|
-
headline: dataStatus === "partial" ? "Entity evidence is incomplete; treat these findings as scoped to the evaluated pages, not the whole site." : "Entity signals are reported as observations, not a ranking, visibility, or machine-understanding score.",
|
|
7036
|
-
caveats: [
|
|
7037
|
-
...report.caveats,
|
|
7038
|
-
"This report deliberately has no aggregate entity score. Add schema, authors, dates, and sameAs only where they accurately fit the page and a documented use case.",
|
|
7039
|
-
...dataStatus === "partial" ? [`Entity coverage is based on ${pages.length} evaluated indexable page${pages.length === 1 ? "" : "s"} from a ${report.status} crawl.`] : [],
|
|
7040
|
-
...socialProfiles.length && !sameAs.length ? ["Social-domain links are unclassified navigation evidence, not proof that the linked profiles are official."] : []
|
|
7041
|
-
],
|
|
7042
|
-
checks,
|
|
7043
|
-
entities: {
|
|
7044
|
-
schemaTypes,
|
|
7045
|
-
sameAs,
|
|
7046
|
-
sameAsByType,
|
|
7047
|
-
socialProfiles,
|
|
7048
|
-
authors
|
|
7049
|
-
}
|
|
7050
|
-
};
|
|
7051
|
-
}
|
|
7052
|
-
//#endregion
|
|
7053
8039
|
//#region packages/core/src/analyze/crawler/geo-gaps.ts
|
|
7054
8040
|
const ELIGIBILITY_RULES = /* @__PURE__ */ new Set([
|
|
7055
8041
|
"connection_error",
|
|
@@ -7117,7 +8103,7 @@ function partialReasons(report) {
|
|
|
7117
8103
|
if (report.requestEvidenceStatus === "partial") reasons.push("request-evidence-partial");
|
|
7118
8104
|
if (report.requestEvidenceStatus === "unavailable") reasons.push("request-evidence-unavailable");
|
|
7119
8105
|
if (report.summary.pageLimitReached) reasons.push("page-limit-reached");
|
|
7120
|
-
if (report.summary.
|
|
8106
|
+
if (report.summary.skippedUrlsByImpact.coverageAffecting > 0) reasons.push("coverage-affecting-skips");
|
|
7121
8107
|
if (report.summary.failedRequests > 0) reasons.push("failed-requests");
|
|
7122
8108
|
if (report.summary.extractionFailures > 0) reasons.push("extraction-failures");
|
|
7123
8109
|
return reasons;
|
|
@@ -7145,6 +8131,8 @@ function geoGapsReport(report, filters = {}) {
|
|
|
7145
8131
|
queuedUrls: report.summary.queuedUrls,
|
|
7146
8132
|
crawledUrls: report.summary.crawledUrls,
|
|
7147
8133
|
skippedUrls: report.summary.skippedUrls,
|
|
8134
|
+
coverageAffectingSkippedUrls: report.summary.skippedUrlsByImpact.coverageAffecting,
|
|
8135
|
+
nonImpactingSkippedUrls: report.summary.skippedUrlsByImpact.nonImpacting,
|
|
7148
8136
|
failedRequests: report.summary.failedRequests,
|
|
7149
8137
|
extractionFailures: report.summary.extractionFailures,
|
|
7150
8138
|
partialReasons: reasons
|
|
@@ -7210,8 +8198,56 @@ function candidatePages(report, options = {}) {
|
|
|
7210
8198
|
}
|
|
7211
8199
|
function auditLlmsTxt(report) {
|
|
7212
8200
|
const llmsTxt = report.ai?.llmsTxt;
|
|
8201
|
+
const validation = report.agentDiscovery?.llmsTxt;
|
|
7213
8202
|
const pages = candidatePages(report);
|
|
7214
8203
|
const issues = [];
|
|
8204
|
+
if (validation?.exists && !/^\s*(?:text\/plain|text\/markdown)\b/iu.test(validation.contentType ?? "")) issues.push({
|
|
8205
|
+
id: "llms-content-type",
|
|
8206
|
+
severity: "medium",
|
|
8207
|
+
title: "llms.txt uses an unexpected content type",
|
|
8208
|
+
plainEnglish: `The file returned ${validation.contentType ?? "no content type"} instead of plain text or Markdown.`,
|
|
8209
|
+
action: "Serve the deterministic text file as text/plain or text/markdown so consumers do not mistake it for HTML.",
|
|
8210
|
+
evidence: { contentType: validation.contentType }
|
|
8211
|
+
});
|
|
8212
|
+
if (validation?.exists && validation.headingCount === 0) issues.push({
|
|
8213
|
+
id: "llms-structure",
|
|
8214
|
+
severity: "medium",
|
|
8215
|
+
title: "llms.txt has no useful heading structure",
|
|
8216
|
+
plainEnglish: "The fetched file did not contain a level-one or level-two Markdown heading.",
|
|
8217
|
+
action: "Start with one clear title, then group a short list of useful entry points under descriptive headings."
|
|
8218
|
+
});
|
|
8219
|
+
if (validation?.duplicateLinks.length) issues.push({
|
|
8220
|
+
id: "llms-duplicate-links",
|
|
8221
|
+
severity: "low",
|
|
8222
|
+
title: "llms.txt repeats the same link",
|
|
8223
|
+
plainEnglish: `${validation.duplicateLinks.length} URL${validation.duplicateLinks.length === 1 ? " appears" : "s appear"} more than once.`,
|
|
8224
|
+
action: "Remove duplicate entries so the file remains short and deliberate.",
|
|
8225
|
+
evidence: { urls: validation.duplicateLinks }
|
|
8226
|
+
});
|
|
8227
|
+
const brokenLinks = validation?.links.filter((link) => !link.status || link.status < 200 || link.status >= 400 || link.error) ?? [];
|
|
8228
|
+
if (brokenLinks.length) issues.push({
|
|
8229
|
+
id: "llms-broken-links",
|
|
8230
|
+
severity: "medium",
|
|
8231
|
+
title: "llms.txt links did not resolve",
|
|
8232
|
+
plainEnglish: `${brokenLinks.length} declared link${brokenLinks.length === 1 ? "" : "s"} did not return a usable response during this audit.`,
|
|
8233
|
+
action: "Update or remove stale entries, then rerun the focused agent-readiness report.",
|
|
8234
|
+
evidence: { links: brokenLinks }
|
|
8235
|
+
});
|
|
8236
|
+
if (validation?.missingCrawlRoutes.length) issues.push({
|
|
8237
|
+
id: "llms-outside-crawl",
|
|
8238
|
+
severity: "low",
|
|
8239
|
+
title: "llms.txt links fall outside the crawl inventory",
|
|
8240
|
+
plainEnglish: `${validation.missingCrawlRoutes.length} same-origin route${validation.missingCrawlRoutes.length === 1 ? " was" : "s were"} not present in the retained crawl.`,
|
|
8241
|
+
action: "Confirm those routes are intentional and indexable, or remove stale declarations.",
|
|
8242
|
+
evidence: { urls: validation.missingCrawlRoutes }
|
|
8243
|
+
});
|
|
8244
|
+
if (validation?.repeatedHashStable === false) issues.push({
|
|
8245
|
+
id: "llms-unstable-body",
|
|
8246
|
+
severity: "medium",
|
|
8247
|
+
title: "llms.txt changed between repeated requests",
|
|
8248
|
+
plainEnglish: "Two requests in the same audit returned different SHA-256 digests.",
|
|
8249
|
+
action: "Generate the file during the build and remove timestamps, random ordering, or runtime rewriting."
|
|
8250
|
+
});
|
|
7215
8251
|
if (pages.length < 3) issues.push({
|
|
7216
8252
|
id: "thin-llms-inventory",
|
|
7217
8253
|
severity: "low",
|
|
@@ -7229,7 +8265,7 @@ function auditLlmsTxt(report) {
|
|
|
7229
8265
|
optional: true,
|
|
7230
8266
|
googleSearchImpact: "none",
|
|
7231
8267
|
guidanceUrl: "https://developers.google.com/search/updates#clarifying-guidance-on-llms-txt-files",
|
|
7232
|
-
headline: llmsTxt?.exists ? "An optional llms.txt file is present. It has no Google Search visibility impact." : "No llms.txt file was found. This is not an SEO issue and requires no action.",
|
|
8268
|
+
headline: llmsTxt?.exists ? validation ? issues.length ? `The optional llms.txt file was validated and ${issues.length} content or link issue${issues.length === 1 ? " needs" : "s need"} review. It has no Google Search visibility impact.` : `The optional llms.txt file was validated with ${validation.links.length} resolving link${validation.links.length === 1 ? "" : "s"} and a stable body. It has no Google Search visibility impact.` : "An optional llms.txt file is present, but its body was not validated in this crawl. It has no Google Search visibility impact." : "No llms.txt file was found. This is not an SEO issue and requires no action.",
|
|
7233
8269
|
issues,
|
|
7234
8270
|
recommendedPages: pages.slice(0, 25).map((page) => ({
|
|
7235
8271
|
url: page.finalUrl,
|
|
@@ -7329,8 +8365,6 @@ const SCORECARD_CHECK_ORDER = [
|
|
|
7329
8365
|
"entity-identity",
|
|
7330
8366
|
"answerable-content"
|
|
7331
8367
|
];
|
|
7332
|
-
const ENTITY_SCHEMA_PATTERN = /^(Organization|LocalBusiness|Person|Product|WebSite)$/i;
|
|
7333
|
-
const SITE_ENTITY_TYPE_PATTERN = /^(Organization|LocalBusiness)$/i;
|
|
7334
8368
|
function indexablePages(report) {
|
|
7335
8369
|
return report.pages.filter((page) => page.indexable && page.status < 400);
|
|
7336
8370
|
}
|
|
@@ -7538,8 +8572,8 @@ function entityIdentityCheck(report) {
|
|
|
7538
8572
|
finding: "Entity identity could not be evaluated because no indexable 2xx pages were available.",
|
|
7539
8573
|
verification
|
|
7540
8574
|
};
|
|
7541
|
-
const hasEntitySchema = Object.keys(entity.entities.schemaTypes).some(
|
|
7542
|
-
const siteSameAs = Object.entries(entity.entities.sameAsByType).filter(([type]) =>
|
|
8575
|
+
const hasEntitySchema = Object.keys(entity.entities.schemaTypes).some(isEntityEvidenceSchemaType);
|
|
8576
|
+
const siteSameAs = Object.entries(entity.entities.sameAsByType).filter(([type]) => isSiteIdentitySchemaType(type)).flatMap(([, urls]) => urls);
|
|
7543
8577
|
const uniqueSiteSameAs = [...new Set(siteSameAs)].sort();
|
|
7544
8578
|
const status = uniqueSiteSameAs.length ? "pass" : hasEntitySchema ? "warn" : "fail";
|
|
7545
8579
|
return {
|
|
@@ -7552,7 +8586,7 @@ function entityIdentityCheck(report) {
|
|
|
7552
8586
|
sameAs: entity.entities.sameAs,
|
|
7553
8587
|
socialProfiles: entity.entities.socialProfiles
|
|
7554
8588
|
},
|
|
7555
|
-
finding: status === "pass" ? "The crawl found sameAs links attached to Organization or LocalBusiness structured data for the site entity." : status === "warn" ? "Entity schema is present, but no Organization or LocalBusiness sameAs evidence connects the site entity to official profiles." : "No Organization, LocalBusiness, Person, Product, or WebSite schema was found on the evaluated pages.",
|
|
8589
|
+
finding: status === "pass" ? "The crawl found sameAs links attached to Organization or LocalBusiness structured data for the site entity." : status === "warn" ? "Entity schema is present, but no Organization or LocalBusiness sameAs evidence connects the site entity to official profiles." : "No Organization, LocalBusiness, Person, Product, SoftwareApplication, or WebSite schema was found on the evaluated pages.",
|
|
7556
8590
|
verification
|
|
7557
8591
|
};
|
|
7558
8592
|
}
|
|
@@ -8045,7 +9079,7 @@ const RULE_DEFINITIONS = [
|
|
|
8045
9079
|
whyItMatters: "External links are part of the page experience and source trail. Broken references make content feel stale and less trustworthy.",
|
|
8046
9080
|
howToFix: "Replace the link with a working source, update the URL, or remove the reference if it no longer supports the page.",
|
|
8047
9081
|
impactIfIgnored: "Readers and agents following citations hit dead ends, which weakens trust in the content.",
|
|
8048
|
-
howToVerify: "Re-run the crawl with external checks enabled and confirm the
|
|
9082
|
+
howToVerify: "Re-run the crawl with external checks enabled and confirm the URL no longer has a confirmed-broken externalLinkChecks result."
|
|
8049
9083
|
},
|
|
8050
9084
|
{
|
|
8051
9085
|
id: "orphan_page",
|
|
@@ -8706,11 +9740,13 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
8706
9740
|
if (page.hasHsts === false) issues.push(issue("hsts_missing", page, void 0, { header: "strict-transport-security" }));
|
|
8707
9741
|
}
|
|
8708
9742
|
for (const link of page.externalLinkChecks ?? []) {
|
|
8709
|
-
if (
|
|
9743
|
+
if (link.state !== "confirmed-broken") continue;
|
|
8710
9744
|
issues.push(issue("broken_external_link", page, link.url, {
|
|
8711
9745
|
url: link.url,
|
|
8712
9746
|
status: link.status,
|
|
8713
|
-
error: link.error
|
|
9747
|
+
error: link.error,
|
|
9748
|
+
state: link.state,
|
|
9749
|
+
attempts: link.attempts
|
|
8714
9750
|
}));
|
|
8715
9751
|
}
|
|
8716
9752
|
if (page.indexable && (page.internalInlinkCount ?? 0) === 0 && !sameUrl(page.url, opts.startUrl)) issues.push(issue("orphan_page", page, "No internal inlinks", { internalInlinkCount: page.internalInlinkCount ?? 0 }));
|
|
@@ -8845,6 +9881,80 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
8845
9881
|
return issues;
|
|
8846
9882
|
}
|
|
8847
9883
|
//#endregion
|
|
9884
|
+
//#region packages/core/src/analyze/crawler/crawl-skip-reasons.ts
|
|
9885
|
+
const CRAWL_SKIP_REASONS = [
|
|
9886
|
+
"asset-url",
|
|
9887
|
+
"configured-exclusion",
|
|
9888
|
+
"duplicate-normalization",
|
|
9889
|
+
"invalid-url",
|
|
9890
|
+
"off-origin",
|
|
9891
|
+
"origin-backpressure",
|
|
9892
|
+
"queue-safety-limit",
|
|
9893
|
+
"robots-disallowed",
|
|
9894
|
+
"robots-uncertain",
|
|
9895
|
+
"legacy-unclassified"
|
|
9896
|
+
];
|
|
9897
|
+
const SKIP_IMPACT = {
|
|
9898
|
+
"asset-url": "non-impacting",
|
|
9899
|
+
"configured-exclusion": "non-impacting",
|
|
9900
|
+
"duplicate-normalization": "non-impacting",
|
|
9901
|
+
"invalid-url": "non-impacting",
|
|
9902
|
+
"off-origin": "non-impacting",
|
|
9903
|
+
"origin-backpressure": "coverage-affecting",
|
|
9904
|
+
"queue-safety-limit": "coverage-affecting",
|
|
9905
|
+
"robots-disallowed": "coverage-affecting",
|
|
9906
|
+
"robots-uncertain": "coverage-affecting",
|
|
9907
|
+
"legacy-unclassified": "coverage-affecting"
|
|
9908
|
+
};
|
|
9909
|
+
const REASON_ORDER = new Map(CRAWL_SKIP_REASONS.map((reason, index) => [reason, index]));
|
|
9910
|
+
function isCrawlSkipReason(value) {
|
|
9911
|
+
return typeof value === "string" && CRAWL_SKIP_REASONS.includes(value);
|
|
9912
|
+
}
|
|
9913
|
+
function validCount$1(value) {
|
|
9914
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : 0;
|
|
9915
|
+
}
|
|
9916
|
+
function crawlSkipImpact(reason) {
|
|
9917
|
+
return SKIP_IMPACT[reason];
|
|
9918
|
+
}
|
|
9919
|
+
function normalizeCrawlSkipReasonCounts(input) {
|
|
9920
|
+
const counts = /* @__PURE__ */ new Map();
|
|
9921
|
+
if (Array.isArray(input.skipReasons)) for (const item of input.skipReasons) {
|
|
9922
|
+
if (!item || typeof item !== "object") continue;
|
|
9923
|
+
const { reason, count } = item;
|
|
9924
|
+
if (!isCrawlSkipReason(reason)) continue;
|
|
9925
|
+
const normalizedCount = validCount$1(count);
|
|
9926
|
+
if (!normalizedCount) continue;
|
|
9927
|
+
counts.set(reason, (counts.get(reason) ?? 0) + normalizedCount);
|
|
9928
|
+
}
|
|
9929
|
+
const declaredTotal = validCount$1(input.skippedUrls);
|
|
9930
|
+
const classifiedTotal = [...counts.values()].reduce((total, count) => total + count, 0);
|
|
9931
|
+
if (declaredTotal > classifiedTotal) counts.set("legacy-unclassified", (counts.get("legacy-unclassified") ?? 0) + declaredTotal - classifiedTotal);
|
|
9932
|
+
const skipReasons = [...counts.entries()].map(([reason, count]) => ({
|
|
9933
|
+
reason,
|
|
9934
|
+
impact: crawlSkipImpact(reason),
|
|
9935
|
+
count
|
|
9936
|
+
})).sort((left, right) => (REASON_ORDER.get(left.reason) ?? Number.MAX_SAFE_INTEGER) - (REASON_ORDER.get(right.reason) ?? Number.MAX_SAFE_INTEGER));
|
|
9937
|
+
const skippedUrlsByImpact = skipReasons.reduce((totals, item) => {
|
|
9938
|
+
if (item.impact === "coverage-affecting") totals.coverageAffecting += item.count;
|
|
9939
|
+
else totals.nonImpacting += item.count;
|
|
9940
|
+
return totals;
|
|
9941
|
+
}, {
|
|
9942
|
+
coverageAffecting: 0,
|
|
9943
|
+
nonImpacting: 0
|
|
9944
|
+
});
|
|
9945
|
+
return {
|
|
9946
|
+
skippedUrls: skippedUrlsByImpact.coverageAffecting + skippedUrlsByImpact.nonImpacting,
|
|
9947
|
+
skipReasons,
|
|
9948
|
+
skippedUrlsByImpact
|
|
9949
|
+
};
|
|
9950
|
+
}
|
|
9951
|
+
function crawlSkipReasonCountsFromRecord(input = {}) {
|
|
9952
|
+
return normalizeCrawlSkipReasonCounts({ skipReasons: Object.entries(input).map(([reason, count]) => ({
|
|
9953
|
+
reason,
|
|
9954
|
+
count
|
|
9955
|
+
})) }).skipReasons;
|
|
9956
|
+
}
|
|
9957
|
+
//#endregion
|
|
8848
9958
|
//#region packages/core/src/analyze/crawler/report.ts
|
|
8849
9959
|
const SENSITIVE_FIELD_NAMES = /* @__PURE__ */ new Set([
|
|
8850
9960
|
"authorization",
|
|
@@ -8883,6 +9993,7 @@ function normalizeCrawlConfig(input) {
|
|
|
8883
9993
|
respectRobots: input.respectRobots ?? true,
|
|
8884
9994
|
useSitemap: input.useSitemap ?? true,
|
|
8885
9995
|
checkExternal: input.checkExternal ?? true,
|
|
9996
|
+
checkAgentDiscovery: input.checkAgentDiscovery ?? false,
|
|
8886
9997
|
js: normalizeJavaScriptRenderingMode(input.js),
|
|
8887
9998
|
refresh: input.refresh ?? false,
|
|
8888
9999
|
fetchRate: normalizeFetchRate(input)
|
|
@@ -9002,12 +10113,13 @@ function summarizeCrawlReport(input) {
|
|
|
9002
10113
|
}
|
|
9003
10114
|
}
|
|
9004
10115
|
for (const issue of input.issues) byCategory[issue.category] = (byCategory[issue.category] ?? 0) + 1;
|
|
10116
|
+
const runStats = crawlRunStats(input.pages, input.stats);
|
|
9005
10117
|
return {
|
|
9006
10118
|
totalPages: input.pages.length,
|
|
9007
10119
|
indexablePages: input.pages.filter((page) => page.indexable).length,
|
|
9008
10120
|
nonIndexablePages: input.pages.filter((page) => !page.indexable).length,
|
|
9009
10121
|
statusErrors: input.pages.filter((page) => page.status === 0 || page.status >= 400).length,
|
|
9010
|
-
...
|
|
10122
|
+
...runStats,
|
|
9011
10123
|
attemptedRequests: input.requestEvidenceStatus === "unavailable" ? 0 : requests.length,
|
|
9012
10124
|
responseRequests: requests.filter((request) => request.outcome === "response").length,
|
|
9013
10125
|
failedRequests: failedRequests.length,
|
|
@@ -9032,11 +10144,17 @@ function crawlRunStats(pages, stats = {}) {
|
|
|
9032
10144
|
for (const url of page.sampleExternalLinks ?? []) discovered.add(url);
|
|
9033
10145
|
observedInternalLinks += page.outgoingInternalCount;
|
|
9034
10146
|
}
|
|
10147
|
+
const skips = normalizeCrawlSkipReasonCounts({
|
|
10148
|
+
skippedUrls: stats.skippedUrls,
|
|
10149
|
+
skipReasons: stats.skipReasons ?? crawlSkipReasonCountsFromRecord(stats.skipReasonCounts)
|
|
10150
|
+
});
|
|
9035
10151
|
return {
|
|
9036
10152
|
discoveredUrls: stats.discoveredUrls ?? discovered.size,
|
|
9037
10153
|
queuedUrls: stats.queuedUrls ?? pages.length,
|
|
9038
10154
|
crawledUrls: stats.crawledUrls ?? pages.length,
|
|
9039
|
-
skippedUrls:
|
|
10155
|
+
skippedUrls: skips.skippedUrls,
|
|
10156
|
+
skipReasons: skips.skipReasons,
|
|
10157
|
+
skippedUrlsByImpact: skips.skippedUrlsByImpact,
|
|
9040
10158
|
failedUrls: stats.failedUrls ?? pages.filter((page) => page.status === 0 || page.status >= 400).length,
|
|
9041
10159
|
observedInternalLinks: stats.observedInternalLinks ?? observedInternalLinks,
|
|
9042
10160
|
pageLimitReached: stats.pageLimitReached ?? false
|
|
@@ -9126,6 +10244,7 @@ function createCrawlReport(input) {
|
|
|
9126
10244
|
...input.ai ? { ai: sanitizeTenantValue(input.ai) } : {},
|
|
9127
10245
|
...input.sitemapDiscovery ? { sitemapDiscovery: sanitizeTenantValue(input.sitemapDiscovery) } : {},
|
|
9128
10246
|
...input.externalLinkVerification ? { externalLinkVerification: sanitizeTenantValue(input.externalLinkVerification) } : {},
|
|
10247
|
+
...input.agentDiscovery ? { agentDiscovery: sanitizeTenantValue(input.agentDiscovery) } : {},
|
|
9129
10248
|
warnings: sanitizeMessages(input.warnings).sort(compareText$13),
|
|
9130
10249
|
caveats: sanitizeMessages(input.caveats).sort(compareText$13)
|
|
9131
10250
|
};
|
|
@@ -9160,7 +10279,7 @@ function normalizeLoadedCrawlReport(report) {
|
|
|
9160
10279
|
}
|
|
9161
10280
|
//#endregion
|
|
9162
10281
|
//#region packages/core/src/analyze/crawler/report-store.ts
|
|
9163
|
-
const CRAWL_REPORT_STORAGE_VERSION =
|
|
10282
|
+
const CRAWL_REPORT_STORAGE_VERSION = 6;
|
|
9164
10283
|
function toMeta(row) {
|
|
9165
10284
|
return {
|
|
9166
10285
|
id: row.id,
|
|
@@ -9180,7 +10299,7 @@ function storageEnvelope(report) {
|
|
|
9180
10299
|
if (report.site) tenant.site = report.site;
|
|
9181
10300
|
return {
|
|
9182
10301
|
kind: "seo.crawl_report",
|
|
9183
|
-
version:
|
|
10302
|
+
version: 6,
|
|
9184
10303
|
savedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9185
10304
|
...tenant.projectId || tenant.site ? { tenant } : {},
|
|
9186
10305
|
report
|
|
@@ -9216,7 +10335,7 @@ function saveCrawlReportToSqlite(report) {
|
|
|
9216
10335
|
totalPages: report.summary.totalPages,
|
|
9217
10336
|
issueCount: report.issues.length,
|
|
9218
10337
|
createdAt: report.generatedAt,
|
|
9219
|
-
storageVersion:
|
|
10338
|
+
storageVersion: 6
|
|
9220
10339
|
};
|
|
9221
10340
|
}
|
|
9222
10341
|
function listCrawlReportsFromSqlite(input = {}) {
|
|
@@ -9303,6 +10422,15 @@ function cancellationRace(signal) {
|
|
|
9303
10422
|
//#endregion
|
|
9304
10423
|
//#region packages/core/src/analyze/crawler/external-link-checks.ts
|
|
9305
10424
|
const EXTERNAL_LINK_SELECTION_LIMIT = 200;
|
|
10425
|
+
const EXTERNAL_LINK_STATES = [
|
|
10426
|
+
"available",
|
|
10427
|
+
"confirmed-broken",
|
|
10428
|
+
"transient",
|
|
10429
|
+
"provider-blocked",
|
|
10430
|
+
"rate-limited",
|
|
10431
|
+
"method-rejected",
|
|
10432
|
+
"unavailable"
|
|
10433
|
+
];
|
|
9306
10434
|
function stableHash(value) {
|
|
9307
10435
|
let hash = 2166136261;
|
|
9308
10436
|
for (const character of value) hash = Math.imul(hash ^ (character.codePointAt(0) ?? 0), 16777619);
|
|
@@ -9372,32 +10500,73 @@ function selectCandidateUrls(candidates, limit) {
|
|
|
9372
10500
|
}
|
|
9373
10501
|
return [...selected];
|
|
9374
10502
|
}
|
|
9375
|
-
|
|
10503
|
+
function isAvailableStatus(status) {
|
|
10504
|
+
return status !== void 0 && status >= 200 && status < 400;
|
|
10505
|
+
}
|
|
10506
|
+
function isBrokenStatus(status) {
|
|
10507
|
+
return status === 404 || status === 410;
|
|
10508
|
+
}
|
|
10509
|
+
function isMethodRejectedStatus(status) {
|
|
10510
|
+
return status === 405 || status === 501;
|
|
10511
|
+
}
|
|
10512
|
+
function isProviderBlockedStatus(status) {
|
|
10513
|
+
return status === 401 || status === 403 || status === 451;
|
|
10514
|
+
}
|
|
10515
|
+
function isTransientStatus(status) {
|
|
10516
|
+
return status === 408 || status === 425 || status !== void 0 && status >= 500;
|
|
10517
|
+
}
|
|
10518
|
+
function stateForAttempts(attempts) {
|
|
10519
|
+
const last = attempts.at(-1);
|
|
10520
|
+
const statuses = attempts.map((attempt) => attempt.status).filter((status) => status !== void 0);
|
|
10521
|
+
if (isAvailableStatus(last?.status)) return isMethodRejectedStatus(attempts[0]?.status) ? "method-rejected" : attempts.length > 1 ? "transient" : "available";
|
|
10522
|
+
if (attempts.length >= 2 && isBrokenStatus(last?.status) && attempts.at(-2)?.status === last?.status) return "confirmed-broken";
|
|
10523
|
+
if (last?.status === 429) return "rate-limited";
|
|
10524
|
+
if (isProviderBlockedStatus(last?.status)) return "provider-blocked";
|
|
10525
|
+
if (isMethodRejectedStatus(last?.status)) return "method-rejected";
|
|
10526
|
+
if (isTransientStatus(last?.status) || statuses.some(isBrokenStatus) || statuses.length > 0 && attempts.some((attempt) => attempt.error)) return "transient";
|
|
10527
|
+
return "unavailable";
|
|
10528
|
+
}
|
|
10529
|
+
async function requestExternalLink(url, method, timeoutMs, fetch, signal) {
|
|
9376
10530
|
const controller = abortController({
|
|
9377
10531
|
timeoutMs,
|
|
9378
10532
|
signal
|
|
9379
10533
|
});
|
|
9380
10534
|
try {
|
|
9381
10535
|
const response = await fetch(url, {
|
|
9382
|
-
method
|
|
10536
|
+
method,
|
|
9383
10537
|
redirect: "follow",
|
|
9384
10538
|
signal: controller.signal
|
|
9385
10539
|
});
|
|
9386
10540
|
await response.body?.cancel().catch(() => void 0);
|
|
9387
10541
|
return {
|
|
9388
|
-
|
|
10542
|
+
method,
|
|
9389
10543
|
status: response.status
|
|
9390
10544
|
};
|
|
9391
10545
|
} catch (error) {
|
|
9392
10546
|
return {
|
|
9393
|
-
|
|
9394
|
-
status: 0,
|
|
10547
|
+
method,
|
|
9395
10548
|
error: error instanceof Error ? error.message : String(error)
|
|
9396
10549
|
};
|
|
9397
10550
|
} finally {
|
|
9398
10551
|
controller.cleanup();
|
|
9399
10552
|
}
|
|
9400
10553
|
}
|
|
10554
|
+
async function checkExternalLink(url, timeoutMs, fetch, signal) {
|
|
10555
|
+
const attempts = [];
|
|
10556
|
+
attempts.push(await requestExternalLink(url, "HEAD", timeoutMs, fetch, signal));
|
|
10557
|
+
const head = attempts[0];
|
|
10558
|
+
if (!signal?.aborted && !isAvailableStatus(head?.status) && head?.status !== 429 && !isProviderBlockedStatus(head?.status)) attempts.push(await requestExternalLink(url, "GET", timeoutMs, fetch, signal));
|
|
10559
|
+
const last = attempts.at(-1);
|
|
10560
|
+
if (!signal?.aborted && isBrokenStatus(last?.status) && attempts.at(-2)?.status !== last?.status) attempts.push(await requestExternalLink(url, "GET", timeoutMs, fetch, signal));
|
|
10561
|
+
const finalAttempt = attempts.at(-1);
|
|
10562
|
+
return {
|
|
10563
|
+
url,
|
|
10564
|
+
...finalAttempt?.status !== void 0 ? { status: finalAttempt.status } : {},
|
|
10565
|
+
...finalAttempt?.error ? { error: finalAttempt.error } : {},
|
|
10566
|
+
state: stateForAttempts(attempts),
|
|
10567
|
+
attempts
|
|
10568
|
+
};
|
|
10569
|
+
}
|
|
9401
10570
|
async function verifyExternalLinks(input) {
|
|
9402
10571
|
const { candidates, discoveredLinkOccurrences, retainedLinkOccurrences } = collectCandidates(input.pages);
|
|
9403
10572
|
const urls = input.signal?.aborted ? [] : selectCandidateUrls(candidates, EXTERNAL_LINK_SELECTION_LIMIT);
|
|
@@ -9412,16 +10581,22 @@ async function verifyExternalLinks(input) {
|
|
|
9412
10581
|
const pageChecks = (page.sampleExternalLinks ?? []).map((url) => checks.get(url)).filter((value) => Boolean(value));
|
|
9413
10582
|
if (pageChecks.length) page.externalLinkChecks = pageChecks;
|
|
9414
10583
|
}
|
|
9415
|
-
const fetchedUrls = [...checks.values()].filter((check) =>
|
|
9416
|
-
const failedUrls = checks.
|
|
10584
|
+
const fetchedUrls = [...checks.values()].filter((check) => check.attempts?.some((attempt) => attempt.status !== void 0)).length;
|
|
10585
|
+
const failedUrls = [...checks.values()].filter((check) => check.state === "unavailable").length;
|
|
10586
|
+
const unresolvedMethodRejectedUrls = [...checks.values()].filter((check) => check.state === "method-rejected" && !isAvailableStatus(check.status)).length;
|
|
9417
10587
|
const deferredUrls = candidates.length - checks.size;
|
|
10588
|
+
const outcomes = Object.fromEntries(EXTERNAL_LINK_STATES.map((state) => [state, [...checks.values()].filter((check) => check.state === state).length]));
|
|
9418
10589
|
const warnings = [];
|
|
9419
10590
|
if (discoveredLinkOccurrences > retainedLinkOccurrences) warnings.push(`Retained ${retainedLinkOccurrences} sampled external link occurrence${retainedLinkOccurrences === 1 ? "" : "s"} from ${discoveredLinkOccurrences} observed occurrence${discoveredLinkOccurrences === 1 ? "" : "s"}.`);
|
|
9420
10591
|
if (candidates.length > urls.length) warnings.push(`Selected ${urls.length} of ${candidates.length} retained external URLs using the ${EXTERNAL_LINK_SELECTION_LIMIT}-URL verification limit.`);
|
|
9421
10592
|
if (checks.size < urls.length) warnings.push(`Stopped before checking ${urls.length - checks.size} selected external URL${urls.length - checks.size === 1 ? "" : "s"}.`);
|
|
9422
|
-
if (failedUrls) warnings.push(`${failedUrls} external URL
|
|
10593
|
+
if (failedUrls) warnings.push(`${failedUrls} external URL${failedUrls === 1 ? "" : "s"} could not be reached after bounded verification.`);
|
|
10594
|
+
if (outcomes.transient) warnings.push(`${outcomes.transient} external URL${outcomes.transient === 1 ? "" : "s"} returned inconsistent or temporary responses. Recheck ${outcomes.transient === 1 ? "it" : "them"} before changing the source page.`);
|
|
10595
|
+
if (outcomes["provider-blocked"]) warnings.push(`${outcomes["provider-blocked"]} external URL${outcomes["provider-blocked"] === 1 ? "" : "s"} blocked automated verification. Open ${outcomes["provider-blocked"] === 1 ? "it" : "them"} manually before changing the source page.`);
|
|
10596
|
+
if (outcomes["rate-limited"]) warnings.push(`${outcomes["rate-limited"]} external URL${outcomes["rate-limited"] === 1 ? "" : "s"} rate-limited verification. Recheck ${outcomes["rate-limited"] === 1 ? "it" : "them"} later.`);
|
|
10597
|
+
if (unresolvedMethodRejectedUrls) warnings.push(`${unresolvedMethodRejectedUrls} external URL${unresolvedMethodRejectedUrls === 1 ? "" : "s"} rejected both verification methods. Open ${unresolvedMethodRejectedUrls === 1 ? "it" : "them"} manually before changing the source page.`);
|
|
9423
10598
|
return {
|
|
9424
|
-
dataStatus: discoveredLinkOccurrences > retainedLinkOccurrences || deferredUrls > 0 || failedUrls > 0 ? "partial" : "complete",
|
|
10599
|
+
dataStatus: discoveredLinkOccurrences > retainedLinkOccurrences || deferredUrls > 0 || failedUrls > 0 || outcomes.transient > 0 || outcomes["provider-blocked"] > 0 || outcomes["rate-limited"] > 0 || unresolvedMethodRejectedUrls > 0 ? "partial" : "complete",
|
|
9425
10600
|
discoveredLinkOccurrences,
|
|
9426
10601
|
retainedUrls: candidates.length,
|
|
9427
10602
|
selectedUrls: urls.length,
|
|
@@ -9429,6 +10604,7 @@ async function verifyExternalLinks(input) {
|
|
|
9429
10604
|
failedUrls,
|
|
9430
10605
|
deferredUrls,
|
|
9431
10606
|
limit: EXTERNAL_LINK_SELECTION_LIMIT,
|
|
10607
|
+
outcomes,
|
|
9432
10608
|
warnings
|
|
9433
10609
|
};
|
|
9434
10610
|
}
|
|
@@ -10029,6 +11205,7 @@ async function crawlOne(url, opts = {}) {
|
|
|
10029
11205
|
sampleExternalLinks: uniqueExternalLinks.slice(0, 25),
|
|
10030
11206
|
internalAnchorSamples: anchorSamples(extracted.links, true),
|
|
10031
11207
|
externalAnchorSamples: anchorSamples(extracted.links, false),
|
|
11208
|
+
markdownAlternates: extracted.markdownAlternates,
|
|
10032
11209
|
schemaTypes: extracted.schemaTypes,
|
|
10033
11210
|
structuredDataFormats: extracted.structuredDataFormats ?? [],
|
|
10034
11211
|
googleRichResults: richResults.assessments,
|
|
@@ -10624,6 +11801,7 @@ function resolveCrawlSiteDependencies(dependencies = {}) {
|
|
|
10624
11801
|
queryPagesTopQueriesBatch: injectedMode === "batch" ? dependencies.queryPagesTopQueriesBatch : injectedMode ? void 0 : queryPagesTopQueriesBatch,
|
|
10625
11802
|
fetchLandingPageValues: dependencies.fetchLandingPageValues ?? fetchLandingPageValues,
|
|
10626
11803
|
landingValueForUrl: dependencies.landingValueForUrl ?? landingValueForUrl,
|
|
11804
|
+
collectAgentDiscovery: dependencies.collectAgentDiscovery ?? collectAgentDiscovery,
|
|
10627
11805
|
now: dependencies.now ?? (() => /* @__PURE__ */ new Date())
|
|
10628
11806
|
};
|
|
10629
11807
|
}
|
|
@@ -10949,6 +12127,11 @@ async function crawlSite(input, dependencies) {
|
|
|
10949
12127
|
const followLinks = config.mode === "site";
|
|
10950
12128
|
let queuedUrls = 0;
|
|
10951
12129
|
let skippedUrls = 0;
|
|
12130
|
+
const skipReasonCounts = {};
|
|
12131
|
+
const recordSkip = (reason) => {
|
|
12132
|
+
skippedUrls += 1;
|
|
12133
|
+
skipReasonCounts[reason] = (skipReasonCounts[reason] ?? 0) + 1;
|
|
12134
|
+
};
|
|
10952
12135
|
let robotsDeferredUrls = 0;
|
|
10953
12136
|
let originBackpressureSkippedUrls = 0;
|
|
10954
12137
|
let queueSafetySkippedUrls = 0;
|
|
@@ -11015,7 +12198,7 @@ async function crawlSite(input, dependencies) {
|
|
|
11015
12198
|
const enqueue = (value, depth) => {
|
|
11016
12199
|
const normalized = normalizeUrl$1(value, config.url);
|
|
11017
12200
|
if (!normalized) {
|
|
11018
|
-
|
|
12201
|
+
recordSkip("invalid-url");
|
|
11019
12202
|
emitStatus("url_skipped", {
|
|
11020
12203
|
url: value,
|
|
11021
12204
|
depth,
|
|
@@ -11032,7 +12215,7 @@ async function crawlSite(input, dependencies) {
|
|
|
11032
12215
|
return;
|
|
11033
12216
|
}
|
|
11034
12217
|
if (!sameOrigin(normalized, origin)) {
|
|
11035
|
-
|
|
12218
|
+
recordSkip("off-origin");
|
|
11036
12219
|
emitStatus("url_skipped", {
|
|
11037
12220
|
url: normalized,
|
|
11038
12221
|
depth,
|
|
@@ -11041,7 +12224,7 @@ async function crawlSite(input, dependencies) {
|
|
|
11041
12224
|
return;
|
|
11042
12225
|
}
|
|
11043
12226
|
if (isLikelyAsset(normalized)) {
|
|
11044
|
-
|
|
12227
|
+
recordSkip("asset-url");
|
|
11045
12228
|
emitStatus("url_skipped", {
|
|
11046
12229
|
url: normalized,
|
|
11047
12230
|
depth,
|
|
@@ -11050,7 +12233,7 @@ async function crawlSite(input, dependencies) {
|
|
|
11050
12233
|
return;
|
|
11051
12234
|
}
|
|
11052
12235
|
if (!passesFilters(normalized, config.include, config.exclude)) {
|
|
11053
|
-
|
|
12236
|
+
recordSkip("configured-exclusion");
|
|
11054
12237
|
emitStatus("url_skipped", {
|
|
11055
12238
|
url: normalized,
|
|
11056
12239
|
depth,
|
|
@@ -11059,7 +12242,7 @@ async function crawlSite(input, dependencies) {
|
|
|
11059
12242
|
return;
|
|
11060
12243
|
}
|
|
11061
12244
|
if (queue.length + visited.size + inFlight.size >= config.maxPages * 5) {
|
|
11062
|
-
|
|
12245
|
+
recordSkip("queue-safety-limit");
|
|
11063
12246
|
queueSafetySkippedUrls += 1;
|
|
11064
12247
|
emitStatus("url_skipped", {
|
|
11065
12248
|
url: normalized,
|
|
@@ -11168,7 +12351,7 @@ async function crawlSite(input, dependencies) {
|
|
|
11168
12351
|
});
|
|
11169
12352
|
requests.push(request);
|
|
11170
12353
|
if (request.outcome === "skipped") {
|
|
11171
|
-
|
|
12354
|
+
recordSkip(request.reason === "robots-disallowed" ? "robots-disallowed" : request.reason === "robots-deferred" ? "robots-uncertain" : "origin-backpressure");
|
|
11172
12355
|
if (request.reason === "robots-deferred") {
|
|
11173
12356
|
robotsDeferredUrls += 1;
|
|
11174
12357
|
warnings.push(`${task.url}: crawl deferred because robots.txt availability is unknown.`);
|
|
@@ -11273,7 +12456,7 @@ async function crawlSite(input, dependencies) {
|
|
|
11273
12456
|
page.declaredIndexability = "robots-blocked";
|
|
11274
12457
|
storeDocument([]);
|
|
11275
12458
|
warnings.push(`${page.url}: skipped because robots.txt disallows it`);
|
|
11276
|
-
|
|
12459
|
+
recordSkip("robots-disallowed");
|
|
11277
12460
|
emitStatus("page_skipped", {
|
|
11278
12461
|
url: page.url,
|
|
11279
12462
|
depth: task.depth,
|
|
@@ -11360,6 +12543,14 @@ async function crawlSite(input, dependencies) {
|
|
|
11360
12543
|
message: "Finished external link checks."
|
|
11361
12544
|
});
|
|
11362
12545
|
}
|
|
12546
|
+
const agentDiscovery = !cancelled && config.checkAgentDiscovery ? await deps.collectAgentDiscovery({
|
|
12547
|
+
startUrl: config.url,
|
|
12548
|
+
pages,
|
|
12549
|
+
timeoutMs: config.timeoutMs,
|
|
12550
|
+
fetch: deps.fetch,
|
|
12551
|
+
signal,
|
|
12552
|
+
concurrency: Math.min(config.concurrency, 4)
|
|
12553
|
+
}) : void 0;
|
|
11363
12554
|
for (const page of pages) {
|
|
11364
12555
|
if (!page.geo) continue;
|
|
11365
12556
|
page.geo = {
|
|
@@ -11386,6 +12577,7 @@ async function crawlSite(input, dependencies) {
|
|
|
11386
12577
|
},
|
|
11387
12578
|
...sitemapDiscovery ? { sitemapDiscovery } : {},
|
|
11388
12579
|
...externalLinkVerification ? { externalLinkVerification } : {},
|
|
12580
|
+
...agentDiscovery ? { agentDiscovery } : {},
|
|
11389
12581
|
status: reportStatus,
|
|
11390
12582
|
warnings,
|
|
11391
12583
|
caveats: crawlCaveats({
|
|
@@ -11400,6 +12592,7 @@ async function crawlSite(input, dependencies) {
|
|
|
11400
12592
|
queuedUrls,
|
|
11401
12593
|
crawledUrls: pages.length,
|
|
11402
12594
|
skippedUrls,
|
|
12595
|
+
skipReasonCounts,
|
|
11403
12596
|
failedUrls,
|
|
11404
12597
|
observedInternalLinks,
|
|
11405
12598
|
pageLimitReached
|
|
@@ -12884,7 +14077,7 @@ function completenessReasons(report) {
|
|
|
12884
14077
|
if (report.status !== "completed") reasons.push(`report-status-${report.status}`);
|
|
12885
14078
|
if (report.summary.pageLimitReached) reasons.push("crawl-page-limit-reached");
|
|
12886
14079
|
if (report.requestEvidenceStatus !== "available") reasons.push(`request-evidence-${report.requestEvidenceStatus}`);
|
|
12887
|
-
if (report.summary.
|
|
14080
|
+
if (report.summary.skippedUrlsByImpact.coverageAffecting > 0) reasons.push("coverage-affecting-urls-skipped");
|
|
12888
14081
|
if (report.summary.failedRequests > 0) reasons.push("requests-failed");
|
|
12889
14082
|
if (report.summary.abortedRequests > 0) reasons.push("requests-aborted");
|
|
12890
14083
|
if (report.summary.extractionFailures > 0) reasons.push("document-extraction-failed");
|
|
@@ -12904,6 +14097,8 @@ function snapshotCompleteness(report) {
|
|
|
12904
14097
|
requestEvidenceStatus: report.requestEvidenceStatus,
|
|
12905
14098
|
pageLimitReached: report.summary.pageLimitReached,
|
|
12906
14099
|
skippedUrls: report.summary.skippedUrls,
|
|
14100
|
+
skipReasons: report.summary.skipReasons,
|
|
14101
|
+
skippedUrlsByImpact: report.summary.skippedUrlsByImpact,
|
|
12907
14102
|
failedRequests: report.summary.failedRequests,
|
|
12908
14103
|
abortedRequests: report.summary.abortedRequests,
|
|
12909
14104
|
extractionFailures: report.summary.extractionFailures,
|
|
@@ -13539,6 +14734,169 @@ function explainOkfValidation(validation) {
|
|
|
13539
14734
|
};
|
|
13540
14735
|
}
|
|
13541
14736
|
//#endregion
|
|
14737
|
+
//#region packages/core/src/analyze/crawler/agent-discovery-schema.ts
|
|
14738
|
+
const agentRepresentationResponseSchema = z.object({
|
|
14739
|
+
requestedUrl: z.string().url(),
|
|
14740
|
+
finalUrl: z.string().url().optional(),
|
|
14741
|
+
status: z.number().int().optional(),
|
|
14742
|
+
contentType: z.string().optional(),
|
|
14743
|
+
bytes: z.number().int().nonnegative().optional(),
|
|
14744
|
+
sha256: z.string().optional(),
|
|
14745
|
+
canonicalUrl: z.string().url().optional(),
|
|
14746
|
+
alternateUrl: z.string().url().optional(),
|
|
14747
|
+
varyAccept: z.boolean(),
|
|
14748
|
+
markdownTokens: z.number().int().nonnegative().optional(),
|
|
14749
|
+
contentSignal: z.string().optional(),
|
|
14750
|
+
error: z.string().optional()
|
|
14751
|
+
});
|
|
14752
|
+
const agentDiscoverySchema = z.object({
|
|
14753
|
+
profile: z.literal("content"),
|
|
14754
|
+
profileApplicability: z.record(z.enum([
|
|
14755
|
+
"content",
|
|
14756
|
+
"api",
|
|
14757
|
+
"application",
|
|
14758
|
+
"commerce"
|
|
14759
|
+
]), z.object({
|
|
14760
|
+
status: z.enum(["evaluated", "notApplicable"]),
|
|
14761
|
+
reason: z.string()
|
|
14762
|
+
})),
|
|
14763
|
+
dataStatus: z.enum([
|
|
14764
|
+
"complete",
|
|
14765
|
+
"partial",
|
|
14766
|
+
"unavailable"
|
|
14767
|
+
]),
|
|
14768
|
+
markdownAlternates: z.object({
|
|
14769
|
+
eligibleHtmlPages: z.number().int().nonnegative(),
|
|
14770
|
+
advertisedPages: z.number().int().nonnegative(),
|
|
14771
|
+
evaluatedPages: z.number().int().nonnegative(),
|
|
14772
|
+
exactByteMatches: z.number().int().nonnegative(),
|
|
14773
|
+
stableResponses: z.number().int().nonnegative(),
|
|
14774
|
+
pages: z.array(z.object({
|
|
14775
|
+
htmlUrl: z.string().url(),
|
|
14776
|
+
advertisedUrls: z.array(z.string().url()),
|
|
14777
|
+
htmlAlternateUnique: z.boolean(),
|
|
14778
|
+
httpAlternateUrls: z.array(z.string().url()),
|
|
14779
|
+
explicit: agentRepresentationResponseSchema.optional(),
|
|
14780
|
+
negotiated: agentRepresentationResponseSchema.optional(),
|
|
14781
|
+
repeated: agentRepresentationResponseSchema.optional(),
|
|
14782
|
+
explicitMatchesNegotiated: z.boolean().nullable(),
|
|
14783
|
+
repeatedHashStable: z.boolean().nullable(),
|
|
14784
|
+
markdownCanonicalMatchesHtml: z.boolean().nullable(),
|
|
14785
|
+
quality: z.object({
|
|
14786
|
+
frontmatterTitle: z.boolean(),
|
|
14787
|
+
h1Count: z.number().int().nonnegative(),
|
|
14788
|
+
codeFenceBalanced: z.boolean(),
|
|
14789
|
+
tableRows: z.number().int().nonnegative(),
|
|
14790
|
+
links: z.number().int().nonnegative(),
|
|
14791
|
+
wordCount: z.number().int().nonnegative(),
|
|
14792
|
+
rawHtmlTags: z.number().int().nonnegative(),
|
|
14793
|
+
rawSvgTags: z.number().int().nonnegative(),
|
|
14794
|
+
rawScriptTags: z.number().int().nonnegative(),
|
|
14795
|
+
rawStyleTags: z.number().int().nonnegative(),
|
|
14796
|
+
suspiciousConcatenations: z.number().int().nonnegative(),
|
|
14797
|
+
repeatedLines: z.number().int().nonnegative(),
|
|
14798
|
+
sourceWordCount: z.number().int().nonnegative(),
|
|
14799
|
+
wordRetentionRatio: z.number().nonnegative().nullable(),
|
|
14800
|
+
introductoryCopyRetained: z.boolean().nullable(),
|
|
14801
|
+
navigationOnly: z.boolean()
|
|
14802
|
+
}).optional()
|
|
14803
|
+
}))
|
|
14804
|
+
}),
|
|
14805
|
+
contentNegotiation: z.object({
|
|
14806
|
+
qZeroHonoured: z.boolean().nullable(),
|
|
14807
|
+
qZeroStatus: z.number().int().optional(),
|
|
14808
|
+
qZeroContentType: z.string().optional(),
|
|
14809
|
+
error: z.string().optional()
|
|
14810
|
+
}),
|
|
14811
|
+
routeManifest: z.object({
|
|
14812
|
+
url: z.string().url(),
|
|
14813
|
+
status: z.number().int().optional(),
|
|
14814
|
+
valid: z.boolean(),
|
|
14815
|
+
declaredHtmlRoutes: z.array(z.string()),
|
|
14816
|
+
declaredMarkdownRoutes: z.array(z.string()),
|
|
14817
|
+
missingHtmlRoutes: z.array(z.string()),
|
|
14818
|
+
missingMarkdownRoutes: z.array(z.string()),
|
|
14819
|
+
orphanMarkdownRoutes: z.array(z.string()),
|
|
14820
|
+
error: z.string().optional()
|
|
14821
|
+
}),
|
|
14822
|
+
agentSkills: z.object({
|
|
14823
|
+
indexUrl: z.string().url(),
|
|
14824
|
+
status: z.number().int().optional(),
|
|
14825
|
+
contentType: z.string().optional(),
|
|
14826
|
+
cors: z.string().optional(),
|
|
14827
|
+
validIndex: z.boolean(),
|
|
14828
|
+
skills: z.array(z.object({
|
|
14829
|
+
name: z.string().optional(),
|
|
14830
|
+
url: z.string().url().optional(),
|
|
14831
|
+
status: z.number().int().optional(),
|
|
14832
|
+
contentType: z.string().optional(),
|
|
14833
|
+
declaredDigest: z.string().optional(),
|
|
14834
|
+
observedDigest: z.string().optional(),
|
|
14835
|
+
digestMatches: z.boolean().nullable(),
|
|
14836
|
+
frontmatterValid: z.boolean().nullable(),
|
|
14837
|
+
sameOrigin: z.boolean(),
|
|
14838
|
+
cors: z.string().optional(),
|
|
14839
|
+
error: z.string().optional()
|
|
14840
|
+
})),
|
|
14841
|
+
error: z.string().optional()
|
|
14842
|
+
}),
|
|
14843
|
+
llmsTxt: z.object({
|
|
14844
|
+
url: z.string().url(),
|
|
14845
|
+
exists: z.boolean(),
|
|
14846
|
+
status: z.number().int().optional(),
|
|
14847
|
+
contentType: z.string().optional(),
|
|
14848
|
+
bytes: z.number().int().nonnegative().optional(),
|
|
14849
|
+
sha256: z.string().optional(),
|
|
14850
|
+
repeatedHashStable: z.boolean().nullable(),
|
|
14851
|
+
headingCount: z.number().int().nonnegative(),
|
|
14852
|
+
totalParsedLinks: z.number().int().nonnegative(),
|
|
14853
|
+
linkLimitReached: z.boolean(),
|
|
14854
|
+
links: z.array(z.object({
|
|
14855
|
+
label: z.string(),
|
|
14856
|
+
url: z.string().url(),
|
|
14857
|
+
sameOrigin: z.boolean(),
|
|
14858
|
+
status: z.number().int().optional(),
|
|
14859
|
+
finalUrl: z.string().url().optional(),
|
|
14860
|
+
redirected: z.boolean(),
|
|
14861
|
+
indexableTarget: z.boolean().optional(),
|
|
14862
|
+
error: z.string().optional()
|
|
14863
|
+
})),
|
|
14864
|
+
invalidLinks: z.array(z.string()),
|
|
14865
|
+
duplicateLinks: z.array(z.string().url()),
|
|
14866
|
+
offSiteLinks: z.array(z.string().url()),
|
|
14867
|
+
redirectedLinks: z.array(z.string().url()),
|
|
14868
|
+
nonIndexableLinks: z.array(z.string().url()),
|
|
14869
|
+
missingCrawlRoutes: z.array(z.string().url()),
|
|
14870
|
+
oversized: z.boolean(),
|
|
14871
|
+
error: z.string().optional()
|
|
14872
|
+
}),
|
|
14873
|
+
contentSignals: z.object({
|
|
14874
|
+
htmlValues: z.array(z.string()),
|
|
14875
|
+
markdownValues: z.array(z.string()),
|
|
14876
|
+
missingHtmlPages: z.number().int().nonnegative(),
|
|
14877
|
+
missingMarkdownPages: z.number().int().nonnegative(),
|
|
14878
|
+
consistent: z.boolean().nullable()
|
|
14879
|
+
}),
|
|
14880
|
+
protocolVariants: z.object({
|
|
14881
|
+
http: z.object({
|
|
14882
|
+
url: z.string().url(),
|
|
14883
|
+
status: z.number().int().optional(),
|
|
14884
|
+
location: z.string().url().optional(),
|
|
14885
|
+
permanentRedirectToHttps: z.boolean().nullable(),
|
|
14886
|
+
error: z.string().optional()
|
|
14887
|
+
}),
|
|
14888
|
+
www: z.object({
|
|
14889
|
+
url: z.string().url(),
|
|
14890
|
+
status: z.number().int().optional(),
|
|
14891
|
+
location: z.string().url().optional(),
|
|
14892
|
+
redirectsToPreferredHost: z.boolean().nullable(),
|
|
14893
|
+
error: z.string().optional()
|
|
14894
|
+
}),
|
|
14895
|
+
hstsOnStartPage: z.boolean().nullable()
|
|
14896
|
+
}),
|
|
14897
|
+
warnings: z.array(z.string())
|
|
14898
|
+
});
|
|
14899
|
+
//#endregion
|
|
13542
14900
|
//#region packages/core/src/analyze/crawler/schemas.ts
|
|
13543
14901
|
const crawlerRuleSeveritySchema = z.enum([
|
|
13544
14902
|
"low",
|
|
@@ -13963,10 +15321,25 @@ const crawlPageSnapshotSchema = z.object({
|
|
|
13963
15321
|
href: z.string().url(),
|
|
13964
15322
|
text: z.string()
|
|
13965
15323
|
})).optional(),
|
|
15324
|
+
markdownAlternates: z.array(z.string().url()).optional(),
|
|
13966
15325
|
externalLinkChecks: z.array(z.object({
|
|
13967
15326
|
url: z.string().url(),
|
|
13968
15327
|
status: z.number().int().optional(),
|
|
13969
|
-
error: z.string().optional()
|
|
15328
|
+
error: z.string().optional(),
|
|
15329
|
+
state: z.enum([
|
|
15330
|
+
"available",
|
|
15331
|
+
"confirmed-broken",
|
|
15332
|
+
"transient",
|
|
15333
|
+
"provider-blocked",
|
|
15334
|
+
"rate-limited",
|
|
15335
|
+
"method-rejected",
|
|
15336
|
+
"unavailable"
|
|
15337
|
+
]).optional(),
|
|
15338
|
+
attempts: z.array(z.object({
|
|
15339
|
+
method: z.enum(["HEAD", "GET"]),
|
|
15340
|
+
status: z.number().int().optional(),
|
|
15341
|
+
error: z.string().optional()
|
|
15342
|
+
})).optional()
|
|
13970
15343
|
})).optional(),
|
|
13971
15344
|
schemaTypes: z.array(z.string()).optional(),
|
|
13972
15345
|
structuredDataFormats: z.array(z.enum([
|
|
@@ -14247,6 +15620,15 @@ const crawlExternalLinkVerificationSchema = z.object({
|
|
|
14247
15620
|
failedUrls: z.number().int().nonnegative(),
|
|
14248
15621
|
deferredUrls: z.number().int().nonnegative(),
|
|
14249
15622
|
limit: z.number().int().positive(),
|
|
15623
|
+
outcomes: z.record(z.enum([
|
|
15624
|
+
"available",
|
|
15625
|
+
"confirmed-broken",
|
|
15626
|
+
"transient",
|
|
15627
|
+
"provider-blocked",
|
|
15628
|
+
"rate-limited",
|
|
15629
|
+
"method-rejected",
|
|
15630
|
+
"unavailable"
|
|
15631
|
+
]), z.number().int().nonnegative()).optional(),
|
|
14250
15632
|
warnings: z.array(z.string())
|
|
14251
15633
|
});
|
|
14252
15634
|
const crawlConfigSchema = z.object({
|
|
@@ -14267,6 +15649,7 @@ const crawlConfigSchema = z.object({
|
|
|
14267
15649
|
respectRobots: z.boolean(),
|
|
14268
15650
|
useSitemap: z.boolean(),
|
|
14269
15651
|
checkExternal: z.boolean(),
|
|
15652
|
+
checkAgentDiscovery: z.boolean(),
|
|
14270
15653
|
js: z.union([z.enum([
|
|
14271
15654
|
"auto",
|
|
14272
15655
|
"on",
|
|
@@ -14297,6 +15680,15 @@ const crawlReportSummarySchema = z.object({
|
|
|
14297
15680
|
queuedUrls: z.number().int(),
|
|
14298
15681
|
crawledUrls: z.number().int(),
|
|
14299
15682
|
skippedUrls: z.number().int(),
|
|
15683
|
+
skipReasons: z.array(z.object({
|
|
15684
|
+
reason: z.enum(CRAWL_SKIP_REASONS),
|
|
15685
|
+
impact: z.enum(["coverage-affecting", "non-impacting"]),
|
|
15686
|
+
count: z.number().int().positive()
|
|
15687
|
+
})),
|
|
15688
|
+
skippedUrlsByImpact: z.object({
|
|
15689
|
+
coverageAffecting: z.number().int().nonnegative(),
|
|
15690
|
+
nonImpacting: z.number().int().nonnegative()
|
|
15691
|
+
}),
|
|
14300
15692
|
failedUrls: z.number().int(),
|
|
14301
15693
|
observedInternalLinks: z.number().int(),
|
|
14302
15694
|
pageLimitReached: z.boolean().default(false),
|
|
@@ -14379,6 +15771,7 @@ const crawlReportBaseSchema = z.object({
|
|
|
14379
15771
|
ai: crawlAiSignalsSchema.optional(),
|
|
14380
15772
|
sitemapDiscovery: crawlSitemapDiscoverySchema.optional(),
|
|
14381
15773
|
externalLinkVerification: crawlExternalLinkVerificationSchema.optional(),
|
|
15774
|
+
agentDiscovery: agentDiscoverySchema.optional(),
|
|
14382
15775
|
warnings: z.array(z.string()),
|
|
14383
15776
|
caveats: z.array(z.string())
|
|
14384
15777
|
});
|
|
@@ -20175,7 +21568,7 @@ function crawlInput(input) {
|
|
|
20175
21568
|
return {
|
|
20176
21569
|
pages,
|
|
20177
21570
|
evidence: sourceEvidence({
|
|
20178
|
-
completeness: report ? report.summary.pageLimitReached ? "truncated" : report.status === "
|
|
21571
|
+
completeness: report ? report.summary.pageLimitReached ? "truncated" : report.status === "partial" || report.summary.skippedUrlsByImpact.coverageAffecting > 0 ? "partial" : report.status === "completed" ? "complete" : "unknown" : "completeness" in input ? input.completeness : "unknown",
|
|
20179
21572
|
rowsReceived: pages.length,
|
|
20180
21573
|
validUrls: urls.length,
|
|
20181
21574
|
uniqueUrls: uniqueUrls.size,
|
|
@@ -25448,14 +26841,14 @@ async function crawlImplementationQueueWorkflow(input, dependencies) {
|
|
|
25448
26841
|
async function diagnosePropertyWorkflow(input) {
|
|
25449
26842
|
const narrative = await reportNarrative(input);
|
|
25450
26843
|
const skippedCount = narrative.diagnosis.skippedSections?.length ?? 0;
|
|
25451
|
-
const stepSummary = narrative.dataStatus === "complete" ? "Generated diagnosis, movement, change, and monitoring narrative." : skippedCount ? `Generated ${narrative.dataStatus === "unavailable" ? "an" : "a"} ${narrative.dataStatus} diagnosis; ${skippedCount} sections were unavailable.` : "Generated a partial diagnosis because one or more source datasets were incomplete.";
|
|
26844
|
+
const stepSummary = input.skipSearchData ? "Skipped provider-backed search diagnosis because no Search Console property was selected." : narrative.dataStatus === "complete" ? "Generated diagnosis, movement, change, and monitoring narrative." : skippedCount ? `Generated ${narrative.dataStatus === "unavailable" ? "an" : "a"} ${narrative.dataStatus} diagnosis; ${skippedCount} sections were unavailable.` : "Generated a partial diagnosis because one or more source datasets were incomplete.";
|
|
25452
26845
|
return workflowReport({
|
|
25453
26846
|
workflow: "diagnose-property",
|
|
25454
26847
|
site: input.site,
|
|
25455
26848
|
summary: narrative.headline,
|
|
25456
26849
|
steps: [{
|
|
25457
26850
|
tool: "seo_report_narrative",
|
|
25458
|
-
status: "completed",
|
|
26851
|
+
status: input.skipSearchData ? "skipped" : "completed",
|
|
25459
26852
|
summary: stepSummary
|
|
25460
26853
|
}],
|
|
25461
26854
|
actions: narrative.priorities.map((priority) => ({
|
|
@@ -27192,6 +28585,6 @@ function isoDate(daysAgo = 0) {
|
|
|
27192
28585
|
//#region packages/core/src/version.ts
|
|
27193
28586
|
const SEO_VERSION = version;
|
|
27194
28587
|
//#endregion
|
|
27195
|
-
export { AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, AI_SEARCH_SCORECARD_METHODOLOGY_ID, AI_SEARCH_SCORECARD_METHODOLOGY_VERSION, AUDIT_PAGE_RULE_IDS, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CRAWL_REPORT_STORAGE_VERSION, CTR_DEFAULT_LIMIT, CTR_DEFAULT_MIN_IMPRESSIONS, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DataForSeoProvider, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LocalCredentialsProvider, OKF_DEFAULT_CONCEPTS, OKF_MAX_CONCEPTS, REPORT_BASELINE_MAX_AGE_MS, REPORT_BASELINE_MAX_DEPTH, REPORT_BASELINE_MAX_PAGES, SCORECARD_CHECK_WEIGHTS, SCORECARD_STATUS_CREDIT, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SemrushProvider, SeoError, ServiceAccountAccessTokenClient, SessionLedger, affectedUrls, aiPromptsForQuery, aiReadiness, aiReferralSourceForValue, aiReferralsReport, aiSearchScorecard, allocateIndexUrlsToProperties, analyzeAiReferralRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertUrlMatchesGscProperty, attachCrawlRecommendations, auditCrawlPages, auditCrawlRequests, auditLlmsTxt, auditPage, authStatus, boundedSitemapInventory, buildOkfBundle, buildPseoAuditReportFromRows, cannibalReport, changeMeasurementCsvRows, classifyCommunityIntent, clearCache, clientProfileSchema, clusterPseoTemplates, clusterQueryRows, combineRobotsValues, communityIntentReport, compareCrawlPages, compareCrawlReports, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlDiff, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportStore, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createContentGroup, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, decayClusterDrafts, decayComparisonRange, decayingReport, deleteChange, deleteClient, deleteContentGroup, deleteCrawlReport, deleteTokens, deriveBrandTerms, detectBrandTerms, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensureSeoCliDirs, entityReadiness, estimateSerpTitleWidth, expectedCtrForPosition, explainOkfValidation, explainRule, fetchCruxFieldData, fetchPage, fetchSearchStatusIncidents, fetchSitemapUrls, fileSize, finalGscDateRange, findOverlappingSearchUpdates, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, generateLlmsTxt, geoGaps, geoGapsReport, getAuthModeStatus, getCacheStats, getChange, getClient, getClientConfig, getContentGroup, getDb, getKeywordProvider, getPreviousRun, getPrivacySnapshot, getRepeatedLinkRecoverUrls, getRunPages, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTokenStorageStatus, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoveragePlan, indexCoverageSignals, indexMonitor, indexWatch, inferPseoTemplate, inferTemplateMovement, insertCrawlRun, insertLinkRecoverRun, inspectUrl, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isCompatibleTechnicalBaseline, isCurrentTechnicalBaseline, isSkippableReportError, isoDate, latestCrawlReport, latestCrawlSummaries, latestFinalGscDate, latestIndexWatchSummary, latestLinkRecoverSummary, linkRecover, linkRecoverRecommendation, listChanges, listClients, listContentGroups, listCrawlReports, listGa4AccountSummaries, listGa4DataStreams, listRules, listSearchUpdates, listSites, loadCrawlReport, localCredentialsProvider, loginWithLoopback, makeRunId, matchGa4WebStreams, measureChange, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, monitoringStatus, monthlyReport, monthlyReportWorkflow, narrativeCsvFiles, normalizeCrawlConfig, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, okfConceptLimit, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, parsePseoPath, performanceAudit, performanceReportIsCacheable, planDueIndexUrls, planIndexCoverageFromUrls, plural, propertyMatchesUrl, providerPreferenceSchema, pseoAuditOptions, pseoAuditReport, pseoCsvFiles, pseoIndexStatus, pseoPresentation, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, readConfig, readOauthClient, readTokens, recommendCrawlDiffItem, recommendationForRule, recordChange, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, renderCrawlCsv, renderCrawlHtml, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderPseoMarkdown, renderWorkflowMarkdown, reportNarrative, reportPresentation, requestHeaders, resolveSegmentRanges, resolveSeoCliPaths, resolveTechnicalBaseline, reviewObservations, roundedPosition, runDoctor, runGa4Report, saveClient, saveCrawlReport, secondPage, segmentComparisonRange, segmentImpact, segmentRangeDays, selectAffectedUrls, selectDueIndexUrls, seoErrorEnvelope, seoToAiQueryReport, sessionSourceInListFilter, setDefaultClient, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteSchema, sqliteCrawlReportStore, strikingDistance, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, technicalWatchWorkflow, templateForUrl, toSeoError, tokenSchema, topCrawlRecommendations, topFixes, trafficAnomaly, unavailableSegmentImpactReport, unicodeTokens, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateOkfFiles, verifyQueryContent, workflowPresentation, writeConfig, writeOauthClient, writeTokens, xRobotsDirectives };
|
|
28588
|
+
export { AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, AI_SEARCH_SCORECARD_METHODOLOGY_ID, AI_SEARCH_SCORECARD_METHODOLOGY_VERSION, AUDIT_PAGE_RULE_IDS, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CRAWL_REPORT_STORAGE_VERSION, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_DEFAULT_MIN_IMPRESSIONS, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DataForSeoProvider, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LocalCredentialsProvider, OKF_DEFAULT_CONCEPTS, OKF_MAX_CONCEPTS, REPORT_BASELINE_MAX_AGE_MS, REPORT_BASELINE_MAX_DEPTH, REPORT_BASELINE_MAX_PAGES, SCORECARD_CHECK_WEIGHTS, SCORECARD_STATUS_CREDIT, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SemrushProvider, SeoError, ServiceAccountAccessTokenClient, SessionLedger, affectedUrls, agentReadiness, aiPromptsForQuery, aiReadiness, aiReferralSourceForValue, aiReferralsReport, aiSearchScorecard, allocateIndexUrlsToProperties, analyzeAiReferralRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertUrlMatchesGscProperty, attachCrawlRecommendations, auditCrawlPages, auditCrawlRequests, auditLlmsTxt, auditPage, authStatus, boundedSitemapInventory, buildOkfBundle, buildPseoAuditReportFromRows, cannibalReport, changeMeasurementCsvRows, classifyCommunityIntent, clearCache, clientProfileSchema, clusterPseoTemplates, clusterQueryRows, collectAgentDiscovery, combineRobotsValues, communityIntentReport, compareCrawlPages, compareCrawlReports, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlDiff, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportStore, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createContentGroup, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, decayClusterDrafts, decayComparisonRange, decayingReport, deleteChange, deleteClient, deleteContentGroup, deleteCrawlReport, deleteTokens, deriveBrandTerms, detectBrandTerms, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensureSeoCliDirs, entityReadiness, estimateSerpTitleWidth, expectedCtrForPosition, explainOkfValidation, explainRule, fetchCruxFieldData, fetchPage, fetchSearchStatusIncidents, fetchSitemapUrls, fileSize, finalGscDateRange, findOverlappingSearchUpdates, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, generateLlmsTxt, geoGaps, geoGapsReport, getAuthModeStatus, getCacheStats, getChange, getClient, getClientConfig, getContentGroup, getDb, getKeywordProvider, getPreviousRun, getPrivacySnapshot, getRepeatedLinkRecoverUrls, getRunPages, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTokenStorageStatus, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoveragePlan, indexCoverageSignals, indexMonitor, indexWatch, inferPseoTemplate, inferTemplateMovement, insertCrawlRun, insertLinkRecoverRun, inspectUrl, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isCompatibleTechnicalBaseline, isCurrentTechnicalBaseline, isEntityEvidenceSchemaType, isSiteIdentitySchemaType, isSkippableReportError, isoDate, latestCrawlReport, latestCrawlSummaries, latestFinalGscDate, latestIndexWatchSummary, latestLinkRecoverSummary, linkRecover, linkRecoverRecommendation, listChanges, listClients, listContentGroups, listCrawlReports, listGa4AccountSummaries, listGa4DataStreams, listRules, listSearchUpdates, listSites, loadCrawlReport, localCredentialsProvider, loginWithLoopback, makeRunId, matchGa4WebStreams, measureChange, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, monitoringStatus, monthlyReport, monthlyReportWorkflow, narrativeCsvFiles, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, okfConceptLimit, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, parsePseoPath, performanceAudit, performanceReportIsCacheable, planDueIndexUrls, planIndexCoverageFromUrls, plural, propertyMatchesUrl, providerPreferenceSchema, pseoAuditOptions, pseoAuditReport, pseoCsvFiles, pseoIndexStatus, pseoPresentation, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, readConfig, readOauthClient, readTokens, recommendCrawlDiffItem, recommendationForRule, recordChange, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, renderCrawlCsv, renderCrawlHtml, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderPseoMarkdown, renderWorkflowMarkdown, reportNarrative, reportPresentation, requestHeaders, resolveSegmentRanges, resolveSeoCliPaths, resolveTechnicalBaseline, reviewObservations, roundedPosition, runDoctor, runGa4Report, saveClient, saveCrawlReport, secondPage, segmentComparisonRange, segmentImpact, segmentRangeDays, selectAffectedUrls, selectDueIndexUrls, seoErrorEnvelope, seoToAiQueryReport, sessionSourceInListFilter, setDefaultClient, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteSchema, sqliteCrawlReportStore, strikingDistance, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, technicalWatchWorkflow, templateForUrl, toSeoError, tokenSchema, topCrawlRecommendations, topFixes, trafficAnomaly, unavailableSegmentImpactReport, unicodeTokens, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateOkfFiles, verifyQueryContent, workflowPresentation, writeConfig, writeOauthClient, writeTokens, xRobotsDirectives };
|
|
27196
28589
|
|
|
27197
28590
|
//# sourceMappingURL=index.js.map
|