seo 0.7.1 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -345
- package/dist/cli.js +84 -84
- package/dist/cli.js.map +1 -1
- package/dist/{dist-S2OSIAML.js → dist-B5I5uxdM.js} +88 -85
- package/dist/dist-B5I5uxdM.js.map +1 -0
- package/dist/{dist-aHJsXHVF.js → dist-ulug7z2k.js} +1 -1
- package/dist/index.d.ts +941 -220
- package/dist/index.js +631 -298
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +952 -281
- package/dist/mcp.js.map +1 -1
- package/dist/{package-BaQwwfSG.js → package-BjCto4m4.js} +2 -2
- package/dist/{package-BaQwwfSG.js.map → package-BjCto4m4.js.map} +1 -1
- package/evals/crawl.json +2 -3
- package/evals/internal-links.json +2 -3
- package/evals/low-ctr.json +1 -2
- package/evals/page.json +2 -3
- package/evals/quick-wins.json +2 -3
- package/evals/seo.json +2 -2
- package/package.json +1 -1
- package/skills/README.md +4 -4
- package/skills/seo/SKILL.md +22 -95
- package/dist/dist-S2OSIAML.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { n as load, t as version } from "./package-
|
|
2
|
+
import { n as load, t as version } from "./package-BjCto4m4.js";
|
|
3
3
|
import { SEO_PROVIDER_API_VERSION } from "./provider-sdk.js";
|
|
4
4
|
import crypto, { createHash, randomUUID, sign } from "node:crypto";
|
|
5
5
|
import { z } from "zod";
|
|
@@ -857,6 +857,98 @@ const RULE_DEFINITIONS = [
|
|
|
857
857
|
howToFix: "Add twitter:card and matching metadata only when you need a specific X preview. Keep Open Graph metadata accurate either way.",
|
|
858
858
|
impactIfIgnored: "X may use fallback metadata or a simple link preview. Organic rankings are not affected.",
|
|
859
859
|
howToVerify: "Re-run the crawl and confirm twitterCard is present if a specific X preview is required."
|
|
860
|
+
},
|
|
861
|
+
{
|
|
862
|
+
id: "ai_search_bot_blocked",
|
|
863
|
+
title: "robots.txt blocks AI search bots",
|
|
864
|
+
category: "agents",
|
|
865
|
+
defaultSeverity: "medium",
|
|
866
|
+
whyItMatters: "AI search bots fetch pages to answer a person's question and cite the source. A blocked bot cannot read the site, so its answers cannot quote or link to it. Training bots are a separate choice and are listed apart.",
|
|
867
|
+
howToFix: "Allow the AI search and user bots in robots.txt if the site should appear in AI answers. Block training bots separately if the person wants that.",
|
|
868
|
+
impactIfIgnored: "ChatGPT, Claude, Perplexity and similar tools can leave the site out of answers and citations.",
|
|
869
|
+
howToVerify: "Re-run the crawl and confirm the search bots in ai.robotsTxt.botAccess show allowed: true.",
|
|
870
|
+
review: {
|
|
871
|
+
question: "Is the site meant to be left out of AI search answers?",
|
|
872
|
+
changeOnlyIf: "Change robots.txt when the person wants the site to appear in AI answers.",
|
|
873
|
+
ifNotNeeded: "Record that blocking AI search bots is a deliberate choice for this site."
|
|
874
|
+
},
|
|
875
|
+
agentHints: { evidenceFields: ["report.ai.robotsTxt.botAccess"] }
|
|
876
|
+
},
|
|
877
|
+
{
|
|
878
|
+
id: "llms_txt_missing",
|
|
879
|
+
title: "No llms.txt",
|
|
880
|
+
category: "agents",
|
|
881
|
+
defaultSeverity: "low",
|
|
882
|
+
whyItMatters: "llms.txt gives agents a short summary of the site and links to its main pages, ideally their Markdown copies. Search engines do not use it for ranking.",
|
|
883
|
+
howToFix: "Publish /llms.txt as plain text with a one-line summary and sections of links to the main pages.",
|
|
884
|
+
impactIfIgnored: "Agents must find the main pages by crawling, which costs them more requests and tokens.",
|
|
885
|
+
howToVerify: "Request /llms.txt and confirm a 200 plain text response.",
|
|
886
|
+
review: {
|
|
887
|
+
question: "Should agents get a reading list for this site?",
|
|
888
|
+
changeOnlyIf: "Add it when agents are expected to read or use the site.",
|
|
889
|
+
ifNotNeeded: "Record that the site has no llms.txt by choice."
|
|
890
|
+
},
|
|
891
|
+
agentHints: { evidenceFields: ["report.ai.llmsTxt"] }
|
|
892
|
+
},
|
|
893
|
+
{
|
|
894
|
+
id: "llms_txt_broken_link",
|
|
895
|
+
title: "llms.txt links to a page that does not load",
|
|
896
|
+
category: "agents",
|
|
897
|
+
defaultSeverity: "low",
|
|
898
|
+
whyItMatters: "An agent that follows a link in llms.txt and gets an error loses its reading list entry.",
|
|
899
|
+
howToFix: "Update or remove each broken link in llms.txt, or restore the page it points to.",
|
|
900
|
+
impactIfIgnored: "Agents waste requests and can miss the page.",
|
|
901
|
+
howToVerify: "Re-run the crawl and confirm ai.llmsTxt.links.broken is empty.",
|
|
902
|
+
agentHints: { evidenceFields: ["report.ai.llmsTxt.links"] }
|
|
903
|
+
},
|
|
904
|
+
{
|
|
905
|
+
id: "markdown_missing",
|
|
906
|
+
title: "No Markdown copy of the pages",
|
|
907
|
+
category: "agents",
|
|
908
|
+
defaultSeverity: "low",
|
|
909
|
+
whyItMatters: "A Markdown copy gives agents the page content without layout, scripts or navigation, at a fraction of the tokens. Neither a request with Accept: text/markdown nor a rel=\"alternate\" link returned one. Search engines do not use it for ranking.",
|
|
910
|
+
howToFix: "Serve a Markdown copy of each page, linked from the page with <link rel=\"alternate\" type=\"text/markdown\">, and return it for Accept: text/markdown with Vary: Accept.",
|
|
911
|
+
impactIfIgnored: "Agents read the full HTML, which costs more tokens and can include layout text.",
|
|
912
|
+
howToVerify: "Request a page with Accept: text/markdown and confirm a text/markdown response.",
|
|
913
|
+
review: {
|
|
914
|
+
question: "Should agents get a Markdown copy of the pages?",
|
|
915
|
+
changeOnlyIf: "Add it when agents are expected to read the site.",
|
|
916
|
+
ifNotNeeded: "Record that the site serves HTML only by choice."
|
|
917
|
+
},
|
|
918
|
+
agentHints: { evidenceFields: ["report.ai.markdownNegotiation", "report.ai.markdownCopies"] }
|
|
919
|
+
},
|
|
920
|
+
{
|
|
921
|
+
id: "markdown_alternate_missing",
|
|
922
|
+
title: "Page has no link to its Markdown copy",
|
|
923
|
+
category: "agents",
|
|
924
|
+
defaultSeverity: "low",
|
|
925
|
+
whyItMatters: "Other pages on the site link to a Markdown copy, but this one does not, so agents cannot find a copy for it.",
|
|
926
|
+
howToFix: "Add <link rel=\"alternate\" type=\"text/markdown\" href=\"...\"> to the page head, pointing at its Markdown copy.",
|
|
927
|
+
impactIfIgnored: "Agents read the full HTML for this page.",
|
|
928
|
+
howToVerify: "Re-run the crawl and confirm the page lists a Markdown alternate.",
|
|
929
|
+
agentHints: { evidenceFields: ["page.markdownAlternates", "report.ai.markdownCopies"] }
|
|
930
|
+
},
|
|
931
|
+
{
|
|
932
|
+
id: "markdown_alternate_on_private_page",
|
|
933
|
+
title: "Signed-in page links to a Markdown copy",
|
|
934
|
+
category: "agents",
|
|
935
|
+
defaultSeverity: "medium",
|
|
936
|
+
whyItMatters: "Signed-in pages should have no Markdown copy. This page needs a sign-in, yet it links to one. A copy can show agents content that a signed-out visitor cannot see, or fail and waste their requests.",
|
|
937
|
+
howToFix: "Remove the rel=\"alternate\" Markdown link, the Link header and any Markdown copy for signed-in routes. Public pages keep theirs, including noindex pages such as privacy and terms.",
|
|
938
|
+
impactIfIgnored: "Agents can request a copy that fails, or read content that was meant to stay behind a sign-in.",
|
|
939
|
+
howToVerify: "Re-run the crawl and confirm the page no longer lists a Markdown alternate.",
|
|
940
|
+
agentHints: { evidenceFields: ["page.markdownAlternates", "report.ai.markdownCopies.broken"] }
|
|
941
|
+
},
|
|
942
|
+
{
|
|
943
|
+
id: "markdown_copy_broken",
|
|
944
|
+
title: "Markdown copy does not load",
|
|
945
|
+
category: "agents",
|
|
946
|
+
defaultSeverity: "medium",
|
|
947
|
+
whyItMatters: "The page links to a Markdown copy that returns an error or HTML, so an agent that follows the link gets nothing useful. If the page needs a sign-in, remove the link instead.",
|
|
948
|
+
howToFix: "Make the linked URL return the page as text/markdown, or fix the link.",
|
|
949
|
+
impactIfIgnored: "Agents get an error or the HTML page instead of the copy.",
|
|
950
|
+
howToVerify: "Request the Markdown URL and confirm a 200 text/markdown response.",
|
|
951
|
+
agentHints: { evidenceFields: ["report.ai.markdownCopies.broken"] }
|
|
860
952
|
}
|
|
861
953
|
];
|
|
862
954
|
const RULE_RECOMMENDATIONS = {
|
|
@@ -885,7 +977,10 @@ const RULE_RECOMMENDATIONS = {
|
|
|
885
977
|
og_title_missing: "review",
|
|
886
978
|
og_description_missing: "review",
|
|
887
979
|
og_image_missing: "review",
|
|
888
|
-
twitter_card_missing: "review"
|
|
980
|
+
twitter_card_missing: "review",
|
|
981
|
+
ai_search_bot_blocked: "review",
|
|
982
|
+
llms_txt_missing: "review",
|
|
983
|
+
markdown_missing: "review"
|
|
889
984
|
};
|
|
890
985
|
const RULES_BY_ID = new Map(RULE_DEFINITIONS.map((rule) => [rule.id, {
|
|
891
986
|
...rule,
|
|
@@ -1586,6 +1681,14 @@ async function listAnalyticsAccounts(gfetch, options) {
|
|
|
1586
1681
|
});
|
|
1587
1682
|
});
|
|
1588
1683
|
}
|
|
1684
|
+
async function listAnalyticsDataStreams(gfetch, propertyId, options) {
|
|
1685
|
+
return (await googleJson(gfetch, `https://analyticsadmin.googleapis.com/v1beta/properties/${propertyId}/dataStreams?pageSize=200`, {
|
|
1686
|
+
...options,
|
|
1687
|
+
label: "Google Analytics data streams",
|
|
1688
|
+
subject: `property ${propertyId}`,
|
|
1689
|
+
maxBytes: 1024 * 1024
|
|
1690
|
+
})).dataStreams ?? [];
|
|
1691
|
+
}
|
|
1589
1692
|
async function runAnalyticsReport(gfetch, propertyId, body, options) {
|
|
1590
1693
|
if (!/^\d+$/.test(propertyId)) throw new SeoError("INVALID_INPUT", "The Google Analytics property must be its numeric id.");
|
|
1591
1694
|
return googleJson(gfetch, `https://analyticsdata.googleapis.com/v1beta/properties/${propertyId}:runReport`, {
|
|
@@ -2102,7 +2205,7 @@ async function cloudCredentials(options = {}) {
|
|
|
2102
2205
|
return credentials;
|
|
2103
2206
|
}
|
|
2104
2207
|
function notSignedIn() {
|
|
2105
|
-
return "Not signed in. Run `seo login` to use your seoskill.dev account on this machine (it prints a link to approve), or `seo auth login` to sign in with Google on this machine only.";
|
|
2208
|
+
return "Not signed in. Run `seo login` to use your seoskill.dev account on this machine (it prints a link to approve), or `seo auth login` to sign in with Google on this machine only. `seo crawl --url <url>` works without sign-in.";
|
|
2106
2209
|
}
|
|
2107
2210
|
function tokenOf(item, what) {
|
|
2108
2211
|
if ("error" in item) throw new SeoError("AUTH_EXPIRED", `${what}: ${item.error}`);
|
|
@@ -9613,7 +9716,7 @@ function clientRenderedContentEvidence(page) {
|
|
|
9613
9716
|
}
|
|
9614
9717
|
};
|
|
9615
9718
|
}
|
|
9616
|
-
function issue(ruleId, page, detail, evidence) {
|
|
9719
|
+
function issue$1(ruleId, page, detail, evidence) {
|
|
9617
9720
|
const rule = explainRule(ruleId);
|
|
9618
9721
|
if (!rule) throw new Error(`Missing rule guidance for ${ruleId}`);
|
|
9619
9722
|
return {
|
|
@@ -9778,7 +9881,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
9778
9881
|
}
|
|
9779
9882
|
for (const page of pages) {
|
|
9780
9883
|
if (page.status === 0) {
|
|
9781
|
-
issues.push(issue("connection_error", page, page.error, {
|
|
9884
|
+
issues.push(issue$1("connection_error", page, page.error, {
|
|
9782
9885
|
status: page.status,
|
|
9783
9886
|
error: page.error
|
|
9784
9887
|
}));
|
|
@@ -9786,23 +9889,23 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
9786
9889
|
}
|
|
9787
9890
|
if (page.accessBlock) continue;
|
|
9788
9891
|
if (page.status >= 500) {
|
|
9789
|
-
issues.push(issue("server_error", page, String(page.status), { status: page.status }));
|
|
9790
|
-
if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue("broken_internal_link", page, String(page.status), {
|
|
9892
|
+
issues.push(issue$1("server_error", page, String(page.status), { status: page.status }));
|
|
9893
|
+
if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
9791
9894
|
status: page.status,
|
|
9792
9895
|
internalInlinkCount: page.internalInlinkCount
|
|
9793
9896
|
}));
|
|
9794
9897
|
continue;
|
|
9795
9898
|
}
|
|
9796
9899
|
if (page.status >= 400) {
|
|
9797
|
-
issues.push(issue("client_error", page, String(page.status), { status: page.status }));
|
|
9798
|
-
if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0) issues.push(issue("broken_internal_link", page, String(page.status), {
|
|
9900
|
+
issues.push(issue$1("client_error", page, String(page.status), { status: page.status }));
|
|
9901
|
+
if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
9799
9902
|
status: page.status,
|
|
9800
9903
|
internalInlinkCount: page.internalInlinkCount
|
|
9801
9904
|
}));
|
|
9802
9905
|
continue;
|
|
9803
9906
|
}
|
|
9804
9907
|
if (page.status >= 300) {
|
|
9805
|
-
issues.push(issue("redirected_url", page, `Status ${page.status}`, {
|
|
9908
|
+
issues.push(issue$1("redirected_url", page, `Status ${page.status}`, {
|
|
9806
9909
|
requestedUrl: page.url,
|
|
9807
9910
|
finalUrl: page.finalUrl,
|
|
9808
9911
|
status: page.status
|
|
@@ -9813,7 +9916,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
9813
9916
|
const redirected = isRedirectedPage(page);
|
|
9814
9917
|
if (redirected) {
|
|
9815
9918
|
const redirectTarget = page.metaRefresh?.targetUrl ?? page.finalUrl;
|
|
9816
|
-
issues.push(issue("redirected_url", page, `Final URL: ${redirectTarget}`, {
|
|
9919
|
+
issues.push(issue$1("redirected_url", page, `Final URL: ${redirectTarget}`, {
|
|
9817
9920
|
requestedUrl: page.url,
|
|
9818
9921
|
finalUrl: redirectTarget,
|
|
9819
9922
|
status: page.status,
|
|
@@ -9824,46 +9927,46 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
9824
9927
|
}));
|
|
9825
9928
|
}
|
|
9826
9929
|
const redirectChain = page.fetchDiagnostics?.redirectChain ?? [];
|
|
9827
|
-
if (redirectChain.length > 1) issues.push(issue("redirect_chain", page, `${redirectChain.length} hops`, {
|
|
9930
|
+
if (redirectChain.length > 1) issues.push(issue$1("redirect_chain", page, `${redirectChain.length} hops`, {
|
|
9828
9931
|
redirectChain,
|
|
9829
9932
|
hops: redirectChain.length
|
|
9830
9933
|
}));
|
|
9831
|
-
if ((page.responseTimeMs ?? 0) > SLOW_RESPONSE_MS) issues.push(issue("slow_response", page, `${page.responseTimeMs}ms`, {
|
|
9934
|
+
if ((page.responseTimeMs ?? 0) > SLOW_RESPONSE_MS) issues.push(issue$1("slow_response", page, `${page.responseTimeMs}ms`, {
|
|
9832
9935
|
responseTimeMs: page.responseTimeMs,
|
|
9833
9936
|
thresholdMs: SLOW_RESPONSE_MS
|
|
9834
9937
|
}));
|
|
9835
9938
|
if (page.auditScope === "status") continue;
|
|
9836
9939
|
if (redirected) continue;
|
|
9837
|
-
if (page.softAuthenticationGate) issues.push(issue("soft_authentication_gate", page, "A password form and login-specific page signals were observed.", page.softAuthenticationGate));
|
|
9940
|
+
if (page.softAuthenticationGate) issues.push(issue$1("soft_authentication_gate", page, "A password form and login-specific page signals were observed.", page.softAuthenticationGate));
|
|
9838
9941
|
const renderedContent = clientRenderedContentEvidence(page);
|
|
9839
|
-
if (renderedContent) issues.push(issue("client_rendered_content", page, "The raw response was an empty or near-empty shell.", renderedContent));
|
|
9840
|
-
if (!renderedContent && isAuditableHtmlPage(page) && page.indexable && page.extractionStatus === "complete" && page.wordCount < NEAR_EMPTY_CONTENT_WORDS) issues.push(issue("near_empty_content", page, `${page.wordCount} extracted words`, {
|
|
9942
|
+
if (renderedContent) issues.push(issue$1("client_rendered_content", page, "The raw response was an empty or near-empty shell.", renderedContent));
|
|
9943
|
+
if (!renderedContent && isAuditableHtmlPage(page) && page.indexable && page.extractionStatus === "complete" && page.wordCount < NEAR_EMPTY_CONTENT_WORDS) issues.push(issue$1("near_empty_content", page, `${page.wordCount} extracted words`, {
|
|
9841
9944
|
wordCount: page.wordCount,
|
|
9842
9945
|
reviewBelowWords: NEAR_EMPTY_CONTENT_WORDS,
|
|
9843
9946
|
contentSample: page.contentSample,
|
|
9844
9947
|
outgoingInternalCount: page.outgoingInternalCount,
|
|
9845
9948
|
rendering: page.fetchDiagnostics?.rendering
|
|
9846
9949
|
}));
|
|
9847
|
-
if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(issue("large_html", page, `${Math.round((page.sizeBytes ?? 0) / 1024)} KB`, {
|
|
9950
|
+
if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(issue$1("large_html", page, `${Math.round((page.sizeBytes ?? 0) / 1024)} KB`, {
|
|
9848
9951
|
sizeBytes: page.sizeBytes,
|
|
9849
9952
|
thresholdBytes: LARGE_HTML_BYTES
|
|
9850
9953
|
}));
|
|
9851
|
-
if (isHtmlPage(page) && !page.compression && (page.sizeBytes ?? 0) > COMPRESSION_MIN_BYTES) issues.push(issue("no_compression", page, void 0, {
|
|
9954
|
+
if (isHtmlPage(page) && !page.compression && (page.sizeBytes ?? 0) > COMPRESSION_MIN_BYTES) issues.push(issue$1("no_compression", page, void 0, {
|
|
9852
9955
|
sizeBytes: page.sizeBytes,
|
|
9853
9956
|
thresholdBytes: COMPRESSION_MIN_BYTES,
|
|
9854
9957
|
compression: page.compression
|
|
9855
9958
|
}));
|
|
9856
|
-
if (page.isHttps === false && !isLoopbackUrl(page.finalUrl || page.url)) issues.push(issue("http_not_secure", page, void 0, { finalUrl: page.finalUrl }));
|
|
9959
|
+
if (page.isHttps === false && !isLoopbackUrl(page.finalUrl || page.url)) issues.push(issue$1("http_not_secure", page, void 0, { finalUrl: page.finalUrl }));
|
|
9857
9960
|
else if (page.isHttps === true) {
|
|
9858
|
-
if ((page.mixedContentCount ?? 0) > 0) issues.push(issue("mixed_content", page, `${page.mixedContentCount} insecure resources`, {
|
|
9961
|
+
if ((page.mixedContentCount ?? 0) > 0) issues.push(issue$1("mixed_content", page, `${page.mixedContentCount} insecure resources`, {
|
|
9859
9962
|
mixedContentCount: page.mixedContentCount,
|
|
9860
9963
|
mixedContentSamples: page.mixedContentSamples
|
|
9861
9964
|
}));
|
|
9862
|
-
if (page.hasHsts === false) issues.push(issue("hsts_missing", page, void 0, { header: "strict-transport-security" }));
|
|
9965
|
+
if (page.hasHsts === false) issues.push(issue$1("hsts_missing", page, void 0, { header: "strict-transport-security" }));
|
|
9863
9966
|
}
|
|
9864
9967
|
for (const link of page.externalLinkChecks ?? []) {
|
|
9865
9968
|
if (link.state !== "confirmed-broken") continue;
|
|
9866
|
-
issues.push(issue("broken_external_link", page, link.url, {
|
|
9969
|
+
issues.push(issue$1("broken_external_link", page, link.url, {
|
|
9867
9970
|
url: link.url,
|
|
9868
9971
|
status: link.status,
|
|
9869
9972
|
error: link.error,
|
|
@@ -9871,64 +9974,64 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
9871
9974
|
attempts: link.attempts
|
|
9872
9975
|
}));
|
|
9873
9976
|
}
|
|
9874
|
-
if (page.indexable && (page.internalInlinkCount ?? 0) === 0 && !sameUrl(page.url, opts.startUrl)) issues.push(issue("orphan_page", page, "No internal inlinks", { internalInlinkCount: page.internalInlinkCount ?? 0 }));
|
|
9875
|
-
if ((page.crawlDepth ?? 0) > DEEP_PAGE_DEPTH) issues.push(issue("deep_page", page, `Depth ${page.crawlDepth}`, {
|
|
9977
|
+
if (page.indexable && (page.internalInlinkCount ?? 0) === 0 && !sameUrl(page.url, opts.startUrl)) issues.push(issue$1("orphan_page", page, "No internal inlinks", { internalInlinkCount: page.internalInlinkCount ?? 0 }));
|
|
9978
|
+
if ((page.crawlDepth ?? 0) > DEEP_PAGE_DEPTH) issues.push(issue$1("deep_page", page, `Depth ${page.crawlDepth}`, {
|
|
9876
9979
|
crawlDepth: page.crawlDepth,
|
|
9877
9980
|
threshold: DEEP_PAGE_DEPTH
|
|
9878
9981
|
}));
|
|
9879
|
-
if (page.indexable && isValuablePage(page) && (page.internalInlinkCount ?? 0) <= WEAK_VALUABLE_INLINKS) issues.push(issue("weak_internal_links_to_valuable_page", page, `${page.internalInlinkCount ?? 0} internal inlinks`, {
|
|
9982
|
+
if (page.indexable && isValuablePage(page) && (page.internalInlinkCount ?? 0) <= WEAK_VALUABLE_INLINKS) issues.push(issue$1("weak_internal_links_to_valuable_page", page, `${page.internalInlinkCount ?? 0} internal inlinks`, {
|
|
9880
9983
|
internalInlinkCount: page.internalInlinkCount ?? 0,
|
|
9881
9984
|
searchMetrics: page.searchMetrics,
|
|
9882
9985
|
analytics: page.analytics
|
|
9883
9986
|
}));
|
|
9884
|
-
if (page.robotsTxt?.allowed === false) issues.push(issue("robots_blocked", page, page.robotsTxt.matchedLine, { robotsTxt: page.robotsTxt }));
|
|
9885
|
-
if (hasXRobotsDirective(page.xRobotsTag, "noindex")) issues.push(issue("x_robots_noindex", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
|
|
9987
|
+
if (page.robotsTxt?.allowed === false) issues.push(issue$1("robots_blocked", page, page.robotsTxt.matchedLine, { robotsTxt: page.robotsTxt }));
|
|
9988
|
+
if (hasXRobotsDirective(page.xRobotsTag, "noindex")) issues.push(issue$1("x_robots_noindex", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
|
|
9886
9989
|
const xRobotsNofollow = hasXRobotsDirective(page.xRobotsTag, "nofollow");
|
|
9887
|
-
if (xRobotsNofollow) issues.push(issue("nofollow", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
|
|
9990
|
+
if (xRobotsNofollow) issues.push(issue$1("nofollow", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
|
|
9888
9991
|
if (!isAuditableHtmlPage(page)) continue;
|
|
9889
9992
|
const title = page.title?.trim();
|
|
9890
9993
|
const titleWidth = title ? estimateSerpTitleWidth(title) : void 0;
|
|
9891
9994
|
const duplicateTitle = duplicateTitles.get(normalizedMetadata(title) ?? "");
|
|
9892
|
-
if (!title) issues.push(issue("missing_title", page));
|
|
9893
|
-
else if (titleWidth?.status === "over-reference") issues.push(issue("title_too_wide", page, `${titleWidth.estimatedPixels}px`, {
|
|
9995
|
+
if (!title) issues.push(issue$1("missing_title", page));
|
|
9996
|
+
else if (titleWidth?.status === "over-reference") issues.push(issue$1("title_too_wide", page, `${titleWidth.estimatedPixels}px`, {
|
|
9894
9997
|
title,
|
|
9895
9998
|
...titleWidth
|
|
9896
9999
|
}));
|
|
9897
|
-
if (title && duplicateTitle) issues.push(issue("title_duplicate", page, duplicateTitle.value, {
|
|
10000
|
+
if (title && duplicateTitle) issues.push(issue$1("title_duplicate", page, duplicateTitle.value, {
|
|
9898
10001
|
title,
|
|
9899
10002
|
duplicateCount: duplicateTitle.count,
|
|
9900
10003
|
sampleUrls: duplicateTitle.sampleUrls
|
|
9901
10004
|
}));
|
|
9902
10005
|
const metaDescription = page.metaDescription?.trim();
|
|
9903
10006
|
const duplicateDescription = duplicateDescriptions.get(normalizedMetadata(metaDescription) ?? "");
|
|
9904
|
-
if (!metaDescription) issues.push(issue("missing_meta_description", page));
|
|
9905
|
-
if (metaDescription && duplicateDescription) issues.push(issue("meta_description_duplicate", page, duplicateDescription.value, {
|
|
10007
|
+
if (!metaDescription) issues.push(issue$1("missing_meta_description", page));
|
|
10008
|
+
if (metaDescription && duplicateDescription) issues.push(issue$1("meta_description_duplicate", page, duplicateDescription.value, {
|
|
9906
10009
|
metaDescription,
|
|
9907
10010
|
duplicateCount: duplicateDescription.count,
|
|
9908
10011
|
sampleUrls: duplicateDescription.sampleUrls
|
|
9909
10012
|
}));
|
|
9910
10013
|
const h1Count = page.h1Count ?? (page.h1 ? 1 : 0);
|
|
9911
|
-
if (h1Count === 0) issues.push(issue("h1_missing", page, "No H1 found", { h1Count }));
|
|
9912
|
-
else if (h1Count > 1) issues.push(issue("h1_multiple", page, `${h1Count} H1 headings`, { h1Count }));
|
|
9913
|
-
if (page.canonicalStatus === "conflicting") issues.push(issue("canonical_conflict", page, void 0, { candidates: page.canonicalCandidates }));
|
|
9914
|
-
else if (page.canonicalStatus === "duplicate") issues.push(issue("canonical_multiple", page, page.canonical, {
|
|
10014
|
+
if (h1Count === 0) issues.push(issue$1("h1_missing", page, "No H1 found", { h1Count }));
|
|
10015
|
+
else if (h1Count > 1) issues.push(issue$1("h1_multiple", page, `${h1Count} H1 headings`, { h1Count }));
|
|
10016
|
+
if (page.canonicalStatus === "conflicting") issues.push(issue$1("canonical_conflict", page, void 0, { candidates: page.canonicalCandidates }));
|
|
10017
|
+
else if (page.canonicalStatus === "duplicate") issues.push(issue$1("canonical_multiple", page, page.canonical, {
|
|
9915
10018
|
canonical: page.canonical,
|
|
9916
10019
|
candidates: page.canonicalCandidates
|
|
9917
10020
|
}));
|
|
9918
|
-
else if (page.canonicalStatus === "outside-head-only") issues.push(issue("canonical_outside_head", page, page.canonicalRaw, { candidates: page.canonicalCandidates }));
|
|
9919
|
-
else if (page.canonicalStatus === "streamed-outside-head") issues.push(issue("canonical_streamed_in_body", page, page.canonical, {
|
|
10021
|
+
else if (page.canonicalStatus === "outside-head-only") issues.push(issue$1("canonical_outside_head", page, page.canonicalRaw, { candidates: page.canonicalCandidates }));
|
|
10022
|
+
else if (page.canonicalStatus === "streamed-outside-head") issues.push(issue$1("canonical_streamed_in_body", page, page.canonical, {
|
|
9920
10023
|
canonical: page.canonical,
|
|
9921
10024
|
candidates: page.canonicalCandidates
|
|
9922
10025
|
}));
|
|
9923
10026
|
if (!page.canonical) {
|
|
9924
|
-
if (!["conflicting", "outside-head-only"].includes(page.canonicalStatus ?? "")) issues.push(page.canonicalRaw ? issue("canonical_invalid", page, page.canonicalRaw, { canonicalRaw: page.canonicalRaw }) : issue("canonical_missing", page));
|
|
10027
|
+
if (!["conflicting", "outside-head-only"].includes(page.canonicalStatus ?? "")) issues.push(page.canonicalRaw ? issue$1("canonical_invalid", page, page.canonicalRaw, { canonicalRaw: page.canonicalRaw }) : issue$1("canonical_missing", page));
|
|
9925
10028
|
} else {
|
|
9926
|
-
if (page.canonicalRaw && !absoluteHttpUrl$1(page.canonicalRaw)) issues.push(issue("canonical_non_absolute", page, page.canonicalRaw, {
|
|
10029
|
+
if (page.canonicalRaw && !absoluteHttpUrl$1(page.canonicalRaw)) issues.push(issue$1("canonical_non_absolute", page, page.canonicalRaw, {
|
|
9927
10030
|
canonicalRaw: page.canonicalRaw,
|
|
9928
10031
|
canonical: page.canonical
|
|
9929
10032
|
}));
|
|
9930
10033
|
const canonicalTarget = pageByUrl.get(urlKey(page.canonical) ?? "");
|
|
9931
|
-
if (canonicalTarget?.canonical && !sameUrl(canonicalTarget.canonical, page.canonical)) issues.push(issue("canonical_chain", page, page.canonical, {
|
|
10034
|
+
if (canonicalTarget?.canonical && !sameUrl(canonicalTarget.canonical, page.canonical)) issues.push(issue$1("canonical_chain", page, page.canonical, {
|
|
9932
10035
|
canonical: page.canonical,
|
|
9933
10036
|
nextCanonical: canonicalTarget.canonical,
|
|
9934
10037
|
chain: [
|
|
@@ -9938,25 +10041,25 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
9938
10041
|
]
|
|
9939
10042
|
}));
|
|
9940
10043
|
}
|
|
9941
|
-
if (hasMetaRobotsDirective(page.metaRobots, "noindex")) issues.push(issue("noindex", page, page.indexability, { metaRobots: page.metaRobots }));
|
|
9942
|
-
if (hasMetaRobotsDirective(page.metaRobots, "nofollow") && !xRobotsNofollow) issues.push(issue("nofollow", page, page.metaRobots, { metaRobots: page.metaRobots }));
|
|
9943
|
-
if (page.canonical && !sameUrl(page.canonical, page.finalUrl) && (page.declaredIndexability === "canonical-hint-other" || page.declaredIndexability === void 0 && /canonical/i.test(page.indexability ?? ""))) issues.push(issue("canonicalized_page", page, page.canonical, {
|
|
10044
|
+
if (hasMetaRobotsDirective(page.metaRobots, "noindex")) issues.push(issue$1("noindex", page, page.indexability, { metaRobots: page.metaRobots }));
|
|
10045
|
+
if (hasMetaRobotsDirective(page.metaRobots, "nofollow") && !xRobotsNofollow) issues.push(issue$1("nofollow", page, page.metaRobots, { metaRobots: page.metaRobots }));
|
|
10046
|
+
if (page.canonical && !sameUrl(page.canonical, page.finalUrl) && (page.declaredIndexability === "canonical-hint-other" || page.declaredIndexability === void 0 && /canonical/i.test(page.indexability ?? ""))) issues.push(issue$1("canonicalized_page", page, page.canonical, {
|
|
9944
10047
|
canonical: page.canonical,
|
|
9945
10048
|
finalUrl: page.finalUrl,
|
|
9946
10049
|
indexability: page.indexability
|
|
9947
10050
|
}));
|
|
9948
|
-
else if (page.canonical && !sameUrl(page.canonical, page.finalUrl)) issues.push(issue("canonical_mismatch", page, page.canonical, {
|
|
10051
|
+
else if (page.canonical && !sameUrl(page.canonical, page.finalUrl)) issues.push(issue$1("canonical_mismatch", page, page.canonical, {
|
|
9949
10052
|
canonical: page.canonical,
|
|
9950
10053
|
finalUrl: page.finalUrl
|
|
9951
10054
|
}));
|
|
9952
10055
|
const contentDuplicate = page.mainContentHash ? duplicateContent.get(page.mainContentHash) : void 0;
|
|
9953
|
-
if (contentDuplicate) issues.push(issue("duplicate_content", page, "Duplicate main content", {
|
|
10056
|
+
if (contentDuplicate) issues.push(issue$1("duplicate_content", page, "Duplicate main content", {
|
|
9954
10057
|
mainContentHash: page.mainContentHash,
|
|
9955
10058
|
duplicateCount: contentDuplicate.count,
|
|
9956
10059
|
sampleUrls: contentDuplicate.sampleUrls
|
|
9957
10060
|
}));
|
|
9958
10061
|
const coverage = queryCoverage(page);
|
|
9959
|
-
if (coverage && coverage.coverage < QUERY_COVERAGE_MIN) issues.push(issue("query_coverage_missing", page, coverage.query, {
|
|
10062
|
+
if (coverage && coverage.coverage < QUERY_COVERAGE_MIN) issues.push(issue$1("query_coverage_missing", page, coverage.query, {
|
|
9960
10063
|
query: coverage.query,
|
|
9961
10064
|
impressions: page.topQuery?.impressions,
|
|
9962
10065
|
matchedTerms: coverage.matchedTerms,
|
|
@@ -9964,36 +10067,36 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
9964
10067
|
coverage: coverage.coverage,
|
|
9965
10068
|
threshold: QUERY_COVERAGE_MIN
|
|
9966
10069
|
}));
|
|
9967
|
-
if ((page.imagesMissingAlt ?? 0) > 0) issues.push(issue("image_missing_alt", page, `${page.imagesMissingAlt} of ${page.imagesTotal ?? 0} images`, {
|
|
10070
|
+
if ((page.imagesMissingAlt ?? 0) > 0) issues.push(issue$1("image_missing_alt", page, `${page.imagesMissingAlt} of ${page.imagesTotal ?? 0} images`, {
|
|
9968
10071
|
imagesTotal: page.imagesTotal,
|
|
9969
10072
|
imagesMissingAlt: page.imagesMissingAlt
|
|
9970
10073
|
}));
|
|
9971
|
-
if (page.oversizedImageCandidates?.length) issues.push(issue("image_oversized_candidate", page, `${page.oversizedImageCandidates.length} image candidate${page.oversizedImageCandidates.length === 1 ? "" : "s"}`, {
|
|
10074
|
+
if (page.oversizedImageCandidates?.length) issues.push(issue$1("image_oversized_candidate", page, `${page.oversizedImageCandidates.length} image candidate${page.oversizedImageCandidates.length === 1 ? "" : "s"}`, {
|
|
9972
10075
|
thresholdPx: OVERSIZED_IMAGE_CANDIDATE_LIMIT,
|
|
9973
10076
|
candidates: page.oversizedImageCandidates.slice(0, 10)
|
|
9974
10077
|
}));
|
|
9975
|
-
if (!page.hasViewport) issues.push(issue("viewport_missing", page));
|
|
9976
|
-
if (!page.lang) issues.push(issue("lang_missing", page));
|
|
10078
|
+
if (!page.hasViewport) issues.push(issue$1("viewport_missing", page));
|
|
10079
|
+
if (!page.lang) issues.push(issue$1("lang_missing", page));
|
|
9977
10080
|
const hreflang = page.hreflang ?? [];
|
|
9978
10081
|
if (hreflang.length) {
|
|
9979
10082
|
const malformed = hreflang.filter((item) => !hasRecognizableHreflangFormat(item.hreflang));
|
|
9980
|
-
if (malformed.length) issues.push(issue("hreflang_invalid", page, `${malformed.length} malformed`, { malformed }));
|
|
10083
|
+
if (malformed.length) issues.push(issue$1("hreflang_invalid", page, `${malformed.length} malformed`, { malformed }));
|
|
9981
10084
|
const duplicateCodes = [...new Set(hreflang.map((item) => item.hreflang.trim().toLowerCase()).filter((code, index, values) => values.indexOf(code) !== index))];
|
|
9982
|
-
if (duplicateCodes.length) issues.push(issue("hreflang_duplicate", page, duplicateCodes.join(", "), { duplicateCodes }));
|
|
9983
|
-
if (!hreflang.some((item) => sameUrl(item.href, page.finalUrl))) issues.push(issue("hreflang_incomplete", page, page.lang, {
|
|
10085
|
+
if (duplicateCodes.length) issues.push(issue$1("hreflang_duplicate", page, duplicateCodes.join(", "), { duplicateCodes }));
|
|
10086
|
+
if (!hreflang.some((item) => sameUrl(item.href, page.finalUrl))) issues.push(issue$1("hreflang_incomplete", page, page.lang, {
|
|
9984
10087
|
lang: page.lang,
|
|
9985
10088
|
finalUrl: page.finalUrl,
|
|
9986
10089
|
hreflang
|
|
9987
10090
|
}));
|
|
9988
10091
|
}
|
|
9989
10092
|
const structuredDataFormats = page.structuredDataFormats;
|
|
9990
|
-
if (isAuditableHtmlPage(page) && page.indexable && structuredDataFormats && structuredDataFormats.length === 0 && (page.schemaTypes?.length ?? 0) === 0 && (page.invalidJsonLdCount ?? 0) === 0 && (page.unrecognizedJsonLdTypes?.length ?? 0) === 0) issues.push(issue("structured_data_missing", page, "No structured data detected", {
|
|
10093
|
+
if (isAuditableHtmlPage(page) && page.indexable && structuredDataFormats && structuredDataFormats.length === 0 && (page.schemaTypes?.length ?? 0) === 0 && (page.invalidJsonLdCount ?? 0) === 0 && (page.unrecognizedJsonLdTypes?.length ?? 0) === 0) issues.push(issue$1("structured_data_missing", page, "No structured data detected", {
|
|
9991
10094
|
structuredDataFormats,
|
|
9992
10095
|
schemaTypes: page.schemaTypes ?? [],
|
|
9993
10096
|
invalidJsonLdCount: page.invalidJsonLdCount ?? 0,
|
|
9994
10097
|
unrecognizedJsonLdTypeCount: page.unrecognizedJsonLdTypes?.length ?? 0
|
|
9995
10098
|
}));
|
|
9996
|
-
if ((page.invalidJsonLdCount ?? 0) > 0) issues.push(issue("jsonld_invalid", page, `${page.invalidJsonLdCount} invalid`, {
|
|
10099
|
+
if ((page.invalidJsonLdCount ?? 0) > 0) issues.push(issue$1("jsonld_invalid", page, `${page.invalidJsonLdCount} invalid`, {
|
|
9997
10100
|
invalidJsonLdCount: page.invalidJsonLdCount,
|
|
9998
10101
|
invalidJsonLdSamples: page.invalidJsonLdSamples
|
|
9999
10102
|
}));
|
|
@@ -10003,15 +10106,15 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
10003
10106
|
if (incompleteRichResultCount) {
|
|
10004
10107
|
const details = incompleteRichResults.map((assessment) => `${assessment.schemaType}: ${assessment.missingRequiredProperties.join(", ")}`);
|
|
10005
10108
|
const omittedDetails = incompleteRichResultCount - details.length;
|
|
10006
|
-
issues.push(issue("rich_result_required_fields_missing", page, [...details, ...omittedDetails > 0 ? [`${omittedDetails} more incomplete assessment${omittedDetails === 1 ? "" : "s"} omitted from bounded detail`] : []].join("; "), {
|
|
10109
|
+
issues.push(issue$1("rich_result_required_fields_missing", page, [...details, ...omittedDetails > 0 ? [`${omittedDetails} more incomplete assessment${omittedDetails === 1 ? "" : "s"} omitted from bounded detail`] : []].join("; "), {
|
|
10007
10110
|
assessments: incompleteRichResults,
|
|
10008
10111
|
...page.googleRichResultsSelection ? { selection: page.googleRichResultsSelection } : {}
|
|
10009
10112
|
}));
|
|
10010
10113
|
}
|
|
10011
|
-
if (!page.openGraphTitle) issues.push(issue("og_title_missing", page));
|
|
10012
|
-
if (!page.openGraphDescription) issues.push(issue("og_description_missing", page));
|
|
10013
|
-
if (!page.openGraphImage) issues.push(issue("og_image_missing", page));
|
|
10014
|
-
if (!page.twitterCard) issues.push(issue("twitter_card_missing", page));
|
|
10114
|
+
if (!page.openGraphTitle) issues.push(issue$1("og_title_missing", page));
|
|
10115
|
+
if (!page.openGraphDescription) issues.push(issue$1("og_description_missing", page));
|
|
10116
|
+
if (!page.openGraphImage) issues.push(issue$1("og_image_missing", page));
|
|
10117
|
+
if (!page.twitterCard) issues.push(issue$1("twitter_card_missing", page));
|
|
10015
10118
|
}
|
|
10016
10119
|
return issues;
|
|
10017
10120
|
}
|
|
@@ -10669,7 +10772,7 @@ const SEVERITY_SCORE = {
|
|
|
10669
10772
|
const EFFORT_BY_CATEGORY = {
|
|
10670
10773
|
canonical: "medium",
|
|
10671
10774
|
content: "high",
|
|
10672
|
-
|
|
10775
|
+
agents: "low",
|
|
10673
10776
|
headings: "low",
|
|
10674
10777
|
images: "medium",
|
|
10675
10778
|
indexability: "medium",
|
|
@@ -11582,6 +11685,92 @@ const crawlSiteChecksSchema = z.object({ soft404: z.object({
|
|
|
11582
11685
|
complete: z.boolean()
|
|
11583
11686
|
}) });
|
|
11584
11687
|
//#endregion
|
|
11688
|
+
//#region packages/core/src/analyze/crawler/site-crawl-ai-schema.ts
|
|
11689
|
+
const crawlAiAvailabilitySchema = z.enum([
|
|
11690
|
+
"available",
|
|
11691
|
+
"absent",
|
|
11692
|
+
"access-blocked",
|
|
11693
|
+
"rate-limited",
|
|
11694
|
+
"unreachable"
|
|
11695
|
+
]);
|
|
11696
|
+
const crawlAiLinkCheckSchema = z.object({
|
|
11697
|
+
url: z.string().url(),
|
|
11698
|
+
status: z.number().int().optional(),
|
|
11699
|
+
contentType: z.string().optional(),
|
|
11700
|
+
error: z.string().optional()
|
|
11701
|
+
});
|
|
11702
|
+
const crawlAiSignalsSchema = z.object({
|
|
11703
|
+
robotsTxt: z.object({
|
|
11704
|
+
url: z.string().url(),
|
|
11705
|
+
exists: z.boolean(),
|
|
11706
|
+
availability: crawlAiAvailabilitySchema,
|
|
11707
|
+
status: z.number().int().optional(),
|
|
11708
|
+
error: z.string().optional(),
|
|
11709
|
+
sitemapUrls: z.array(z.string().url()),
|
|
11710
|
+
contentSignals: z.array(z.string()).optional(),
|
|
11711
|
+
botAccess: z.array(z.object({
|
|
11712
|
+
userAgent: z.string(),
|
|
11713
|
+
purpose: z.enum([
|
|
11714
|
+
"search",
|
|
11715
|
+
"training",
|
|
11716
|
+
"other"
|
|
11717
|
+
]),
|
|
11718
|
+
allowed: z.boolean().nullable(),
|
|
11719
|
+
declared: z.boolean(),
|
|
11720
|
+
coveredByWildcard: z.boolean()
|
|
11721
|
+
}))
|
|
11722
|
+
}).optional(),
|
|
11723
|
+
llmsTxt: z.object({
|
|
11724
|
+
url: z.string().url(),
|
|
11725
|
+
exists: z.boolean(),
|
|
11726
|
+
availability: crawlAiAvailabilitySchema.optional(),
|
|
11727
|
+
status: z.number().int().optional(),
|
|
11728
|
+
error: z.string().optional(),
|
|
11729
|
+
links: z.object({
|
|
11730
|
+
found: z.number().int().nonnegative(),
|
|
11731
|
+
checked: z.number().int().nonnegative(),
|
|
11732
|
+
limit: z.number().int().positive(),
|
|
11733
|
+
broken: z.array(crawlAiLinkCheckSchema)
|
|
11734
|
+
}).optional()
|
|
11735
|
+
}).optional(),
|
|
11736
|
+
agentResources: z.array(z.object({
|
|
11737
|
+
url: z.string().url(),
|
|
11738
|
+
exists: z.boolean(),
|
|
11739
|
+
status: z.number().int().optional(),
|
|
11740
|
+
contentType: z.string().optional(),
|
|
11741
|
+
validJson: z.boolean().optional()
|
|
11742
|
+
})).optional(),
|
|
11743
|
+
markdownNegotiation: z.object({
|
|
11744
|
+
url: z.string().url(),
|
|
11745
|
+
availability: crawlAiAvailabilitySchema,
|
|
11746
|
+
status: z.number().int().optional(),
|
|
11747
|
+
contentType: z.string().optional(),
|
|
11748
|
+
markdown: z.boolean(),
|
|
11749
|
+
varyAccept: z.boolean(),
|
|
11750
|
+
error: z.string().optional()
|
|
11751
|
+
}).optional(),
|
|
11752
|
+
markdownCopies: z.object({
|
|
11753
|
+
publicPages: z.number().int().nonnegative(),
|
|
11754
|
+
publicPagesWithLink: z.number().int().nonnegative(),
|
|
11755
|
+
privatePagesWithLink: z.number().int().nonnegative(),
|
|
11756
|
+
checked: z.number().int().nonnegative(),
|
|
11757
|
+
limit: z.number().int().positive(),
|
|
11758
|
+
broken: z.array(crawlAiLinkCheckSchema.extend({
|
|
11759
|
+
page: z.string().url(),
|
|
11760
|
+
public: z.boolean()
|
|
11761
|
+
})),
|
|
11762
|
+
largest: z.object({
|
|
11763
|
+
url: z.string().url(),
|
|
11764
|
+
estimatedTokens: z.number().int().nonnegative()
|
|
11765
|
+
}).optional()
|
|
11766
|
+
}).optional(),
|
|
11767
|
+
startPageHeaders: z.object({
|
|
11768
|
+
url: z.string().url(),
|
|
11769
|
+
linkRelations: z.array(z.string()),
|
|
11770
|
+
contentSignal: z.string().optional()
|
|
11771
|
+
}).optional()
|
|
11772
|
+
});
|
|
11773
|
+
//#endregion
|
|
11585
11774
|
//#region packages/core/src/analyze/crawler/schemas.ts
|
|
11586
11775
|
const crawlerRuleSeveritySchema = z.enum([
|
|
11587
11776
|
"low",
|
|
@@ -11604,7 +11793,7 @@ const crawlerRuleCategorySchema = z.enum([
|
|
|
11604
11793
|
"security",
|
|
11605
11794
|
"social",
|
|
11606
11795
|
"structured-data",
|
|
11607
|
-
"
|
|
11796
|
+
"agents"
|
|
11608
11797
|
]);
|
|
11609
11798
|
const crawlerRuleIdSchema = z.enum(listRules().map((rule) => rule.id));
|
|
11610
11799
|
const crawlerRuleInfoSchema = z.object({
|
|
@@ -12230,41 +12419,6 @@ const crawlIssueGroupSchema = z.object({
|
|
|
12230
12419
|
count: z.number().int(),
|
|
12231
12420
|
sampleUrls: z.array(z.string().url())
|
|
12232
12421
|
});
|
|
12233
|
-
const crawlAiSignalsSchema = z.object({
|
|
12234
|
-
robotsTxt: z.object({
|
|
12235
|
-
url: z.string().url(),
|
|
12236
|
-
exists: z.boolean(),
|
|
12237
|
-
availability: z.enum([
|
|
12238
|
-
"available",
|
|
12239
|
-
"absent",
|
|
12240
|
-
"access-blocked",
|
|
12241
|
-
"rate-limited",
|
|
12242
|
-
"unreachable"
|
|
12243
|
-
]),
|
|
12244
|
-
status: z.number().int().optional(),
|
|
12245
|
-
error: z.string().optional(),
|
|
12246
|
-
sitemapUrls: z.array(z.string().url()),
|
|
12247
|
-
contentSignals: z.array(z.string()).optional(),
|
|
12248
|
-
botAccess: z.array(z.object({
|
|
12249
|
-
userAgent: z.string(),
|
|
12250
|
-
allowed: z.boolean().nullable(),
|
|
12251
|
-
declared: z.boolean(),
|
|
12252
|
-
coveredByWildcard: z.boolean()
|
|
12253
|
-
}))
|
|
12254
|
-
}).optional(),
|
|
12255
|
-
llmsTxt: z.object({
|
|
12256
|
-
url: z.string().url(),
|
|
12257
|
-
exists: z.boolean(),
|
|
12258
|
-
status: z.number().int().optional()
|
|
12259
|
-
}).optional(),
|
|
12260
|
-
agentResources: z.array(z.object({
|
|
12261
|
-
url: z.string().url(),
|
|
12262
|
-
exists: z.boolean(),
|
|
12263
|
-
status: z.number().int().optional(),
|
|
12264
|
-
contentType: z.string().optional(),
|
|
12265
|
-
validJson: z.boolean().optional()
|
|
12266
|
-
})).optional()
|
|
12267
|
-
});
|
|
12268
12422
|
const crawlTopFixSchema = crawlIssueGroupSchema.extend({
|
|
12269
12423
|
recommendation: crawlerRuleRecommendationSchema,
|
|
12270
12424
|
score: z.number(),
|
|
@@ -12576,6 +12730,58 @@ const crawlerJsonSchemas = {
|
|
|
12576
12730
|
pageSnapshot: z.toJSONSchema(crawlPageSnapshotSchema)
|
|
12577
12731
|
};
|
|
12578
12732
|
//#endregion
|
|
12733
|
+
//#region packages/core/src/analyze/crawler/agent-ready.ts
|
|
12734
|
+
function needsSignIn(page) {
|
|
12735
|
+
return page.status === 401 || page.status === 403 || Boolean(page.softAuthenticationGate);
|
|
12736
|
+
}
|
|
12737
|
+
function isPublicPage(page) {
|
|
12738
|
+
return page.status >= 200 && page.status < 300 && !needsSignIn(page) && /\btext\/html\b/i.test(page.contentType ?? "");
|
|
12739
|
+
}
|
|
12740
|
+
function expectsMarkdown(page) {
|
|
12741
|
+
return isPublicPage(page) && page.indexable;
|
|
12742
|
+
}
|
|
12743
|
+
function issue(ruleId, url, detail, evidence) {
|
|
12744
|
+
const rule = explainRule(ruleId);
|
|
12745
|
+
if (!rule) throw new Error(`Missing rule guidance for ${ruleId}`);
|
|
12746
|
+
return {
|
|
12747
|
+
ruleId,
|
|
12748
|
+
title: rule.title,
|
|
12749
|
+
category: rule.category,
|
|
12750
|
+
severity: rule.defaultSeverity,
|
|
12751
|
+
url,
|
|
12752
|
+
detail,
|
|
12753
|
+
...evidence ? { evidence } : {}
|
|
12754
|
+
};
|
|
12755
|
+
}
|
|
12756
|
+
function agentReadyIssues(input) {
|
|
12757
|
+
const { ai, startUrl } = input;
|
|
12758
|
+
const issues = [];
|
|
12759
|
+
const blocked = (ai.robotsTxt?.botAccess ?? []).filter((bot) => bot.purpose === "search" && bot.allowed === false);
|
|
12760
|
+
if (blocked.length && ai.robotsTxt) issues.push(issue("ai_search_bot_blocked", ai.robotsTxt.url, `robots.txt blocks ${blocked.map((bot) => bot.userAgent).join(", ")} from the start URL.`, { blocked: blocked.map((bot) => bot.userAgent) }));
|
|
12761
|
+
if (ai.llmsTxt?.availability === "absent") issues.push(issue("llms_txt_missing", ai.llmsTxt.url, `/llms.txt returned ${ai.llmsTxt.status === void 0 ? "no file" : `HTTP ${ai.llmsTxt.status}`}.`));
|
|
12762
|
+
for (const link of ai.llmsTxt?.links?.broken ?? []) issues.push(issue("llms_txt_broken_link", link.url, link.status === void 0 ? `The link in llms.txt could not be fetched: ${link.error ?? "no response"}.` : `The link in llms.txt returned HTTP ${link.status}.`, {
|
|
12763
|
+
llmsTxt: ai.llmsTxt?.url,
|
|
12764
|
+
...link
|
|
12765
|
+
}));
|
|
12766
|
+
const expected = input.pages.filter(expectsMarkdown);
|
|
12767
|
+
const linked = expected.filter((page) => page.markdownAlternates?.length);
|
|
12768
|
+
const negotiation = ai.markdownNegotiation;
|
|
12769
|
+
if (expected.length && !linked.length && negotiation?.availability === "available" && !negotiation.markdown) issues.push(issue("markdown_missing", startUrl, `Accept: text/markdown returned ${negotiation.contentType || "no content type"}, and none of ${expected.length} public pages links to a Markdown copy.`, { negotiation }));
|
|
12770
|
+
if (linked.length) for (const page of expected) {
|
|
12771
|
+
if (page.markdownAlternates?.length) continue;
|
|
12772
|
+
issues.push(issue("markdown_alternate_missing", page.url, `${linked.length} of ${expected.length} public pages link to a Markdown copy; this one does not.`));
|
|
12773
|
+
}
|
|
12774
|
+
for (const page of input.pages) {
|
|
12775
|
+
if (!needsSignIn(page) || !page.markdownAlternates?.length) continue;
|
|
12776
|
+
issues.push(issue("markdown_alternate_on_private_page", page.url, `The page needs a sign-in but links to a Markdown copy at ${page.markdownAlternates[0]}.`, { markdownAlternates: page.markdownAlternates }));
|
|
12777
|
+
}
|
|
12778
|
+
for (const copy of ai.markdownCopies?.broken ?? []) {
|
|
12779
|
+
if (!copy.public) continue;
|
|
12780
|
+
issues.push(issue("markdown_copy_broken", copy.page, copy.status === void 0 ? `The Markdown copy at ${copy.url} could not be fetched: ${copy.error ?? "no response"}.` : `The Markdown copy at ${copy.url} returned HTTP ${copy.status}${copy.contentType ? ` as ${copy.contentType}` : ""}.`, { ...copy }));
|
|
12781
|
+
}
|
|
12782
|
+
return issues;
|
|
12783
|
+
}
|
|
12784
|
+
//#endregion
|
|
12579
12785
|
//#region packages/core/src/analyze/crawler/crawl-caveats.ts
|
|
12580
12786
|
function crawlCaveats(input) {
|
|
12581
12787
|
const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
|
|
@@ -13013,66 +13219,137 @@ async function collectCrawlSiteChecks(input) {
|
|
|
13013
13219
|
//#endregion
|
|
13014
13220
|
//#region packages/core/src/analyze/crawler/site-crawl-ai-signals.ts
|
|
13015
13221
|
const MAX_AUXILIARY_RESPONSE_BYTES = 2 * 1024 * 1024;
|
|
13016
|
-
const
|
|
13017
|
-
|
|
13018
|
-
"OAI-SearchBot",
|
|
13019
|
-
"ChatGPT-User",
|
|
13020
|
-
"
|
|
13021
|
-
"Claude-User",
|
|
13022
|
-
"
|
|
13023
|
-
"
|
|
13024
|
-
"
|
|
13025
|
-
"
|
|
13026
|
-
"
|
|
13027
|
-
"Applebot",
|
|
13028
|
-
"
|
|
13029
|
-
"
|
|
13030
|
-
"
|
|
13031
|
-
"
|
|
13032
|
-
"
|
|
13033
|
-
"Meta-
|
|
13034
|
-
"
|
|
13035
|
-
"
|
|
13036
|
-
"
|
|
13037
|
-
"
|
|
13222
|
+
const LLMS_LINK_LIMIT = 20;
|
|
13223
|
+
const AI_BOTS = [
|
|
13224
|
+
["OAI-SearchBot", "search"],
|
|
13225
|
+
["ChatGPT-User", "search"],
|
|
13226
|
+
["Claude-SearchBot", "search"],
|
|
13227
|
+
["Claude-User", "search"],
|
|
13228
|
+
["PerplexityBot", "search"],
|
|
13229
|
+
["Perplexity-User", "search"],
|
|
13230
|
+
["DuckAssistBot", "search"],
|
|
13231
|
+
["MistralAI-User", "search"],
|
|
13232
|
+
["Meta-ExternalFetcher", "search"],
|
|
13233
|
+
["Applebot", "search"],
|
|
13234
|
+
["Bingbot", "search"],
|
|
13235
|
+
["GPTBot", "training"],
|
|
13236
|
+
["ClaudeBot", "training"],
|
|
13237
|
+
["Google-Extended", "training"],
|
|
13238
|
+
["Applebot-Extended", "training"],
|
|
13239
|
+
["Meta-ExternalAgent", "training"],
|
|
13240
|
+
["CCBot", "training"],
|
|
13241
|
+
["Bytespider", "training"],
|
|
13242
|
+
["cohere-ai", "training"],
|
|
13243
|
+
["Amazonbot", "other"],
|
|
13244
|
+
["Diffbot", "other"]
|
|
13038
13245
|
];
|
|
13039
13246
|
const AGENT_RESOURCE_PATHS = [
|
|
13040
|
-
"/.well-known/agent.json",
|
|
13041
|
-
"/
|
|
13247
|
+
"/.well-known/agent-skills/index.json",
|
|
13248
|
+
"/.well-known/mcp/server-card.json",
|
|
13042
13249
|
"/.well-known/mcp.json",
|
|
13043
|
-
"/.well-known/
|
|
13044
|
-
"
|
|
13045
|
-
"/
|
|
13250
|
+
"/.well-known/api-catalog",
|
|
13251
|
+
"/openapi.json",
|
|
13252
|
+
"/.well-known/oauth-protected-resource",
|
|
13253
|
+
"/.well-known/oauth-authorization-server",
|
|
13254
|
+
"/.well-known/http-message-signatures-directory"
|
|
13046
13255
|
];
|
|
13047
|
-
|
|
13048
|
-
|
|
13256
|
+
function availabilityOf(status) {
|
|
13257
|
+
if (status >= 200 && status < 300) return "available";
|
|
13258
|
+
if (status === 401 || status === 403) return "access-blocked";
|
|
13259
|
+
if (status === 429) return "rate-limited";
|
|
13260
|
+
if (status >= 400 && status < 500) return "absent";
|
|
13261
|
+
return "unreachable";
|
|
13262
|
+
}
|
|
13263
|
+
function errorText(error) {
|
|
13264
|
+
return error instanceof Error ? error.message : String(error);
|
|
13265
|
+
}
|
|
13266
|
+
async function get(input, url, headers, readBody = true) {
|
|
13049
13267
|
const controller = abortController({
|
|
13050
13268
|
timeoutMs: input.timeoutMs,
|
|
13051
13269
|
signal: input.signal
|
|
13052
13270
|
});
|
|
13053
13271
|
try {
|
|
13054
|
-
const response = await input.fetch(
|
|
13272
|
+
const response = await input.fetch(url, {
|
|
13055
13273
|
profile: "bot",
|
|
13056
13274
|
redirect: "follow",
|
|
13057
|
-
signal: controller.signal
|
|
13275
|
+
signal: controller.signal,
|
|
13276
|
+
...headers ? { headers } : {}
|
|
13058
13277
|
});
|
|
13059
|
-
|
|
13060
|
-
|
|
13061
|
-
await response.body?.cancel().catch(() => void 0);
|
|
13278
|
+
let text = "";
|
|
13279
|
+
if (readBody) text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, `${new URL(url).pathname} response`);
|
|
13280
|
+
else await response.body?.cancel().catch(() => void 0);
|
|
13062
13281
|
return {
|
|
13063
|
-
url: response.url ||
|
|
13064
|
-
|
|
13065
|
-
status: response.status
|
|
13282
|
+
url: response.url || url,
|
|
13283
|
+
availability: availabilityOf(response.status),
|
|
13284
|
+
status: response.status,
|
|
13285
|
+
contentType: response.headers.get("content-type") ?? "",
|
|
13286
|
+
headers: response.headers,
|
|
13287
|
+
text
|
|
13066
13288
|
};
|
|
13067
|
-
} catch {
|
|
13289
|
+
} catch (error) {
|
|
13068
13290
|
return {
|
|
13069
|
-
url
|
|
13070
|
-
|
|
13291
|
+
url,
|
|
13292
|
+
availability: "unreachable",
|
|
13293
|
+
contentType: "",
|
|
13294
|
+
text: "",
|
|
13295
|
+
error: errorText(error)
|
|
13071
13296
|
};
|
|
13072
13297
|
} finally {
|
|
13073
13298
|
controller.cleanup();
|
|
13074
13299
|
}
|
|
13075
13300
|
}
|
|
13301
|
+
const isHtml = (contentType) => /\btext\/html\b/i.test(contentType);
|
|
13302
|
+
const isMarkdown = (contentType) => /\btext\/(?:x-)?markdown\b/i.test(contentType);
|
|
13303
|
+
function brokenLink(result, expectText) {
|
|
13304
|
+
if (result.availability === "available" && !(expectText && isHtml(result.contentType))) return void 0;
|
|
13305
|
+
return {
|
|
13306
|
+
url: result.url,
|
|
13307
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
13308
|
+
...result.contentType ? { contentType: result.contentType } : {},
|
|
13309
|
+
...result.error ? { error: result.error } : {}
|
|
13310
|
+
};
|
|
13311
|
+
}
|
|
13312
|
+
function llmsLinks(text, base) {
|
|
13313
|
+
const origin = new URL(base).origin;
|
|
13314
|
+
const links = [];
|
|
13315
|
+
for (const match of text.matchAll(/\]\(\s*<?([^)\s>]+)>?/g)) try {
|
|
13316
|
+
const url = new URL(match[1] ?? "", base);
|
|
13317
|
+
url.hash = "";
|
|
13318
|
+
if (url.origin === origin && !links.includes(url.toString())) links.push(url.toString());
|
|
13319
|
+
} catch {}
|
|
13320
|
+
return links;
|
|
13321
|
+
}
|
|
13322
|
+
async function checkLlmsTxt(input) {
|
|
13323
|
+
const result = await get(input, new URL("/llms.txt", input.url).toString());
|
|
13324
|
+
const availability = result.availability === "available" && isHtml(result.contentType) ? "absent" : result.availability;
|
|
13325
|
+
const exists = availability === "available";
|
|
13326
|
+
const base = {
|
|
13327
|
+
url: result.url,
|
|
13328
|
+
exists,
|
|
13329
|
+
availability,
|
|
13330
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
13331
|
+
...result.error ? { error: result.error } : {}
|
|
13332
|
+
};
|
|
13333
|
+
if (!exists) return base;
|
|
13334
|
+
const found = llmsLinks(result.text, result.url);
|
|
13335
|
+
const broken = [];
|
|
13336
|
+
let checked = 0;
|
|
13337
|
+
for (const url of found.slice(0, LLMS_LINK_LIMIT)) {
|
|
13338
|
+
if (input.signal?.aborted) break;
|
|
13339
|
+
checked += 1;
|
|
13340
|
+
const link = brokenLink(await get(input, url, void 0, false), false);
|
|
13341
|
+
if (link) broken.push(link);
|
|
13342
|
+
}
|
|
13343
|
+
return {
|
|
13344
|
+
...base,
|
|
13345
|
+
links: {
|
|
13346
|
+
found: found.length,
|
|
13347
|
+
checked,
|
|
13348
|
+
limit: LLMS_LINK_LIMIT,
|
|
13349
|
+
broken
|
|
13350
|
+
}
|
|
13351
|
+
};
|
|
13352
|
+
}
|
|
13076
13353
|
function parseRobots(robotsUrl, text) {
|
|
13077
13354
|
return robotsParserModule(robotsUrl, text);
|
|
13078
13355
|
}
|
|
@@ -13104,118 +13381,135 @@ function sitemapUrlsFromRobots(text) {
|
|
|
13104
13381
|
return [...urls];
|
|
13105
13382
|
}
|
|
13106
13383
|
async function checkRobotsAiAccess(input) {
|
|
13107
|
-
const
|
|
13108
|
-
const
|
|
13109
|
-
|
|
13110
|
-
|
|
13111
|
-
|
|
13112
|
-
|
|
13113
|
-
|
|
13114
|
-
|
|
13115
|
-
redirect: "follow",
|
|
13116
|
-
signal: controller.signal
|
|
13117
|
-
});
|
|
13118
|
-
const text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, "robots.txt response");
|
|
13119
|
-
const available = response.status >= 200 && response.status < 300;
|
|
13120
|
-
const absent = response.status >= 400 && response.status < 500 && response.status !== 429;
|
|
13121
|
-
const availability = available ? "available" : [401, 403].includes(response.status) ? "access-blocked" : response.status === 429 ? "rate-limited" : absent ? "absent" : "unreachable";
|
|
13122
|
-
const exists = available;
|
|
13123
|
-
const declared = exists ? declaredUserAgents(text) : /* @__PURE__ */ new Set();
|
|
13124
|
-
const parsed = parseRobots(robotsUrl, exists ? text : "");
|
|
13125
|
-
return {
|
|
13126
|
-
url: response.url || robotsUrl,
|
|
13384
|
+
const result = await get(input, new URL("/robots.txt", input.url).toString());
|
|
13385
|
+
const exists = result.availability === "available";
|
|
13386
|
+
const unknown = result.availability === "rate-limited" || result.availability === "unreachable";
|
|
13387
|
+
const declared = exists ? declaredUserAgents(result.text) : /* @__PURE__ */ new Set();
|
|
13388
|
+
const parsed = parseRobots(result.url, exists ? result.text : "");
|
|
13389
|
+
return {
|
|
13390
|
+
robotsTxt: {
|
|
13391
|
+
url: result.url,
|
|
13127
13392
|
exists,
|
|
13128
|
-
availability,
|
|
13129
|
-
status:
|
|
13130
|
-
...
|
|
13131
|
-
sitemapUrls: exists ? sitemapUrlsFromRobots(text) : [],
|
|
13132
|
-
contentSignals: exists ? contentSignalsFromRobots(text) : [],
|
|
13133
|
-
botAccess:
|
|
13134
|
-
const lower = userAgent.toLowerCase();
|
|
13135
|
-
return {
|
|
13136
|
-
userAgent,
|
|
13137
|
-
allowed: availability === "rate-limited" || availability === "unreachable" ? null : parsed.isAllowed(input.url, userAgent) ?? true,
|
|
13138
|
-
declared: declared.has(lower),
|
|
13139
|
-
coveredByWildcard: declared.has("*")
|
|
13140
|
-
};
|
|
13141
|
-
})
|
|
13142
|
-
};
|
|
13143
|
-
} catch (error) {
|
|
13144
|
-
return {
|
|
13145
|
-
url: robotsUrl,
|
|
13146
|
-
exists: false,
|
|
13147
|
-
availability: "unreachable",
|
|
13148
|
-
error: error instanceof Error ? error.message : String(error),
|
|
13149
|
-
sitemapUrls: [],
|
|
13150
|
-
botAccess: AI_BOT_USER_AGENTS.map((userAgent) => ({
|
|
13393
|
+
availability: result.availability,
|
|
13394
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
13395
|
+
...unknown ? { error: result.error ?? `robots.txt returned HTTP ${result.status}.` } : {},
|
|
13396
|
+
sitemapUrls: exists ? sitemapUrlsFromRobots(result.text) : [],
|
|
13397
|
+
...unknown ? {} : { contentSignals: exists ? contentSignalsFromRobots(result.text) : [] },
|
|
13398
|
+
botAccess: AI_BOTS.map(([userAgent, purpose]) => ({
|
|
13151
13399
|
userAgent,
|
|
13152
|
-
|
|
13153
|
-
|
|
13154
|
-
|
|
13400
|
+
purpose,
|
|
13401
|
+
allowed: unknown ? null : parsed.isAllowed(input.url, userAgent) ?? true,
|
|
13402
|
+
declared: declared.has(userAgent.toLowerCase()),
|
|
13403
|
+
coveredByWildcard: declared.has("*")
|
|
13155
13404
|
}))
|
|
13156
|
-
}
|
|
13157
|
-
|
|
13158
|
-
|
|
13159
|
-
}
|
|
13405
|
+
},
|
|
13406
|
+
allows: (url) => unknown ? null : parsed.isAllowed(url, SEO_CRAWLER_USER_AGENT) ?? true
|
|
13407
|
+
};
|
|
13160
13408
|
}
|
|
13161
|
-
async function
|
|
13162
|
-
const
|
|
13163
|
-
const
|
|
13164
|
-
|
|
13165
|
-
|
|
13166
|
-
|
|
13167
|
-
try {
|
|
13168
|
-
const response = await input.fetch(url, {
|
|
13169
|
-
profile: "bot",
|
|
13170
|
-
redirect: "follow",
|
|
13171
|
-
signal: controller.signal
|
|
13172
|
-
});
|
|
13173
|
-
const contentType = response.headers.get("content-type") ?? "";
|
|
13174
|
-
const text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, "Agent resource response");
|
|
13175
|
-
const exists = response.status >= 200 && response.status < 300;
|
|
13409
|
+
async function checkAgentResources(input) {
|
|
13410
|
+
const resources = [];
|
|
13411
|
+
for (const path of AGENT_RESOURCE_PATHS) {
|
|
13412
|
+
if (input.signal?.aborted) break;
|
|
13413
|
+
const result = await get(input, new URL(path, input.url).toString());
|
|
13414
|
+
const exists = result.availability === "available" && !isHtml(result.contentType);
|
|
13176
13415
|
let validJson;
|
|
13177
|
-
if (exists && /\bjson\b/i.test(contentType)) try {
|
|
13178
|
-
JSON.parse(text);
|
|
13416
|
+
if (exists && /\bjson\b/i.test(result.contentType)) try {
|
|
13417
|
+
JSON.parse(result.text);
|
|
13179
13418
|
validJson = true;
|
|
13180
13419
|
} catch {
|
|
13181
13420
|
validJson = false;
|
|
13182
13421
|
}
|
|
13183
|
-
|
|
13184
|
-
url:
|
|
13422
|
+
resources.push({
|
|
13423
|
+
url: result.url,
|
|
13185
13424
|
exists,
|
|
13186
|
-
status:
|
|
13187
|
-
contentType,
|
|
13425
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
13426
|
+
...result.contentType ? { contentType: result.contentType } : {},
|
|
13188
13427
|
...validJson === void 0 ? {} : { validJson }
|
|
13189
|
-
};
|
|
13190
|
-
} catch {
|
|
13191
|
-
return {
|
|
13192
|
-
url,
|
|
13193
|
-
exists: false
|
|
13194
|
-
};
|
|
13195
|
-
} finally {
|
|
13196
|
-
controller.cleanup();
|
|
13197
|
-
}
|
|
13198
|
-
}
|
|
13199
|
-
async function checkAgentResources(input) {
|
|
13200
|
-
const resources = [];
|
|
13201
|
-
for (const path of AGENT_RESOURCE_PATHS) {
|
|
13202
|
-
if (input.signal?.aborted) break;
|
|
13203
|
-
resources.push(await checkAgentResource({
|
|
13204
|
-
baseUrl: input.url,
|
|
13205
|
-
path,
|
|
13206
|
-
timeoutMs: input.timeoutMs,
|
|
13207
|
-
fetch: input.fetch,
|
|
13208
|
-
signal: input.signal
|
|
13209
|
-
}));
|
|
13428
|
+
});
|
|
13210
13429
|
}
|
|
13211
13430
|
return resources;
|
|
13212
13431
|
}
|
|
13213
|
-
function
|
|
13214
|
-
|
|
13432
|
+
async function checkMarkdownNegotiation(input) {
|
|
13433
|
+
const result = await get(input, input.url, { accept: "text/markdown" }, false);
|
|
13434
|
+
const vary = result.headers?.get("vary") ?? "";
|
|
13435
|
+
return {
|
|
13436
|
+
url: result.url,
|
|
13437
|
+
availability: result.availability,
|
|
13438
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
13439
|
+
...result.contentType ? { contentType: result.contentType } : {},
|
|
13440
|
+
markdown: result.availability === "available" && isMarkdown(result.contentType),
|
|
13441
|
+
varyAccept: vary.split(",").some((value) => value.trim().toLowerCase() === "accept"),
|
|
13442
|
+
...result.error ? { error: result.error } : {}
|
|
13443
|
+
};
|
|
13444
|
+
}
|
|
13445
|
+
async function collectCrawlAiSignals(input) {
|
|
13446
|
+
const robots = await checkRobotsAiAccess(input);
|
|
13447
|
+
const allows = input.respectRobots ? robots.allows : () => true;
|
|
13448
|
+
const startAllowed = allows(input.url);
|
|
13449
|
+
const [llmsTxt, agentResources, markdownNegotiation] = await Promise.all([
|
|
13215
13450
|
checkLlmsTxt(input),
|
|
13216
|
-
|
|
13217
|
-
|
|
13451
|
+
checkAgentResources(input),
|
|
13452
|
+
startAllowed ? checkMarkdownNegotiation(input) : Promise.resolve({
|
|
13453
|
+
url: input.url,
|
|
13454
|
+
availability: "unreachable",
|
|
13455
|
+
markdown: false,
|
|
13456
|
+
varyAccept: false,
|
|
13457
|
+
error: startAllowed === null ? "Not requested: robots.txt could not be read." : "Not requested: robots.txt disallows the start URL."
|
|
13458
|
+
})
|
|
13218
13459
|
]);
|
|
13460
|
+
return {
|
|
13461
|
+
signals: {
|
|
13462
|
+
llmsTxt,
|
|
13463
|
+
robotsTxt: robots.robotsTxt,
|
|
13464
|
+
agentResources,
|
|
13465
|
+
markdownNegotiation
|
|
13466
|
+
},
|
|
13467
|
+
allows
|
|
13468
|
+
};
|
|
13469
|
+
}
|
|
13470
|
+
async function checkMarkdownCopies(input) {
|
|
13471
|
+
const linked = input.pages.filter((page) => page.markdownAlternates?.length).sort((a, b) => Number(a.public) - Number(b.public) || a.url.localeCompare(b.url));
|
|
13472
|
+
const broken = [];
|
|
13473
|
+
let largest;
|
|
13474
|
+
let checked = 0;
|
|
13475
|
+
for (const page of linked.slice(0, 20)) {
|
|
13476
|
+
if (input.signal?.aborted) break;
|
|
13477
|
+
const url = page.markdownAlternates?.[0];
|
|
13478
|
+
if (!url || !input.allows(url)) continue;
|
|
13479
|
+
checked += 1;
|
|
13480
|
+
const result = await get({
|
|
13481
|
+
...input,
|
|
13482
|
+
url
|
|
13483
|
+
}, url);
|
|
13484
|
+
const link = brokenLink(result, true);
|
|
13485
|
+
if (link) {
|
|
13486
|
+
broken.push({
|
|
13487
|
+
...link,
|
|
13488
|
+
page: page.url,
|
|
13489
|
+
public: page.public
|
|
13490
|
+
});
|
|
13491
|
+
continue;
|
|
13492
|
+
}
|
|
13493
|
+
const estimatedTokens = Math.ceil(result.text.length / 4);
|
|
13494
|
+
if (!largest || estimatedTokens > largest.estimatedTokens) largest = {
|
|
13495
|
+
url: result.url,
|
|
13496
|
+
estimatedTokens
|
|
13497
|
+
};
|
|
13498
|
+
}
|
|
13499
|
+
return {
|
|
13500
|
+
publicPages: input.pages.filter((page) => page.public).length,
|
|
13501
|
+
publicPagesWithLink: linked.filter((page) => page.public).length,
|
|
13502
|
+
privatePagesWithLink: linked.filter((page) => !page.public).length,
|
|
13503
|
+
checked,
|
|
13504
|
+
limit: 20,
|
|
13505
|
+
broken,
|
|
13506
|
+
...largest ? { largest } : {}
|
|
13507
|
+
};
|
|
13508
|
+
}
|
|
13509
|
+
function linkRelations(header) {
|
|
13510
|
+
const relations = /* @__PURE__ */ new Set();
|
|
13511
|
+
for (const match of (header ?? "").matchAll(/rel\s*=\s*"?([^";,]+)"?/gi)) for (const value of (match[1] ?? "").trim().toLowerCase().split(/\s+/)) if (value) relations.add(value);
|
|
13512
|
+
return [...relations].sort();
|
|
13219
13513
|
}
|
|
13220
13514
|
//#endregion
|
|
13221
13515
|
//#region packages/core/src/analyze/crawler/site-crawl-providers.ts
|
|
@@ -15731,12 +16025,11 @@ async function crawlSite(input, dependencies) {
|
|
|
15731
16025
|
url: config.url,
|
|
15732
16026
|
timeoutMs: config.timeoutMs,
|
|
15733
16027
|
fetch: deps.fetch,
|
|
16028
|
+
respectRobots: config.respectRobots,
|
|
15734
16029
|
signal
|
|
15735
16030
|
});
|
|
15736
|
-
const llmsTxt = aiSignals?.
|
|
16031
|
+
const llmsTxt = aiSignals?.signals.llmsTxt;
|
|
15737
16032
|
const healthSitemapUrls = healthStrategy && !config.sitemapUrl && !isCancelled() ? await robotsSession?.sitemapUrls(origin) : void 0;
|
|
15738
|
-
const robotsTxt = aiSignals?.[1];
|
|
15739
|
-
const agentResources = aiSignals?.[2];
|
|
15740
16033
|
cancelled = isCancelled();
|
|
15741
16034
|
const enqueue = (value, depth) => {
|
|
15742
16035
|
const normalized = normalizeUrl$1(value, config.url);
|
|
@@ -15814,7 +16107,7 @@ async function crawlSite(input, dependencies) {
|
|
|
15814
16107
|
url: config.url,
|
|
15815
16108
|
sitemapUrl: config.sitemapUrl,
|
|
15816
16109
|
maxPages: config.maxPages,
|
|
15817
|
-
declaredSitemapUrls: robotsTxt?.sitemapUrls ?? healthSitemapUrls ?? [],
|
|
16110
|
+
declaredSitemapUrls: aiSignals?.signals.robotsTxt?.sitemapUrls ?? healthSitemapUrls ?? [],
|
|
15818
16111
|
warnings,
|
|
15819
16112
|
fetchSitemapUrls: deps.fetchSitemapUrls
|
|
15820
16113
|
});
|
|
@@ -16130,6 +16423,26 @@ async function crawlSite(input, dependencies) {
|
|
|
16130
16423
|
llmsTxtStatus: llmsTxt.status
|
|
16131
16424
|
};
|
|
16132
16425
|
}
|
|
16426
|
+
const startPage = pages.find((page) => page.url === config.url || page.finalUrl === config.url);
|
|
16427
|
+
const ai = aiSignals && !cancelled ? {
|
|
16428
|
+
...aiSignals.signals,
|
|
16429
|
+
markdownCopies: await checkMarkdownCopies({
|
|
16430
|
+
allows: aiSignals.allows,
|
|
16431
|
+
pages: pages.map((page) => ({
|
|
16432
|
+
url: page.url,
|
|
16433
|
+
markdownAlternates: page.markdownAlternates,
|
|
16434
|
+
public: isPublicPage(page)
|
|
16435
|
+
})),
|
|
16436
|
+
timeoutMs: config.timeoutMs,
|
|
16437
|
+
fetch: deps.fetch,
|
|
16438
|
+
signal
|
|
16439
|
+
}),
|
|
16440
|
+
...startPage ? { startPageHeaders: {
|
|
16441
|
+
url: startPage.url,
|
|
16442
|
+
linkRelations: linkRelations(startPage.responseHeaders?.link),
|
|
16443
|
+
...startPage.responseHeaders?.["content-signal"] ? { contentSignal: startPage.responseHeaders["content-signal"] } : {}
|
|
16444
|
+
} } : {}
|
|
16445
|
+
} : aiSignals?.signals;
|
|
16133
16446
|
const report = createCrawlReport({
|
|
16134
16447
|
config,
|
|
16135
16448
|
site,
|
|
@@ -16139,17 +16452,15 @@ async function crawlSite(input, dependencies) {
|
|
|
16139
16452
|
pages,
|
|
16140
16453
|
requests,
|
|
16141
16454
|
requestEvidenceStatus,
|
|
16142
|
-
|
|
16143
|
-
...llmsTxt ? { llmsTxt } : {},
|
|
16144
|
-
...robotsTxt ? { robotsTxt } : {},
|
|
16145
|
-
...agentResources ? { agentResources } : {}
|
|
16146
|
-
} } : {},
|
|
16455
|
+
...ai ? { ai } : {},
|
|
16147
16456
|
...sitemapDiscovery ? { sitemapDiscovery } : {},
|
|
16148
16457
|
...externalLinkVerification ? { externalLinkVerification } : {},
|
|
16149
|
-
...siteCheckResult ? {
|
|
16150
|
-
|
|
16151
|
-
|
|
16152
|
-
|
|
16458
|
+
...siteCheckResult ? { siteChecks: siteCheckResult.checks } : {},
|
|
16459
|
+
additionalIssues: [...siteCheckResult?.issues ?? [], ...ai ? agentReadyIssues({
|
|
16460
|
+
startUrl: config.url,
|
|
16461
|
+
ai,
|
|
16462
|
+
pages
|
|
16463
|
+
}) : []],
|
|
16153
16464
|
status: reportStatus,
|
|
16154
16465
|
warnings,
|
|
16155
16466
|
caveats: crawlCaveats({
|
|
@@ -20613,9 +20924,9 @@ function strikingDistance(input, dependencies = defaultDependencies$5) {
|
|
|
20613
20924
|
//#region packages/core/src/updates/search-status.ts
|
|
20614
20925
|
const INCIDENTS_URL = "https://status.search.google.com/incidents.json";
|
|
20615
20926
|
async function fetchSearchStatusIncidents() {
|
|
20616
|
-
const response = await fetch$1(INCIDENTS_URL);
|
|
20927
|
+
const response = await fetch$1(INCIDENTS_URL, { signal: AbortSignal.timeout(8e3) });
|
|
20617
20928
|
if (!response.ok) throw new SeoError("OPTIONAL_PROVIDER_UNAVAILABLE", `Google Search Status fetch failed with ${response.status}.`);
|
|
20618
|
-
return
|
|
20929
|
+
return readJson(response, "Google Search Status", { maxBytes: 10 * 1024 * 1024 });
|
|
20619
20930
|
}
|
|
20620
20931
|
async function listSearchUpdates(input = {}) {
|
|
20621
20932
|
return searchUpdatesFromIncidents(await fetchSearchStatusIncidents(), input);
|
|
@@ -23900,7 +24211,7 @@ async function fetchCruxFieldData(input) {
|
|
|
23900
24211
|
headers: { "content-type": "application/json" },
|
|
23901
24212
|
signal: controller.signal
|
|
23902
24213
|
});
|
|
23903
|
-
const json = await response
|
|
24214
|
+
const json = await readJson(response, "CrUX").catch(() => ({}));
|
|
23904
24215
|
if (response.status === 404) return {
|
|
23905
24216
|
response,
|
|
23906
24217
|
json
|
|
@@ -25591,7 +25902,7 @@ function workflowReport(input) {
|
|
|
25591
25902
|
const CATEGORY_TO_PRIORITY = {
|
|
25592
25903
|
canonical: "technical",
|
|
25593
25904
|
content: "content",
|
|
25594
|
-
|
|
25905
|
+
agents: "technical",
|
|
25595
25906
|
headings: "content",
|
|
25596
25907
|
images: "content",
|
|
25597
25908
|
indexability: "technical",
|
|
@@ -26609,6 +26920,67 @@ const DIMENSIONS = [
|
|
|
26609
26920
|
"searchAppearance"
|
|
26610
26921
|
];
|
|
26611
26922
|
//#endregion
|
|
26923
|
+
//#region packages/core/src/ga4/matching.ts
|
|
26924
|
+
function hostnameFromUrl(value) {
|
|
26925
|
+
try {
|
|
26926
|
+
const url = new URL(value);
|
|
26927
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
26928
|
+
return url.hostname.toLowerCase().replace(/\.$/, "") || void 0;
|
|
26929
|
+
} catch {
|
|
26930
|
+
return;
|
|
26931
|
+
}
|
|
26932
|
+
}
|
|
26933
|
+
function domainFromProperty(value) {
|
|
26934
|
+
if (!value.startsWith("sc-domain:")) return void 0;
|
|
26935
|
+
const domain = value.slice(10).trim().toLowerCase();
|
|
26936
|
+
if (!domain || /[/:]/.test(domain)) return void 0;
|
|
26937
|
+
return domain.replace(/\.$/, "") || void 0;
|
|
26938
|
+
}
|
|
26939
|
+
function matchForSite(site, streamHostname) {
|
|
26940
|
+
const domain = domainFromProperty(site);
|
|
26941
|
+
if (domain) return streamHostname === domain || streamHostname.endsWith(`.${domain}`) ? "domain" : void 0;
|
|
26942
|
+
return hostnameFromUrl(site) === streamHostname ? "hostname" : void 0;
|
|
26943
|
+
}
|
|
26944
|
+
function matchGa4WebStreams(site, candidates) {
|
|
26945
|
+
return candidates.flatMap((candidate) => {
|
|
26946
|
+
const defaultUri = candidate.stream.webStreamData?.defaultUri;
|
|
26947
|
+
if (!defaultUri) return [];
|
|
26948
|
+
const hostname = hostnameFromUrl(defaultUri);
|
|
26949
|
+
if (!hostname) return [];
|
|
26950
|
+
const match = matchForSite(site, hostname);
|
|
26951
|
+
if (!match) return [];
|
|
26952
|
+
return [{
|
|
26953
|
+
...candidate,
|
|
26954
|
+
defaultUri,
|
|
26955
|
+
hostname,
|
|
26956
|
+
match
|
|
26957
|
+
}];
|
|
26958
|
+
});
|
|
26959
|
+
}
|
|
26960
|
+
function ga4MatchReason(match, site) {
|
|
26961
|
+
const stream = match.stream.displayName ?? match.defaultUri;
|
|
26962
|
+
const location = match.match === "domain" ? "is inside" : "matches";
|
|
26963
|
+
return `Matched Google Analytics web stream ${stream} (${match.defaultUri}): its hostname ${location} ${site}.`;
|
|
26964
|
+
}
|
|
26965
|
+
`${[
|
|
26966
|
+
"chatgpt.com",
|
|
26967
|
+
"chatgpt",
|
|
26968
|
+
"chat.openai.com",
|
|
26969
|
+
"perplexity.ai",
|
|
26970
|
+
"perplexity",
|
|
26971
|
+
"claude.ai",
|
|
26972
|
+
"gemini.google.com",
|
|
26973
|
+
"copilot.microsoft.com",
|
|
26974
|
+
"copilot.com",
|
|
26975
|
+
"chat.deepseek.com",
|
|
26976
|
+
"meta.ai",
|
|
26977
|
+
"grok.com",
|
|
26978
|
+
"chat.mistral.ai",
|
|
26979
|
+
"you.com",
|
|
26980
|
+
"phind.com",
|
|
26981
|
+
"poe.com"
|
|
26982
|
+
].map((source) => source.replaceAll(".", "\\.")).join("|")}`;
|
|
26983
|
+
//#endregion
|
|
26612
26984
|
//#region packages/core/src/commands/clicky.ts
|
|
26613
26985
|
const CLICKY_KEY_PREFIX = "x-clicky-";
|
|
26614
26986
|
//#endregion
|
|
@@ -26728,7 +27100,7 @@ function checkPages(urls) {
|
|
|
26728
27100
|
//#endregion
|
|
26729
27101
|
//#region packages/core/src/commands/report.ts
|
|
26730
27102
|
const SECTIONS = {
|
|
26731
|
-
overview: "Site totals (clicks, impressions, CTR, position) against the previous period, by device, with the top pages and queries and their change.",
|
|
27103
|
+
overview: "Site totals (clicks, impressions, CTR, position) against the previous period, by device, with the top pages and queries and their change, and Google Analytics sessions for the same dates from the property whose web stream matches the site.",
|
|
26732
27104
|
"quick-wins": "Query and page pairs at average positions 4 to 10 with a click-through rate below the site’s own curve for that position.",
|
|
26733
27105
|
"low-ctr": "Query and page pairs at average positions 1 to 4 with a click-through rate below the site’s own curve: titles and snippets to review with the person.",
|
|
26734
27106
|
"striking-distance": "Queries just outside the top positions, with impressions, for pages that already rank.",
|
|
@@ -27037,7 +27409,11 @@ function nodeCommandDeps(options) {
|
|
|
27037
27409
|
pageMarkdown: async (url) => extractMainMarkdown(await fetchPage(url, { refresh: options.refresh })),
|
|
27038
27410
|
call: options.call,
|
|
27039
27411
|
bing: localBing,
|
|
27040
|
-
bingConnect: LOCAL_BING_CONNECT
|
|
27412
|
+
bingConnect: LOCAL_BING_CONNECT,
|
|
27413
|
+
analytics: {
|
|
27414
|
+
accounts: localAccounts,
|
|
27415
|
+
signIn: LOCAL_SIGN_IN.signIn
|
|
27416
|
+
}
|
|
27041
27417
|
};
|
|
27042
27418
|
}
|
|
27043
27419
|
const LOCAL_BING_CONNECT = "Connect Bing on your seoskill.dev account and run `seo login`, or run `seo providers bing connect` with an API key from Bing Webmaster Tools (Settings, API access).";
|
|
@@ -27494,49 +27870,6 @@ function updatePostmortemCsvFiles(report) {
|
|
|
27494
27870
|
];
|
|
27495
27871
|
}
|
|
27496
27872
|
//#endregion
|
|
27497
|
-
//#region packages/core/src/ga4/matching.ts
|
|
27498
|
-
function hostnameFromUrl(value) {
|
|
27499
|
-
try {
|
|
27500
|
-
const url = new URL(value);
|
|
27501
|
-
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
27502
|
-
return url.hostname.toLowerCase().replace(/\.$/, "") || void 0;
|
|
27503
|
-
} catch {
|
|
27504
|
-
return;
|
|
27505
|
-
}
|
|
27506
|
-
}
|
|
27507
|
-
function domainFromProperty(value) {
|
|
27508
|
-
if (!value.startsWith("sc-domain:")) return void 0;
|
|
27509
|
-
const domain = value.slice(10).trim().toLowerCase();
|
|
27510
|
-
if (!domain || /[/:]/.test(domain)) return void 0;
|
|
27511
|
-
return domain.replace(/\.$/, "") || void 0;
|
|
27512
|
-
}
|
|
27513
|
-
function matchForSite(site, streamHostname) {
|
|
27514
|
-
const domain = domainFromProperty(site);
|
|
27515
|
-
if (domain) return streamHostname === domain || streamHostname.endsWith(`.${domain}`) ? "domain" : void 0;
|
|
27516
|
-
return hostnameFromUrl(site) === streamHostname ? "hostname" : void 0;
|
|
27517
|
-
}
|
|
27518
|
-
function matchGa4WebStreams(site, candidates) {
|
|
27519
|
-
return candidates.flatMap((candidate) => {
|
|
27520
|
-
const defaultUri = candidate.stream.webStreamData?.defaultUri;
|
|
27521
|
-
if (!defaultUri) return [];
|
|
27522
|
-
const hostname = hostnameFromUrl(defaultUri);
|
|
27523
|
-
if (!hostname) return [];
|
|
27524
|
-
const match = matchForSite(site, hostname);
|
|
27525
|
-
if (!match) return [];
|
|
27526
|
-
return [{
|
|
27527
|
-
...candidate,
|
|
27528
|
-
defaultUri,
|
|
27529
|
-
hostname,
|
|
27530
|
-
match
|
|
27531
|
-
}];
|
|
27532
|
-
});
|
|
27533
|
-
}
|
|
27534
|
-
function ga4MatchReason(match, site) {
|
|
27535
|
-
const stream = match.stream.displayName ?? match.defaultUri;
|
|
27536
|
-
const location = match.match === "domain" ? "is inside" : "matches";
|
|
27537
|
-
return `Matched Google Analytics web stream ${stream} (${match.defaultUri}): its hostname ${location} ${site}.`;
|
|
27538
|
-
}
|
|
27539
|
-
//#endregion
|
|
27540
27873
|
//#region packages/core/src/gsc/local-export.ts
|
|
27541
27874
|
const DEFAULT_SEARCH_CONSOLE_EXPORT_ROW_LIMIT = 5e3;
|
|
27542
27875
|
const MAX_SEARCH_CONSOLE_EXPORT_ROW_LIMIT = 1e5;
|
|
@@ -29385,6 +29718,6 @@ function isoDate(daysAgo = 0) {
|
|
|
29385
29718
|
return date.toISOString().slice(0, 10);
|
|
29386
29719
|
}
|
|
29387
29720
|
//#endregion
|
|
29388
|
-
export { AHREFS_API_KEY_ENV, AHREFS_API_KEY_SECRET, AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, ANALYTICS_PROVIDER_IDS, AUDIT_PAGE_RULE_IDS, AhrefsClient, BING_API_KEY_ENV, BING_API_KEY_SECRET, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CLICKY_SITEKEY_ENV, CLICKY_SITE_KEYS_SECRET, CLOUD_SIGN_IN, CLOUD_URL, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DATAFORSEO_CREDENTIAL_SECRET, DATAFORSEO_LOGIN_ENV, DATAFORSEO_PASSWORD_ENV, DEFAULT_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, DEFAULT_UNREACHED_PAGE_LIMIT, DataForSeoClient, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, GoogleApiError, INCIDENTS_URL, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LOCAL_BING_CONNECT, LOCAL_SIGN_IN, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MANAGED_PROVIDER_SECRET_NAMES, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, MAX_SEARCH_CONSOLE_EXPORT_FILES, MAX_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, PROVIDER_SECRET_NAMES, ProviderError, ProviderExtensionRegistry, RESEARCH_IMPORT_COLUMN_KEYS, ResponseSizeLimitError, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_PROVIDER_API_VERSION, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, STEP_CHANGE_RULES, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_FAILURE_CONTEXTS, TELEMETRY_FAILURE_REASONS, TELEMETRY_NOTICE, TELEMETRY_OPERATIONS, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, agentActionsView, aiReferralSourceForValue, aiReferralsReport, analyticsConnectionLabel, analyticsConnectionProviderId, analyticsConnectionSchema, analyticsProviderDetails, analyticsProviderSchema, analyzeAiReferralRows, analyzeCannibalImportRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, auditCrawlPages, auditCrawlRequests, auditPage, authStatus, bearerFetch, boundedSitemapInventory, buildAgentFindings, buildQueryClusterReportFromRows, buildTelemetryPayload, cannibalImportReport, cannibalReport, cannibalReportWith, changeWindow, checkDatabaseReadiness, classifyTelemetryFailure, clearCache, cloudAccountFor, cloudBing, cloudCredentials, cloudGoogleEmails, cloudGoogleToken, cloudKeys, cloudLinked, cloudLogout, clusterQueryRows, combineRobotsValues, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, ctrUnderperformersReportWith, decayClusterDrafts, decayComparisonRange, decayingReport, decayingReportWith, deleteAhrefsApiKey, deleteBingApiKey, deleteClickySiteKey, deleteDataForSeoCredentials, deleteManagedProviderSecrets, deleteProviderExtensionAccount, deleteProviderExtensionCredentials, deleteProviderSecret, deleteTokens, deriveBrandTerms, describeInstalledProviderExtension, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensurePrivateDatabaseFiles, ensureSeoCliDirs, estimateSerpTitleWidth, expectedCtrForPosition, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fileSize, finalGscDateRange, findOverlappingSearchUpdates, finishCloudLogin, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, getAuthModeStatus, getCacheStats, getClientConfig, getDb, getPrivacySnapshot, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleJson, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoverageSignals, inferTemplateMovement, initializeTelemetry, inspectProviderPackage, inspectSearchConsoleUrl, inspectUrl, installProviderPackage, installedProviderPackageSchema, installedProviderPackagesSchema, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isSkippableReportError, isTelemetryCi, isTelemetryOperation, isTelemetryReport, isoDate, isoWeek, latestFinalGscDate, linkRecover, linkRecoverRecommendation, listAnalyticsAccounts, listGa4AccountSummaries, listGa4DataStreams, listGoogleAccounts, listInstalledProviderExtensions, listRules, listSearchConsoleSites, listSearchUpdates, listSites, loadInstalledProviderExtensions, loadProviderExtensions, loadProviderPackage, loadSearchConsoleExport, localAccounts, localBing, localCredentialsProvider, localError, localGoogleFetch, localRefreshClicky, localResearchKeys, loginWithLoopback, makeRunId, matchGa4WebStreams, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, nodeCommandDeps, normalizeBrandText, normalizeClickySiteId, normalizeClickySiteKey, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, normalizeProviderExtensionAccount, normalizedResearchColumnName, notSignedIn, noteCacheWrite, openCloudLink, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, pendingCloudLogin, performanceAudit, performanceReportIsCacheable, plural, providerCapabilitySchema, providerConnectionFieldSchema, providerExactVersionSchema, providerExtensionIdSchema, providerIdSchema, providerIntegritySchema, providerKindSchema, providerPackageManifestSchema, providerPackageNameSchema, providerPreferenceSchema, providerRegistrationSchema, providerRequestJson, providerRequestText, pruneLogs, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, quickWinsReportWith, readAhrefsApiKey, readAllTokens, readBingApiKey, readBoundedResponseText, readClickySiteKey, readClickySiteKeys, readConfig, readDataForSeoCredentials, readInstalledProviderPackages, readJson, readJsonOrUndefined, readOauthClient, readProviderExtensionAccount, readProviderExtensionCredentials, readProviderSecret, readTokens, recommendationForRule, reconcileExportPagesWithCrawl, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, registrableDomain, removeInstalledProviderPackage, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderWorkflowMarkdown, reportNarrative, requestHeaders, requirePageVerification, researchImportColumns, resetSeoData, resolveLocalSite, resolveSegmentRanges, resolveSeoCliPaths, reviewObservations, roundedPosition, runAnalyticsProviderLandingPages, runAnalyticsReport, runDoctor, runGa4Report, runProviderExtensionAction, runtimeForProvider, safeProviderAdapterError, saveInstalledProviderPackage, searchAnalyticsRows, searchUpdatesFromIncidents, secondPage, secondPageWith, segmentComparisonRange, segmentImpact, segmentRangeDays, seoErrorEnvelope, sessionSourceInListFilter, setActiveGoogleAccount, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, startCloudLogin, strikingDistance, strikingDistanceWith, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, telemetryErrorCategory, toSeoError, tokenSchema, tokenStoreSchema, topFixes, trackTelemetryCommandFailed, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, trafficAnomalyWith, unavailableSegmentImpactReport, unicodeTokens, uninstallProviderExtension, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateResearchImportColumnSources, verifyAnalyticsProvider, verifyProviderExtension, verifyQueryContent, verifyQueryContentWith, withAgentReportContract, workflowPresentation, writeAhrefsApiKey, writeBingApiKey, writeClickySiteKey, writeConfig, writeDataForSeoCredentials, writeInstalledProviderPackages, writeOauthClient, writeProviderExtensionAccount, writeProviderExtensionCredentials, writeProviderSecret, writeTokens, xRobotsDirectives };
|
|
29721
|
+
export { AHREFS_API_KEY_ENV, AHREFS_API_KEY_SECRET, AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, ANALYTICS_PROVIDER_IDS, AUDIT_PAGE_RULE_IDS, AhrefsClient, BING_API_KEY_ENV, BING_API_KEY_SECRET, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CLICKY_SITEKEY_ENV, CLICKY_SITE_KEYS_SECRET, CLOUD_SIGN_IN, CLOUD_URL, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DATAFORSEO_CREDENTIAL_SECRET, DATAFORSEO_LOGIN_ENV, DATAFORSEO_PASSWORD_ENV, DEFAULT_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, DEFAULT_UNREACHED_PAGE_LIMIT, DataForSeoClient, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, GoogleApiError, INCIDENTS_URL, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LOCAL_BING_CONNECT, LOCAL_SIGN_IN, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MANAGED_PROVIDER_SECRET_NAMES, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, MAX_SEARCH_CONSOLE_EXPORT_FILES, MAX_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, PROVIDER_SECRET_NAMES, ProviderError, ProviderExtensionRegistry, RESEARCH_IMPORT_COLUMN_KEYS, ResponseSizeLimitError, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_PROVIDER_API_VERSION, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, STEP_CHANGE_RULES, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_FAILURE_CONTEXTS, TELEMETRY_FAILURE_REASONS, TELEMETRY_NOTICE, TELEMETRY_OPERATIONS, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, agentActionsView, aiReferralSourceForValue, aiReferralsReport, analyticsConnectionLabel, analyticsConnectionProviderId, analyticsConnectionSchema, analyticsProviderDetails, analyticsProviderSchema, analyzeAiReferralRows, analyzeCannibalImportRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, auditCrawlPages, auditCrawlRequests, auditPage, authStatus, bearerFetch, boundedSitemapInventory, buildAgentFindings, buildQueryClusterReportFromRows, buildTelemetryPayload, cannibalImportReport, cannibalReport, cannibalReportWith, changeWindow, checkDatabaseReadiness, classifyTelemetryFailure, clearCache, cloudAccountFor, cloudBing, cloudCredentials, cloudGoogleEmails, cloudGoogleToken, cloudKeys, cloudLinked, cloudLogout, clusterQueryRows, combineRobotsValues, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, ctrUnderperformersReportWith, decayClusterDrafts, decayComparisonRange, decayingReport, decayingReportWith, deleteAhrefsApiKey, deleteBingApiKey, deleteClickySiteKey, deleteDataForSeoCredentials, deleteManagedProviderSecrets, deleteProviderExtensionAccount, deleteProviderExtensionCredentials, deleteProviderSecret, deleteTokens, deriveBrandTerms, describeInstalledProviderExtension, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensurePrivateDatabaseFiles, ensureSeoCliDirs, estimateSerpTitleWidth, expectedCtrForPosition, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fileSize, finalGscDateRange, findOverlappingSearchUpdates, finishCloudLogin, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, getAuthModeStatus, getCacheStats, getClientConfig, getDb, getPrivacySnapshot, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleJson, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoverageSignals, inferTemplateMovement, initializeTelemetry, inspectProviderPackage, inspectSearchConsoleUrl, inspectUrl, installProviderPackage, installedProviderPackageSchema, installedProviderPackagesSchema, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isSkippableReportError, isTelemetryCi, isTelemetryOperation, isTelemetryReport, isoDate, isoWeek, latestFinalGscDate, linkRecover, linkRecoverRecommendation, listAnalyticsAccounts, listAnalyticsDataStreams, listGa4AccountSummaries, listGa4DataStreams, listGoogleAccounts, listInstalledProviderExtensions, listRules, listSearchConsoleSites, listSearchUpdates, listSites, loadInstalledProviderExtensions, loadProviderExtensions, loadProviderPackage, loadSearchConsoleExport, localAccounts, localBing, localCredentialsProvider, localError, localGoogleFetch, localRefreshClicky, localResearchKeys, loginWithLoopback, makeRunId, matchGa4WebStreams, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, nodeCommandDeps, normalizeBrandText, normalizeClickySiteId, normalizeClickySiteKey, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, normalizeProviderExtensionAccount, normalizedResearchColumnName, notSignedIn, noteCacheWrite, openCloudLink, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, pendingCloudLogin, performanceAudit, performanceReportIsCacheable, plural, providerCapabilitySchema, providerConnectionFieldSchema, providerExactVersionSchema, providerExtensionIdSchema, providerIdSchema, providerIntegritySchema, providerKindSchema, providerPackageManifestSchema, providerPackageNameSchema, providerPreferenceSchema, providerRegistrationSchema, providerRequestJson, providerRequestText, pruneLogs, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, quickWinsReportWith, readAhrefsApiKey, readAllTokens, readBingApiKey, readBoundedResponseText, readClickySiteKey, readClickySiteKeys, readConfig, readDataForSeoCredentials, readInstalledProviderPackages, readJson, readJsonOrUndefined, readOauthClient, readProviderExtensionAccount, readProviderExtensionCredentials, readProviderSecret, readTokens, recommendationForRule, reconcileExportPagesWithCrawl, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, registrableDomain, removeInstalledProviderPackage, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderWorkflowMarkdown, reportNarrative, requestHeaders, requirePageVerification, researchImportColumns, resetSeoData, resolveLocalSite, resolveSegmentRanges, resolveSeoCliPaths, reviewObservations, roundedPosition, runAnalyticsProviderLandingPages, runAnalyticsReport, runDoctor, runGa4Report, runProviderExtensionAction, runtimeForProvider, safeProviderAdapterError, saveInstalledProviderPackage, searchAnalyticsRows, searchUpdatesFromIncidents, secondPage, secondPageWith, segmentComparisonRange, segmentImpact, segmentRangeDays, seoErrorEnvelope, sessionSourceInListFilter, setActiveGoogleAccount, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, startCloudLogin, strikingDistance, strikingDistanceWith, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, telemetryErrorCategory, toSeoError, tokenSchema, tokenStoreSchema, topFixes, trackTelemetryCommandFailed, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, trafficAnomalyWith, unavailableSegmentImpactReport, unicodeTokens, uninstallProviderExtension, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateResearchImportColumnSources, verifyAnalyticsProvider, verifyProviderExtension, verifyQueryContent, verifyQueryContentWith, withAgentReportContract, workflowPresentation, writeAhrefsApiKey, writeBingApiKey, writeClickySiteKey, writeConfig, writeDataForSeoCredentials, writeInstalledProviderPackages, writeOauthClient, writeProviderExtensionAccount, writeProviderExtensionCredentials, writeProviderSecret, writeTokens, xRobotsDirectives };
|
|
29389
29722
|
|
|
29390
29723
|
//# sourceMappingURL=index.js.map
|