seo 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -345
- package/dist/cli.js +84 -84
- package/dist/cli.js.map +1 -1
- package/dist/{dist-DVFMlaqF.js → dist-BN_s3D0r.js} +92 -83
- package/dist/dist-BN_s3D0r.js.map +1 -0
- package/dist/{dist-C21OaeU2.js → dist-sB8HDnCm.js} +1 -1
- package/dist/index.d.ts +941 -220
- package/dist/index.js +620 -295
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +883 -272
- package/dist/mcp.js.map +1 -1
- package/dist/{package-CmplBMg7.js → package-D6HG7niq.js} +2 -2
- package/dist/{package-CmplBMg7.js.map → package-D6HG7niq.js.map} +1 -1
- package/evals/crawl.json +2 -3
- package/evals/internal-links.json +2 -3
- package/evals/low-ctr.json +1 -2
- package/evals/page.json +2 -3
- package/evals/quick-wins.json +2 -3
- package/evals/seo.json +2 -2
- package/package.json +1 -1
- package/skills/README.md +4 -4
- package/skills/seo/SKILL.md +22 -95
- package/dist/dist-DVFMlaqF.js.map +0 -1
package/dist/mcp.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { i as __toESM, n as load, r as __commonJSMin, t as version } from "./package-
|
|
2
|
+
import { i as __toESM, n as load, r as __commonJSMin, t as version } from "./package-D6HG7niq.js";
|
|
3
3
|
import { createHash, randomUUID, sign } from "node:crypto";
|
|
4
4
|
import { ZodOptional, z } from "zod";
|
|
5
5
|
import { arch, homedir, platform, totalmem } from "node:os";
|
|
@@ -13036,6 +13036,98 @@ const RULE_DEFINITIONS = [
|
|
|
13036
13036
|
howToFix: "Add twitter:card and matching metadata only when you need a specific X preview. Keep Open Graph metadata accurate either way.",
|
|
13037
13037
|
impactIfIgnored: "X may use fallback metadata or a simple link preview. Organic rankings are not affected.",
|
|
13038
13038
|
howToVerify: "Re-run the crawl and confirm twitterCard is present if a specific X preview is required."
|
|
13039
|
+
},
|
|
13040
|
+
{
|
|
13041
|
+
id: "ai_search_bot_blocked",
|
|
13042
|
+
title: "robots.txt blocks AI search bots",
|
|
13043
|
+
category: "agents",
|
|
13044
|
+
defaultSeverity: "medium",
|
|
13045
|
+
whyItMatters: "AI search bots fetch pages to answer a person's question and cite the source. A blocked bot cannot read the site, so its answers cannot quote or link to it. Training bots are a separate choice and are listed apart.",
|
|
13046
|
+
howToFix: "Allow the AI search and user bots in robots.txt if the site should appear in AI answers. Block training bots separately if the person wants that.",
|
|
13047
|
+
impactIfIgnored: "ChatGPT, Claude, Perplexity and similar tools can leave the site out of answers and citations.",
|
|
13048
|
+
howToVerify: "Re-run the crawl and confirm the search bots in ai.robotsTxt.botAccess show allowed: true.",
|
|
13049
|
+
review: {
|
|
13050
|
+
question: "Is the site meant to be left out of AI search answers?",
|
|
13051
|
+
changeOnlyIf: "Change robots.txt when the person wants the site to appear in AI answers.",
|
|
13052
|
+
ifNotNeeded: "Record that blocking AI search bots is a deliberate choice for this site."
|
|
13053
|
+
},
|
|
13054
|
+
agentHints: { evidenceFields: ["report.ai.robotsTxt.botAccess"] }
|
|
13055
|
+
},
|
|
13056
|
+
{
|
|
13057
|
+
id: "llms_txt_missing",
|
|
13058
|
+
title: "No llms.txt",
|
|
13059
|
+
category: "agents",
|
|
13060
|
+
defaultSeverity: "low",
|
|
13061
|
+
whyItMatters: "llms.txt gives agents a short summary of the site and links to its main pages, ideally their Markdown copies. Search engines do not use it for ranking.",
|
|
13062
|
+
howToFix: "Publish /llms.txt as plain text with a one-line summary and sections of links to the main pages.",
|
|
13063
|
+
impactIfIgnored: "Agents must find the main pages by crawling, which costs them more requests and tokens.",
|
|
13064
|
+
howToVerify: "Request /llms.txt and confirm a 200 plain text response.",
|
|
13065
|
+
review: {
|
|
13066
|
+
question: "Should agents get a reading list for this site?",
|
|
13067
|
+
changeOnlyIf: "Add it when agents are expected to read or use the site.",
|
|
13068
|
+
ifNotNeeded: "Record that the site has no llms.txt by choice."
|
|
13069
|
+
},
|
|
13070
|
+
agentHints: { evidenceFields: ["report.ai.llmsTxt"] }
|
|
13071
|
+
},
|
|
13072
|
+
{
|
|
13073
|
+
id: "llms_txt_broken_link",
|
|
13074
|
+
title: "llms.txt links to a page that does not load",
|
|
13075
|
+
category: "agents",
|
|
13076
|
+
defaultSeverity: "low",
|
|
13077
|
+
whyItMatters: "An agent that follows a link in llms.txt and gets an error loses its reading list entry.",
|
|
13078
|
+
howToFix: "Update or remove each broken link in llms.txt, or restore the page it points to.",
|
|
13079
|
+
impactIfIgnored: "Agents waste requests and can miss the page.",
|
|
13080
|
+
howToVerify: "Re-run the crawl and confirm ai.llmsTxt.links.broken is empty.",
|
|
13081
|
+
agentHints: { evidenceFields: ["report.ai.llmsTxt.links"] }
|
|
13082
|
+
},
|
|
13083
|
+
{
|
|
13084
|
+
id: "markdown_missing",
|
|
13085
|
+
title: "No Markdown copy of the pages",
|
|
13086
|
+
category: "agents",
|
|
13087
|
+
defaultSeverity: "low",
|
|
13088
|
+
whyItMatters: "A Markdown copy gives agents the page content without layout, scripts or navigation, at a fraction of the tokens. Neither a request with Accept: text/markdown nor a rel=\"alternate\" link returned one. Search engines do not use it for ranking.",
|
|
13089
|
+
howToFix: "Serve a Markdown copy of each page, linked from the page with <link rel=\"alternate\" type=\"text/markdown\">, and return it for Accept: text/markdown with Vary: Accept.",
|
|
13090
|
+
impactIfIgnored: "Agents read the full HTML, which costs more tokens and can include layout text.",
|
|
13091
|
+
howToVerify: "Request a page with Accept: text/markdown and confirm a text/markdown response.",
|
|
13092
|
+
review: {
|
|
13093
|
+
question: "Should agents get a Markdown copy of the pages?",
|
|
13094
|
+
changeOnlyIf: "Add it when agents are expected to read the site.",
|
|
13095
|
+
ifNotNeeded: "Record that the site serves HTML only by choice."
|
|
13096
|
+
},
|
|
13097
|
+
agentHints: { evidenceFields: ["report.ai.markdownNegotiation", "report.ai.markdownCopies"] }
|
|
13098
|
+
},
|
|
13099
|
+
{
|
|
13100
|
+
id: "markdown_alternate_missing",
|
|
13101
|
+
title: "Page has no link to its Markdown copy",
|
|
13102
|
+
category: "agents",
|
|
13103
|
+
defaultSeverity: "low",
|
|
13104
|
+
whyItMatters: "Other pages on the site link to a Markdown copy, but this one does not, so agents cannot find a copy for it.",
|
|
13105
|
+
howToFix: "Add <link rel=\"alternate\" type=\"text/markdown\" href=\"...\"> to the page head, pointing at its Markdown copy.",
|
|
13106
|
+
impactIfIgnored: "Agents read the full HTML for this page.",
|
|
13107
|
+
howToVerify: "Re-run the crawl and confirm the page lists a Markdown alternate.",
|
|
13108
|
+
agentHints: { evidenceFields: ["page.markdownAlternates", "report.ai.markdownCopies"] }
|
|
13109
|
+
},
|
|
13110
|
+
{
|
|
13111
|
+
id: "markdown_alternate_on_private_page",
|
|
13112
|
+
title: "Signed-in page links to a Markdown copy",
|
|
13113
|
+
category: "agents",
|
|
13114
|
+
defaultSeverity: "medium",
|
|
13115
|
+
whyItMatters: "Signed-in pages should have no Markdown copy. This page needs a sign-in, yet it links to one. A copy can show agents content that a signed-out visitor cannot see, or fail and waste their requests.",
|
|
13116
|
+
howToFix: "Remove the rel=\"alternate\" Markdown link, the Link header and any Markdown copy for signed-in routes. Public pages keep theirs, including noindex pages such as privacy and terms.",
|
|
13117
|
+
impactIfIgnored: "Agents can request a copy that fails, or read content that was meant to stay behind a sign-in.",
|
|
13118
|
+
howToVerify: "Re-run the crawl and confirm the page no longer lists a Markdown alternate.",
|
|
13119
|
+
agentHints: { evidenceFields: ["page.markdownAlternates", "report.ai.markdownCopies.broken"] }
|
|
13120
|
+
},
|
|
13121
|
+
{
|
|
13122
|
+
id: "markdown_copy_broken",
|
|
13123
|
+
title: "Markdown copy does not load",
|
|
13124
|
+
category: "agents",
|
|
13125
|
+
defaultSeverity: "medium",
|
|
13126
|
+
whyItMatters: "The page links to a Markdown copy that returns an error or HTML, so an agent that follows the link gets nothing useful. If the page needs a sign-in, remove the link instead.",
|
|
13127
|
+
howToFix: "Make the linked URL return the page as text/markdown, or fix the link.",
|
|
13128
|
+
impactIfIgnored: "Agents get an error or the HTML page instead of the copy.",
|
|
13129
|
+
howToVerify: "Request the Markdown URL and confirm a 200 text/markdown response.",
|
|
13130
|
+
agentHints: { evidenceFields: ["report.ai.markdownCopies.broken"] }
|
|
13039
13131
|
}
|
|
13040
13132
|
];
|
|
13041
13133
|
const RULE_RECOMMENDATIONS = {
|
|
@@ -13064,7 +13156,10 @@ const RULE_RECOMMENDATIONS = {
|
|
|
13064
13156
|
og_title_missing: "review",
|
|
13065
13157
|
og_description_missing: "review",
|
|
13066
13158
|
og_image_missing: "review",
|
|
13067
|
-
twitter_card_missing: "review"
|
|
13159
|
+
twitter_card_missing: "review",
|
|
13160
|
+
ai_search_bot_blocked: "review",
|
|
13161
|
+
llms_txt_missing: "review",
|
|
13162
|
+
markdown_missing: "review"
|
|
13068
13163
|
};
|
|
13069
13164
|
const RULES_BY_ID = new Map(RULE_DEFINITIONS.map((rule) => [rule.id, {
|
|
13070
13165
|
...rule,
|
|
@@ -13762,6 +13857,14 @@ async function listAnalyticsAccounts(gfetch, options) {
|
|
|
13762
13857
|
});
|
|
13763
13858
|
});
|
|
13764
13859
|
}
|
|
13860
|
+
async function listAnalyticsDataStreams(gfetch, propertyId, options) {
|
|
13861
|
+
return (await googleJson(gfetch, `https://analyticsadmin.googleapis.com/v1beta/properties/${propertyId}/dataStreams?pageSize=200`, {
|
|
13862
|
+
...options,
|
|
13863
|
+
label: "Google Analytics data streams",
|
|
13864
|
+
subject: `property ${propertyId}`,
|
|
13865
|
+
maxBytes: 1024 * 1024
|
|
13866
|
+
})).dataStreams ?? [];
|
|
13867
|
+
}
|
|
13765
13868
|
async function runAnalyticsReport(gfetch, propertyId, body, options) {
|
|
13766
13869
|
if (!/^\d+$/.test(propertyId)) throw new SeoError("INVALID_INPUT", "The Google Analytics property must be its numeric id.");
|
|
13767
13870
|
return googleJson(gfetch, `https://analyticsdata.googleapis.com/v1beta/properties/${propertyId}:runReport`, {
|
|
@@ -14430,7 +14533,7 @@ async function readableSites(google, signIn) {
|
|
|
14430
14533
|
permissionLevel: entry.permissionLevel ?? ""
|
|
14431
14534
|
})).sort((a, b) => a.siteUrl.localeCompare(b.siteUrl));
|
|
14432
14535
|
}
|
|
14433
|
-
const hostOf$
|
|
14536
|
+
const hostOf$2 = (value) => {
|
|
14434
14537
|
try {
|
|
14435
14538
|
return new URL(value.includes("://") ? value : `https://${value.replace(/^sc-domain:/, "")}`).hostname.replace(/^www\./, "");
|
|
14436
14539
|
} catch {
|
|
@@ -14448,14 +14551,14 @@ async function runSites(input) {
|
|
|
14448
14551
|
};
|
|
14449
14552
|
})), input.bing ? listBingSites(input.bing).catch((error) => error instanceof Error ? error : new Error(String(error))) : Promise.resolve(null)]);
|
|
14450
14553
|
const bingSites = Array.isArray(bing) ? bing.filter((site) => site.verified) : [];
|
|
14451
|
-
const bingByHost = new Map(bingSites.map((site) => [hostOf$
|
|
14554
|
+
const bingByHost = new Map(bingSites.map((site) => [hostOf$2(site.url), site]));
|
|
14452
14555
|
const properties = google.flatMap((account) => account.properties instanceof Error ? [] : account.properties.map((property) => ({
|
|
14453
14556
|
...property,
|
|
14454
14557
|
...named ? { account: account.email ?? "" } : {},
|
|
14455
|
-
...Array.isArray(bing) ? { bing: bingByHost.get(hostOf$
|
|
14558
|
+
...Array.isArray(bing) ? { bing: bingByHost.get(hostOf$2(property.siteUrl))?.url ?? "" } : {}
|
|
14456
14559
|
})));
|
|
14457
|
-
const googleHosts = new Set(properties.map((property) => hostOf$
|
|
14458
|
-
const bingOnly = bingSites.filter((site) => !googleHosts.has(hostOf$
|
|
14560
|
+
const googleHosts = new Set(properties.map((property) => hostOf$2(property.siteUrl)));
|
|
14561
|
+
const bingOnly = bingSites.filter((site) => !googleHosts.has(hostOf$2(site.url)));
|
|
14459
14562
|
const failed = google.filter((account) => account.properties instanceof Error);
|
|
14460
14563
|
const analytics = google.flatMap((account) => (account.analytics ?? []).flatMap((summary) => summary.propertySummaries.map((property) => ({
|
|
14461
14564
|
property: property.property.replace("properties/", ""),
|
|
@@ -14767,7 +14870,7 @@ async function cloudCredentials(options = {}) {
|
|
|
14767
14870
|
return credentials;
|
|
14768
14871
|
}
|
|
14769
14872
|
function notSignedIn() {
|
|
14770
|
-
return "Not signed in. Run `seo login` to use your seoskill.dev account on this machine (it prints a link to approve), or `seo auth login` to sign in with Google on this machine only.";
|
|
14873
|
+
return "Not signed in. Run `seo login` to use your seoskill.dev account on this machine (it prints a link to approve), or `seo auth login` to sign in with Google on this machine only. `seo crawl --url <url>` works without sign-in.";
|
|
14771
14874
|
}
|
|
14772
14875
|
function tokenOf(item, what) {
|
|
14773
14876
|
if ("error" in item) throw new SeoError("AUTH_EXPIRED", `${what}: ${item.error}`);
|
|
@@ -21662,7 +21765,7 @@ function clientRenderedContentEvidence(page) {
|
|
|
21662
21765
|
}
|
|
21663
21766
|
};
|
|
21664
21767
|
}
|
|
21665
|
-
function issue(ruleId, page, detail, evidence) {
|
|
21768
|
+
function issue$1(ruleId, page, detail, evidence) {
|
|
21666
21769
|
const rule = explainRule(ruleId);
|
|
21667
21770
|
if (!rule) throw new Error(`Missing rule guidance for ${ruleId}`);
|
|
21668
21771
|
return {
|
|
@@ -21827,7 +21930,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
21827
21930
|
}
|
|
21828
21931
|
for (const page of pages) {
|
|
21829
21932
|
if (page.status === 0) {
|
|
21830
|
-
issues.push(issue("connection_error", page, page.error, {
|
|
21933
|
+
issues.push(issue$1("connection_error", page, page.error, {
|
|
21831
21934
|
status: page.status,
|
|
21832
21935
|
error: page.error
|
|
21833
21936
|
}));
|
|
@@ -21835,23 +21938,23 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
21835
21938
|
}
|
|
21836
21939
|
if (page.accessBlock) continue;
|
|
21837
21940
|
if (page.status >= 500) {
|
|
21838
|
-
issues.push(issue("server_error", page, String(page.status), { status: page.status }));
|
|
21839
|
-
if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue("broken_internal_link", page, String(page.status), {
|
|
21941
|
+
issues.push(issue$1("server_error", page, String(page.status), { status: page.status }));
|
|
21942
|
+
if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
21840
21943
|
status: page.status,
|
|
21841
21944
|
internalInlinkCount: page.internalInlinkCount
|
|
21842
21945
|
}));
|
|
21843
21946
|
continue;
|
|
21844
21947
|
}
|
|
21845
21948
|
if (page.status >= 400) {
|
|
21846
|
-
issues.push(issue("client_error", page, String(page.status), { status: page.status }));
|
|
21847
|
-
if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0) issues.push(issue("broken_internal_link", page, String(page.status), {
|
|
21949
|
+
issues.push(issue$1("client_error", page, String(page.status), { status: page.status }));
|
|
21950
|
+
if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
21848
21951
|
status: page.status,
|
|
21849
21952
|
internalInlinkCount: page.internalInlinkCount
|
|
21850
21953
|
}));
|
|
21851
21954
|
continue;
|
|
21852
21955
|
}
|
|
21853
21956
|
if (page.status >= 300) {
|
|
21854
|
-
issues.push(issue("redirected_url", page, `Status ${page.status}`, {
|
|
21957
|
+
issues.push(issue$1("redirected_url", page, `Status ${page.status}`, {
|
|
21855
21958
|
requestedUrl: page.url,
|
|
21856
21959
|
finalUrl: page.finalUrl,
|
|
21857
21960
|
status: page.status
|
|
@@ -21862,7 +21965,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
21862
21965
|
const redirected = isRedirectedPage(page);
|
|
21863
21966
|
if (redirected) {
|
|
21864
21967
|
const redirectTarget = page.metaRefresh?.targetUrl ?? page.finalUrl;
|
|
21865
|
-
issues.push(issue("redirected_url", page, `Final URL: ${redirectTarget}`, {
|
|
21968
|
+
issues.push(issue$1("redirected_url", page, `Final URL: ${redirectTarget}`, {
|
|
21866
21969
|
requestedUrl: page.url,
|
|
21867
21970
|
finalUrl: redirectTarget,
|
|
21868
21971
|
status: page.status,
|
|
@@ -21873,46 +21976,46 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
21873
21976
|
}));
|
|
21874
21977
|
}
|
|
21875
21978
|
const redirectChain = page.fetchDiagnostics?.redirectChain ?? [];
|
|
21876
|
-
if (redirectChain.length > 1) issues.push(issue("redirect_chain", page, `${redirectChain.length} hops`, {
|
|
21979
|
+
if (redirectChain.length > 1) issues.push(issue$1("redirect_chain", page, `${redirectChain.length} hops`, {
|
|
21877
21980
|
redirectChain,
|
|
21878
21981
|
hops: redirectChain.length
|
|
21879
21982
|
}));
|
|
21880
|
-
if ((page.responseTimeMs ?? 0) > SLOW_RESPONSE_MS) issues.push(issue("slow_response", page, `${page.responseTimeMs}ms`, {
|
|
21983
|
+
if ((page.responseTimeMs ?? 0) > SLOW_RESPONSE_MS) issues.push(issue$1("slow_response", page, `${page.responseTimeMs}ms`, {
|
|
21881
21984
|
responseTimeMs: page.responseTimeMs,
|
|
21882
21985
|
thresholdMs: SLOW_RESPONSE_MS
|
|
21883
21986
|
}));
|
|
21884
21987
|
if (page.auditScope === "status") continue;
|
|
21885
21988
|
if (redirected) continue;
|
|
21886
|
-
if (page.softAuthenticationGate) issues.push(issue("soft_authentication_gate", page, "A password form and login-specific page signals were observed.", page.softAuthenticationGate));
|
|
21989
|
+
if (page.softAuthenticationGate) issues.push(issue$1("soft_authentication_gate", page, "A password form and login-specific page signals were observed.", page.softAuthenticationGate));
|
|
21887
21990
|
const renderedContent = clientRenderedContentEvidence(page);
|
|
21888
|
-
if (renderedContent) issues.push(issue("client_rendered_content", page, "The raw response was an empty or near-empty shell.", renderedContent));
|
|
21889
|
-
if (!renderedContent && isAuditableHtmlPage(page) && page.indexable && page.extractionStatus === "complete" && page.wordCount < NEAR_EMPTY_CONTENT_WORDS) issues.push(issue("near_empty_content", page, `${page.wordCount} extracted words`, {
|
|
21991
|
+
if (renderedContent) issues.push(issue$1("client_rendered_content", page, "The raw response was an empty or near-empty shell.", renderedContent));
|
|
21992
|
+
if (!renderedContent && isAuditableHtmlPage(page) && page.indexable && page.extractionStatus === "complete" && page.wordCount < NEAR_EMPTY_CONTENT_WORDS) issues.push(issue$1("near_empty_content", page, `${page.wordCount} extracted words`, {
|
|
21890
21993
|
wordCount: page.wordCount,
|
|
21891
21994
|
reviewBelowWords: NEAR_EMPTY_CONTENT_WORDS,
|
|
21892
21995
|
contentSample: page.contentSample,
|
|
21893
21996
|
outgoingInternalCount: page.outgoingInternalCount,
|
|
21894
21997
|
rendering: page.fetchDiagnostics?.rendering
|
|
21895
21998
|
}));
|
|
21896
|
-
if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(issue("large_html", page, `${Math.round((page.sizeBytes ?? 0) / 1024)} KB`, {
|
|
21999
|
+
if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(issue$1("large_html", page, `${Math.round((page.sizeBytes ?? 0) / 1024)} KB`, {
|
|
21897
22000
|
sizeBytes: page.sizeBytes,
|
|
21898
22001
|
thresholdBytes: LARGE_HTML_BYTES
|
|
21899
22002
|
}));
|
|
21900
|
-
if (isHtmlPage(page) && !page.compression && (page.sizeBytes ?? 0) > COMPRESSION_MIN_BYTES) issues.push(issue("no_compression", page, void 0, {
|
|
22003
|
+
if (isHtmlPage(page) && !page.compression && (page.sizeBytes ?? 0) > COMPRESSION_MIN_BYTES) issues.push(issue$1("no_compression", page, void 0, {
|
|
21901
22004
|
sizeBytes: page.sizeBytes,
|
|
21902
22005
|
thresholdBytes: COMPRESSION_MIN_BYTES,
|
|
21903
22006
|
compression: page.compression
|
|
21904
22007
|
}));
|
|
21905
|
-
if (page.isHttps === false && !isLoopbackUrl(page.finalUrl || page.url)) issues.push(issue("http_not_secure", page, void 0, { finalUrl: page.finalUrl }));
|
|
22008
|
+
if (page.isHttps === false && !isLoopbackUrl(page.finalUrl || page.url)) issues.push(issue$1("http_not_secure", page, void 0, { finalUrl: page.finalUrl }));
|
|
21906
22009
|
else if (page.isHttps === true) {
|
|
21907
|
-
if ((page.mixedContentCount ?? 0) > 0) issues.push(issue("mixed_content", page, `${page.mixedContentCount} insecure resources`, {
|
|
22010
|
+
if ((page.mixedContentCount ?? 0) > 0) issues.push(issue$1("mixed_content", page, `${page.mixedContentCount} insecure resources`, {
|
|
21908
22011
|
mixedContentCount: page.mixedContentCount,
|
|
21909
22012
|
mixedContentSamples: page.mixedContentSamples
|
|
21910
22013
|
}));
|
|
21911
|
-
if (page.hasHsts === false) issues.push(issue("hsts_missing", page, void 0, { header: "strict-transport-security" }));
|
|
22014
|
+
if (page.hasHsts === false) issues.push(issue$1("hsts_missing", page, void 0, { header: "strict-transport-security" }));
|
|
21912
22015
|
}
|
|
21913
22016
|
for (const link of page.externalLinkChecks ?? []) {
|
|
21914
22017
|
if (link.state !== "confirmed-broken") continue;
|
|
21915
|
-
issues.push(issue("broken_external_link", page, link.url, {
|
|
22018
|
+
issues.push(issue$1("broken_external_link", page, link.url, {
|
|
21916
22019
|
url: link.url,
|
|
21917
22020
|
status: link.status,
|
|
21918
22021
|
error: link.error,
|
|
@@ -21920,64 +22023,64 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
21920
22023
|
attempts: link.attempts
|
|
21921
22024
|
}));
|
|
21922
22025
|
}
|
|
21923
|
-
if (page.indexable && (page.internalInlinkCount ?? 0) === 0 && !sameUrl(page.url, opts.startUrl)) issues.push(issue("orphan_page", page, "No internal inlinks", { internalInlinkCount: page.internalInlinkCount ?? 0 }));
|
|
21924
|
-
if ((page.crawlDepth ?? 0) > DEEP_PAGE_DEPTH) issues.push(issue("deep_page", page, `Depth ${page.crawlDepth}`, {
|
|
22026
|
+
if (page.indexable && (page.internalInlinkCount ?? 0) === 0 && !sameUrl(page.url, opts.startUrl)) issues.push(issue$1("orphan_page", page, "No internal inlinks", { internalInlinkCount: page.internalInlinkCount ?? 0 }));
|
|
22027
|
+
if ((page.crawlDepth ?? 0) > DEEP_PAGE_DEPTH) issues.push(issue$1("deep_page", page, `Depth ${page.crawlDepth}`, {
|
|
21925
22028
|
crawlDepth: page.crawlDepth,
|
|
21926
22029
|
threshold: DEEP_PAGE_DEPTH
|
|
21927
22030
|
}));
|
|
21928
|
-
if (page.indexable && isValuablePage(page) && (page.internalInlinkCount ?? 0) <= WEAK_VALUABLE_INLINKS) issues.push(issue("weak_internal_links_to_valuable_page", page, `${page.internalInlinkCount ?? 0} internal inlinks`, {
|
|
22031
|
+
if (page.indexable && isValuablePage(page) && (page.internalInlinkCount ?? 0) <= WEAK_VALUABLE_INLINKS) issues.push(issue$1("weak_internal_links_to_valuable_page", page, `${page.internalInlinkCount ?? 0} internal inlinks`, {
|
|
21929
22032
|
internalInlinkCount: page.internalInlinkCount ?? 0,
|
|
21930
22033
|
searchMetrics: page.searchMetrics,
|
|
21931
22034
|
analytics: page.analytics
|
|
21932
22035
|
}));
|
|
21933
|
-
if (page.robotsTxt?.allowed === false) issues.push(issue("robots_blocked", page, page.robotsTxt.matchedLine, { robotsTxt: page.robotsTxt }));
|
|
21934
|
-
if (hasXRobotsDirective(page.xRobotsTag, "noindex")) issues.push(issue("x_robots_noindex", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
|
|
22036
|
+
if (page.robotsTxt?.allowed === false) issues.push(issue$1("robots_blocked", page, page.robotsTxt.matchedLine, { robotsTxt: page.robotsTxt }));
|
|
22037
|
+
if (hasXRobotsDirective(page.xRobotsTag, "noindex")) issues.push(issue$1("x_robots_noindex", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
|
|
21935
22038
|
const xRobotsNofollow = hasXRobotsDirective(page.xRobotsTag, "nofollow");
|
|
21936
|
-
if (xRobotsNofollow) issues.push(issue("nofollow", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
|
|
22039
|
+
if (xRobotsNofollow) issues.push(issue$1("nofollow", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
|
|
21937
22040
|
if (!isAuditableHtmlPage(page)) continue;
|
|
21938
22041
|
const title = page.title?.trim();
|
|
21939
22042
|
const titleWidth = title ? estimateSerpTitleWidth(title) : void 0;
|
|
21940
22043
|
const duplicateTitle = duplicateTitles.get(normalizedMetadata(title) ?? "");
|
|
21941
|
-
if (!title) issues.push(issue("missing_title", page));
|
|
21942
|
-
else if (titleWidth?.status === "over-reference") issues.push(issue("title_too_wide", page, `${titleWidth.estimatedPixels}px`, {
|
|
22044
|
+
if (!title) issues.push(issue$1("missing_title", page));
|
|
22045
|
+
else if (titleWidth?.status === "over-reference") issues.push(issue$1("title_too_wide", page, `${titleWidth.estimatedPixels}px`, {
|
|
21943
22046
|
title,
|
|
21944
22047
|
...titleWidth
|
|
21945
22048
|
}));
|
|
21946
|
-
if (title && duplicateTitle) issues.push(issue("title_duplicate", page, duplicateTitle.value, {
|
|
22049
|
+
if (title && duplicateTitle) issues.push(issue$1("title_duplicate", page, duplicateTitle.value, {
|
|
21947
22050
|
title,
|
|
21948
22051
|
duplicateCount: duplicateTitle.count,
|
|
21949
22052
|
sampleUrls: duplicateTitle.sampleUrls
|
|
21950
22053
|
}));
|
|
21951
22054
|
const metaDescription = page.metaDescription?.trim();
|
|
21952
22055
|
const duplicateDescription = duplicateDescriptions.get(normalizedMetadata(metaDescription) ?? "");
|
|
21953
|
-
if (!metaDescription) issues.push(issue("missing_meta_description", page));
|
|
21954
|
-
if (metaDescription && duplicateDescription) issues.push(issue("meta_description_duplicate", page, duplicateDescription.value, {
|
|
22056
|
+
if (!metaDescription) issues.push(issue$1("missing_meta_description", page));
|
|
22057
|
+
if (metaDescription && duplicateDescription) issues.push(issue$1("meta_description_duplicate", page, duplicateDescription.value, {
|
|
21955
22058
|
metaDescription,
|
|
21956
22059
|
duplicateCount: duplicateDescription.count,
|
|
21957
22060
|
sampleUrls: duplicateDescription.sampleUrls
|
|
21958
22061
|
}));
|
|
21959
22062
|
const h1Count = page.h1Count ?? (page.h1 ? 1 : 0);
|
|
21960
|
-
if (h1Count === 0) issues.push(issue("h1_missing", page, "No H1 found", { h1Count }));
|
|
21961
|
-
else if (h1Count > 1) issues.push(issue("h1_multiple", page, `${h1Count} H1 headings`, { h1Count }));
|
|
21962
|
-
if (page.canonicalStatus === "conflicting") issues.push(issue("canonical_conflict", page, void 0, { candidates: page.canonicalCandidates }));
|
|
21963
|
-
else if (page.canonicalStatus === "duplicate") issues.push(issue("canonical_multiple", page, page.canonical, {
|
|
22063
|
+
if (h1Count === 0) issues.push(issue$1("h1_missing", page, "No H1 found", { h1Count }));
|
|
22064
|
+
else if (h1Count > 1) issues.push(issue$1("h1_multiple", page, `${h1Count} H1 headings`, { h1Count }));
|
|
22065
|
+
if (page.canonicalStatus === "conflicting") issues.push(issue$1("canonical_conflict", page, void 0, { candidates: page.canonicalCandidates }));
|
|
22066
|
+
else if (page.canonicalStatus === "duplicate") issues.push(issue$1("canonical_multiple", page, page.canonical, {
|
|
21964
22067
|
canonical: page.canonical,
|
|
21965
22068
|
candidates: page.canonicalCandidates
|
|
21966
22069
|
}));
|
|
21967
|
-
else if (page.canonicalStatus === "outside-head-only") issues.push(issue("canonical_outside_head", page, page.canonicalRaw, { candidates: page.canonicalCandidates }));
|
|
21968
|
-
else if (page.canonicalStatus === "streamed-outside-head") issues.push(issue("canonical_streamed_in_body", page, page.canonical, {
|
|
22070
|
+
else if (page.canonicalStatus === "outside-head-only") issues.push(issue$1("canonical_outside_head", page, page.canonicalRaw, { candidates: page.canonicalCandidates }));
|
|
22071
|
+
else if (page.canonicalStatus === "streamed-outside-head") issues.push(issue$1("canonical_streamed_in_body", page, page.canonical, {
|
|
21969
22072
|
canonical: page.canonical,
|
|
21970
22073
|
candidates: page.canonicalCandidates
|
|
21971
22074
|
}));
|
|
21972
22075
|
if (!page.canonical) {
|
|
21973
|
-
if (!["conflicting", "outside-head-only"].includes(page.canonicalStatus ?? "")) issues.push(page.canonicalRaw ? issue("canonical_invalid", page, page.canonicalRaw, { canonicalRaw: page.canonicalRaw }) : issue("canonical_missing", page));
|
|
22076
|
+
if (!["conflicting", "outside-head-only"].includes(page.canonicalStatus ?? "")) issues.push(page.canonicalRaw ? issue$1("canonical_invalid", page, page.canonicalRaw, { canonicalRaw: page.canonicalRaw }) : issue$1("canonical_missing", page));
|
|
21974
22077
|
} else {
|
|
21975
|
-
if (page.canonicalRaw && !absoluteHttpUrl(page.canonicalRaw)) issues.push(issue("canonical_non_absolute", page, page.canonicalRaw, {
|
|
22078
|
+
if (page.canonicalRaw && !absoluteHttpUrl(page.canonicalRaw)) issues.push(issue$1("canonical_non_absolute", page, page.canonicalRaw, {
|
|
21976
22079
|
canonicalRaw: page.canonicalRaw,
|
|
21977
22080
|
canonical: page.canonical
|
|
21978
22081
|
}));
|
|
21979
22082
|
const canonicalTarget = pageByUrl.get(urlKey(page.canonical) ?? "");
|
|
21980
|
-
if (canonicalTarget?.canonical && !sameUrl(canonicalTarget.canonical, page.canonical)) issues.push(issue("canonical_chain", page, page.canonical, {
|
|
22083
|
+
if (canonicalTarget?.canonical && !sameUrl(canonicalTarget.canonical, page.canonical)) issues.push(issue$1("canonical_chain", page, page.canonical, {
|
|
21981
22084
|
canonical: page.canonical,
|
|
21982
22085
|
nextCanonical: canonicalTarget.canonical,
|
|
21983
22086
|
chain: [
|
|
@@ -21987,25 +22090,25 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
21987
22090
|
]
|
|
21988
22091
|
}));
|
|
21989
22092
|
}
|
|
21990
|
-
if (hasMetaRobotsDirective(page.metaRobots, "noindex")) issues.push(issue("noindex", page, page.indexability, { metaRobots: page.metaRobots }));
|
|
21991
|
-
if (hasMetaRobotsDirective(page.metaRobots, "nofollow") && !xRobotsNofollow) issues.push(issue("nofollow", page, page.metaRobots, { metaRobots: page.metaRobots }));
|
|
21992
|
-
if (page.canonical && !sameUrl(page.canonical, page.finalUrl) && (page.declaredIndexability === "canonical-hint-other" || page.declaredIndexability === void 0 && /canonical/i.test(page.indexability ?? ""))) issues.push(issue("canonicalized_page", page, page.canonical, {
|
|
22093
|
+
if (hasMetaRobotsDirective(page.metaRobots, "noindex")) issues.push(issue$1("noindex", page, page.indexability, { metaRobots: page.metaRobots }));
|
|
22094
|
+
if (hasMetaRobotsDirective(page.metaRobots, "nofollow") && !xRobotsNofollow) issues.push(issue$1("nofollow", page, page.metaRobots, { metaRobots: page.metaRobots }));
|
|
22095
|
+
if (page.canonical && !sameUrl(page.canonical, page.finalUrl) && (page.declaredIndexability === "canonical-hint-other" || page.declaredIndexability === void 0 && /canonical/i.test(page.indexability ?? ""))) issues.push(issue$1("canonicalized_page", page, page.canonical, {
|
|
21993
22096
|
canonical: page.canonical,
|
|
21994
22097
|
finalUrl: page.finalUrl,
|
|
21995
22098
|
indexability: page.indexability
|
|
21996
22099
|
}));
|
|
21997
|
-
else if (page.canonical && !sameUrl(page.canonical, page.finalUrl)) issues.push(issue("canonical_mismatch", page, page.canonical, {
|
|
22100
|
+
else if (page.canonical && !sameUrl(page.canonical, page.finalUrl)) issues.push(issue$1("canonical_mismatch", page, page.canonical, {
|
|
21998
22101
|
canonical: page.canonical,
|
|
21999
22102
|
finalUrl: page.finalUrl
|
|
22000
22103
|
}));
|
|
22001
22104
|
const contentDuplicate = page.mainContentHash ? duplicateContent.get(page.mainContentHash) : void 0;
|
|
22002
|
-
if (contentDuplicate) issues.push(issue("duplicate_content", page, "Duplicate main content", {
|
|
22105
|
+
if (contentDuplicate) issues.push(issue$1("duplicate_content", page, "Duplicate main content", {
|
|
22003
22106
|
mainContentHash: page.mainContentHash,
|
|
22004
22107
|
duplicateCount: contentDuplicate.count,
|
|
22005
22108
|
sampleUrls: contentDuplicate.sampleUrls
|
|
22006
22109
|
}));
|
|
22007
22110
|
const coverage = queryCoverage(page);
|
|
22008
|
-
if (coverage && coverage.coverage < QUERY_COVERAGE_MIN) issues.push(issue("query_coverage_missing", page, coverage.query, {
|
|
22111
|
+
if (coverage && coverage.coverage < QUERY_COVERAGE_MIN) issues.push(issue$1("query_coverage_missing", page, coverage.query, {
|
|
22009
22112
|
query: coverage.query,
|
|
22010
22113
|
impressions: page.topQuery?.impressions,
|
|
22011
22114
|
matchedTerms: coverage.matchedTerms,
|
|
@@ -22013,36 +22116,36 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
22013
22116
|
coverage: coverage.coverage,
|
|
22014
22117
|
threshold: QUERY_COVERAGE_MIN
|
|
22015
22118
|
}));
|
|
22016
|
-
if ((page.imagesMissingAlt ?? 0) > 0) issues.push(issue("image_missing_alt", page, `${page.imagesMissingAlt} of ${page.imagesTotal ?? 0} images`, {
|
|
22119
|
+
if ((page.imagesMissingAlt ?? 0) > 0) issues.push(issue$1("image_missing_alt", page, `${page.imagesMissingAlt} of ${page.imagesTotal ?? 0} images`, {
|
|
22017
22120
|
imagesTotal: page.imagesTotal,
|
|
22018
22121
|
imagesMissingAlt: page.imagesMissingAlt
|
|
22019
22122
|
}));
|
|
22020
|
-
if (page.oversizedImageCandidates?.length) issues.push(issue("image_oversized_candidate", page, `${page.oversizedImageCandidates.length} image candidate${page.oversizedImageCandidates.length === 1 ? "" : "s"}`, {
|
|
22123
|
+
if (page.oversizedImageCandidates?.length) issues.push(issue$1("image_oversized_candidate", page, `${page.oversizedImageCandidates.length} image candidate${page.oversizedImageCandidates.length === 1 ? "" : "s"}`, {
|
|
22021
22124
|
thresholdPx: OVERSIZED_IMAGE_CANDIDATE_LIMIT,
|
|
22022
22125
|
candidates: page.oversizedImageCandidates.slice(0, 10)
|
|
22023
22126
|
}));
|
|
22024
|
-
if (!page.hasViewport) issues.push(issue("viewport_missing", page));
|
|
22025
|
-
if (!page.lang) issues.push(issue("lang_missing", page));
|
|
22127
|
+
if (!page.hasViewport) issues.push(issue$1("viewport_missing", page));
|
|
22128
|
+
if (!page.lang) issues.push(issue$1("lang_missing", page));
|
|
22026
22129
|
const hreflang = page.hreflang ?? [];
|
|
22027
22130
|
if (hreflang.length) {
|
|
22028
22131
|
const malformed = hreflang.filter((item) => !hasRecognizableHreflangFormat(item.hreflang));
|
|
22029
|
-
if (malformed.length) issues.push(issue("hreflang_invalid", page, `${malformed.length} malformed`, { malformed }));
|
|
22132
|
+
if (malformed.length) issues.push(issue$1("hreflang_invalid", page, `${malformed.length} malformed`, { malformed }));
|
|
22030
22133
|
const duplicateCodes = [...new Set(hreflang.map((item) => item.hreflang.trim().toLowerCase()).filter((code, index, values) => values.indexOf(code) !== index))];
|
|
22031
|
-
if (duplicateCodes.length) issues.push(issue("hreflang_duplicate", page, duplicateCodes.join(", "), { duplicateCodes }));
|
|
22032
|
-
if (!hreflang.some((item) => sameUrl(item.href, page.finalUrl))) issues.push(issue("hreflang_incomplete", page, page.lang, {
|
|
22134
|
+
if (duplicateCodes.length) issues.push(issue$1("hreflang_duplicate", page, duplicateCodes.join(", "), { duplicateCodes }));
|
|
22135
|
+
if (!hreflang.some((item) => sameUrl(item.href, page.finalUrl))) issues.push(issue$1("hreflang_incomplete", page, page.lang, {
|
|
22033
22136
|
lang: page.lang,
|
|
22034
22137
|
finalUrl: page.finalUrl,
|
|
22035
22138
|
hreflang
|
|
22036
22139
|
}));
|
|
22037
22140
|
}
|
|
22038
22141
|
const structuredDataFormats = page.structuredDataFormats;
|
|
22039
|
-
if (isAuditableHtmlPage(page) && page.indexable && structuredDataFormats && structuredDataFormats.length === 0 && (page.schemaTypes?.length ?? 0) === 0 && (page.invalidJsonLdCount ?? 0) === 0 && (page.unrecognizedJsonLdTypes?.length ?? 0) === 0) issues.push(issue("structured_data_missing", page, "No structured data detected", {
|
|
22142
|
+
if (isAuditableHtmlPage(page) && page.indexable && structuredDataFormats && structuredDataFormats.length === 0 && (page.schemaTypes?.length ?? 0) === 0 && (page.invalidJsonLdCount ?? 0) === 0 && (page.unrecognizedJsonLdTypes?.length ?? 0) === 0) issues.push(issue$1("structured_data_missing", page, "No structured data detected", {
|
|
22040
22143
|
structuredDataFormats,
|
|
22041
22144
|
schemaTypes: page.schemaTypes ?? [],
|
|
22042
22145
|
invalidJsonLdCount: page.invalidJsonLdCount ?? 0,
|
|
22043
22146
|
unrecognizedJsonLdTypeCount: page.unrecognizedJsonLdTypes?.length ?? 0
|
|
22044
22147
|
}));
|
|
22045
|
-
if ((page.invalidJsonLdCount ?? 0) > 0) issues.push(issue("jsonld_invalid", page, `${page.invalidJsonLdCount} invalid`, {
|
|
22148
|
+
if ((page.invalidJsonLdCount ?? 0) > 0) issues.push(issue$1("jsonld_invalid", page, `${page.invalidJsonLdCount} invalid`, {
|
|
22046
22149
|
invalidJsonLdCount: page.invalidJsonLdCount,
|
|
22047
22150
|
invalidJsonLdSamples: page.invalidJsonLdSamples
|
|
22048
22151
|
}));
|
|
@@ -22052,15 +22155,15 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
22052
22155
|
if (incompleteRichResultCount) {
|
|
22053
22156
|
const details = incompleteRichResults.map((assessment) => `${assessment.schemaType}: ${assessment.missingRequiredProperties.join(", ")}`);
|
|
22054
22157
|
const omittedDetails = incompleteRichResultCount - details.length;
|
|
22055
|
-
issues.push(issue("rich_result_required_fields_missing", page, [...details, ...omittedDetails > 0 ? [`${omittedDetails} more incomplete assessment${omittedDetails === 1 ? "" : "s"} omitted from bounded detail`] : []].join("; "), {
|
|
22158
|
+
issues.push(issue$1("rich_result_required_fields_missing", page, [...details, ...omittedDetails > 0 ? [`${omittedDetails} more incomplete assessment${omittedDetails === 1 ? "" : "s"} omitted from bounded detail`] : []].join("; "), {
|
|
22056
22159
|
assessments: incompleteRichResults,
|
|
22057
22160
|
...page.googleRichResultsSelection ? { selection: page.googleRichResultsSelection } : {}
|
|
22058
22161
|
}));
|
|
22059
22162
|
}
|
|
22060
|
-
if (!page.openGraphTitle) issues.push(issue("og_title_missing", page));
|
|
22061
|
-
if (!page.openGraphDescription) issues.push(issue("og_description_missing", page));
|
|
22062
|
-
if (!page.openGraphImage) issues.push(issue("og_image_missing", page));
|
|
22063
|
-
if (!page.twitterCard) issues.push(issue("twitter_card_missing", page));
|
|
22163
|
+
if (!page.openGraphTitle) issues.push(issue$1("og_title_missing", page));
|
|
22164
|
+
if (!page.openGraphDescription) issues.push(issue$1("og_description_missing", page));
|
|
22165
|
+
if (!page.openGraphImage) issues.push(issue$1("og_image_missing", page));
|
|
22166
|
+
if (!page.twitterCard) issues.push(issue$1("twitter_card_missing", page));
|
|
22064
22167
|
}
|
|
22065
22168
|
return issues;
|
|
22066
22169
|
}
|
|
@@ -22150,7 +22253,7 @@ const SEVERITY_SCORE = {
|
|
|
22150
22253
|
const EFFORT_BY_CATEGORY = {
|
|
22151
22254
|
canonical: "medium",
|
|
22152
22255
|
content: "high",
|
|
22153
|
-
|
|
22256
|
+
agents: "low",
|
|
22154
22257
|
headings: "low",
|
|
22155
22258
|
images: "medium",
|
|
22156
22259
|
indexability: "medium",
|
|
@@ -22717,6 +22820,92 @@ const crawlSiteChecksSchema = z.object({ soft404: z.object({
|
|
|
22717
22820
|
complete: z.boolean()
|
|
22718
22821
|
}) });
|
|
22719
22822
|
//#endregion
|
|
22823
|
+
//#region packages/core/dist/analyze/crawler/site-crawl-ai-schema.js
|
|
22824
|
+
const crawlAiAvailabilitySchema = z.enum([
|
|
22825
|
+
"available",
|
|
22826
|
+
"absent",
|
|
22827
|
+
"access-blocked",
|
|
22828
|
+
"rate-limited",
|
|
22829
|
+
"unreachable"
|
|
22830
|
+
]);
|
|
22831
|
+
const crawlAiLinkCheckSchema = z.object({
|
|
22832
|
+
url: z.string().url(),
|
|
22833
|
+
status: z.number().int().optional(),
|
|
22834
|
+
contentType: z.string().optional(),
|
|
22835
|
+
error: z.string().optional()
|
|
22836
|
+
});
|
|
22837
|
+
const crawlAiSignalsSchema = z.object({
|
|
22838
|
+
robotsTxt: z.object({
|
|
22839
|
+
url: z.string().url(),
|
|
22840
|
+
exists: z.boolean(),
|
|
22841
|
+
availability: crawlAiAvailabilitySchema,
|
|
22842
|
+
status: z.number().int().optional(),
|
|
22843
|
+
error: z.string().optional(),
|
|
22844
|
+
sitemapUrls: z.array(z.string().url()),
|
|
22845
|
+
contentSignals: z.array(z.string()).optional(),
|
|
22846
|
+
botAccess: z.array(z.object({
|
|
22847
|
+
userAgent: z.string(),
|
|
22848
|
+
purpose: z.enum([
|
|
22849
|
+
"search",
|
|
22850
|
+
"training",
|
|
22851
|
+
"other"
|
|
22852
|
+
]),
|
|
22853
|
+
allowed: z.boolean().nullable(),
|
|
22854
|
+
declared: z.boolean(),
|
|
22855
|
+
coveredByWildcard: z.boolean()
|
|
22856
|
+
}))
|
|
22857
|
+
}).optional(),
|
|
22858
|
+
llmsTxt: z.object({
|
|
22859
|
+
url: z.string().url(),
|
|
22860
|
+
exists: z.boolean(),
|
|
22861
|
+
availability: crawlAiAvailabilitySchema.optional(),
|
|
22862
|
+
status: z.number().int().optional(),
|
|
22863
|
+
error: z.string().optional(),
|
|
22864
|
+
links: z.object({
|
|
22865
|
+
found: z.number().int().nonnegative(),
|
|
22866
|
+
checked: z.number().int().nonnegative(),
|
|
22867
|
+
limit: z.number().int().positive(),
|
|
22868
|
+
broken: z.array(crawlAiLinkCheckSchema)
|
|
22869
|
+
}).optional()
|
|
22870
|
+
}).optional(),
|
|
22871
|
+
agentResources: z.array(z.object({
|
|
22872
|
+
url: z.string().url(),
|
|
22873
|
+
exists: z.boolean(),
|
|
22874
|
+
status: z.number().int().optional(),
|
|
22875
|
+
contentType: z.string().optional(),
|
|
22876
|
+
validJson: z.boolean().optional()
|
|
22877
|
+
})).optional(),
|
|
22878
|
+
markdownNegotiation: z.object({
|
|
22879
|
+
url: z.string().url(),
|
|
22880
|
+
availability: crawlAiAvailabilitySchema,
|
|
22881
|
+
status: z.number().int().optional(),
|
|
22882
|
+
contentType: z.string().optional(),
|
|
22883
|
+
markdown: z.boolean(),
|
|
22884
|
+
varyAccept: z.boolean(),
|
|
22885
|
+
error: z.string().optional()
|
|
22886
|
+
}).optional(),
|
|
22887
|
+
markdownCopies: z.object({
|
|
22888
|
+
publicPages: z.number().int().nonnegative(),
|
|
22889
|
+
publicPagesWithLink: z.number().int().nonnegative(),
|
|
22890
|
+
privatePagesWithLink: z.number().int().nonnegative(),
|
|
22891
|
+
checked: z.number().int().nonnegative(),
|
|
22892
|
+
limit: z.number().int().positive(),
|
|
22893
|
+
broken: z.array(crawlAiLinkCheckSchema.extend({
|
|
22894
|
+
page: z.string().url(),
|
|
22895
|
+
public: z.boolean()
|
|
22896
|
+
})),
|
|
22897
|
+
largest: z.object({
|
|
22898
|
+
url: z.string().url(),
|
|
22899
|
+
estimatedTokens: z.number().int().nonnegative()
|
|
22900
|
+
}).optional()
|
|
22901
|
+
}).optional(),
|
|
22902
|
+
startPageHeaders: z.object({
|
|
22903
|
+
url: z.string().url(),
|
|
22904
|
+
linkRelations: z.array(z.string()),
|
|
22905
|
+
contentSignal: z.string().optional()
|
|
22906
|
+
}).optional()
|
|
22907
|
+
});
|
|
22908
|
+
//#endregion
|
|
22720
22909
|
//#region packages/core/dist/analyze/crawler/schemas.js
|
|
22721
22910
|
const crawlerRuleSeveritySchema = z.enum([
|
|
22722
22911
|
"low",
|
|
@@ -22739,7 +22928,7 @@ const crawlerRuleCategorySchema = z.enum([
|
|
|
22739
22928
|
"security",
|
|
22740
22929
|
"social",
|
|
22741
22930
|
"structured-data",
|
|
22742
|
-
"
|
|
22931
|
+
"agents"
|
|
22743
22932
|
]);
|
|
22744
22933
|
const crawlerRuleIdSchema = z.enum(listRules().map((rule) => rule.id));
|
|
22745
22934
|
const crawlerRuleInfoSchema = z.object({
|
|
@@ -23365,41 +23554,6 @@ const crawlIssueGroupSchema = z.object({
|
|
|
23365
23554
|
count: z.number().int(),
|
|
23366
23555
|
sampleUrls: z.array(z.string().url())
|
|
23367
23556
|
});
|
|
23368
|
-
const crawlAiSignalsSchema = z.object({
|
|
23369
|
-
robotsTxt: z.object({
|
|
23370
|
-
url: z.string().url(),
|
|
23371
|
-
exists: z.boolean(),
|
|
23372
|
-
availability: z.enum([
|
|
23373
|
-
"available",
|
|
23374
|
-
"absent",
|
|
23375
|
-
"access-blocked",
|
|
23376
|
-
"rate-limited",
|
|
23377
|
-
"unreachable"
|
|
23378
|
-
]),
|
|
23379
|
-
status: z.number().int().optional(),
|
|
23380
|
-
error: z.string().optional(),
|
|
23381
|
-
sitemapUrls: z.array(z.string().url()),
|
|
23382
|
-
contentSignals: z.array(z.string()).optional(),
|
|
23383
|
-
botAccess: z.array(z.object({
|
|
23384
|
-
userAgent: z.string(),
|
|
23385
|
-
allowed: z.boolean().nullable(),
|
|
23386
|
-
declared: z.boolean(),
|
|
23387
|
-
coveredByWildcard: z.boolean()
|
|
23388
|
-
}))
|
|
23389
|
-
}).optional(),
|
|
23390
|
-
llmsTxt: z.object({
|
|
23391
|
-
url: z.string().url(),
|
|
23392
|
-
exists: z.boolean(),
|
|
23393
|
-
status: z.number().int().optional()
|
|
23394
|
-
}).optional(),
|
|
23395
|
-
agentResources: z.array(z.object({
|
|
23396
|
-
url: z.string().url(),
|
|
23397
|
-
exists: z.boolean(),
|
|
23398
|
-
status: z.number().int().optional(),
|
|
23399
|
-
contentType: z.string().optional(),
|
|
23400
|
-
validJson: z.boolean().optional()
|
|
23401
|
-
})).optional()
|
|
23402
|
-
});
|
|
23403
23557
|
const crawlTopFixSchema = crawlIssueGroupSchema.extend({
|
|
23404
23558
|
recommendation: crawlerRuleRecommendationSchema,
|
|
23405
23559
|
score: z.number(),
|
|
@@ -23696,6 +23850,58 @@ const crawlReportSchema = z.discriminatedUnion("requestEvidenceStatus", [
|
|
|
23696
23850
|
]);
|
|
23697
23851
|
z.toJSONSchema(crawlReportSchema), z.toJSONSchema(crawlIssueGroupSchema), z.toJSONSchema(crawlTopFixSchema), z.toJSONSchema(crawlerRuleInfoSchema), z.toJSONSchema(crawlRequestObservationSchema), z.toJSONSchema(crawlPageSnapshotSchema);
|
|
23698
23852
|
//#endregion
|
|
23853
|
+
//#region packages/core/dist/analyze/crawler/agent-ready.js
|
|
23854
|
+
function needsSignIn(page) {
|
|
23855
|
+
return page.status === 401 || page.status === 403 || Boolean(page.softAuthenticationGate);
|
|
23856
|
+
}
|
|
23857
|
+
function isPublicPage(page) {
|
|
23858
|
+
return page.status >= 200 && page.status < 300 && !needsSignIn(page) && /\btext\/html\b/i.test(page.contentType ?? "");
|
|
23859
|
+
}
|
|
23860
|
+
function expectsMarkdown(page) {
|
|
23861
|
+
return isPublicPage(page) && page.indexable;
|
|
23862
|
+
}
|
|
23863
|
+
function issue(ruleId, url, detail, evidence) {
|
|
23864
|
+
const rule = explainRule(ruleId);
|
|
23865
|
+
if (!rule) throw new Error(`Missing rule guidance for ${ruleId}`);
|
|
23866
|
+
return {
|
|
23867
|
+
ruleId,
|
|
23868
|
+
title: rule.title,
|
|
23869
|
+
category: rule.category,
|
|
23870
|
+
severity: rule.defaultSeverity,
|
|
23871
|
+
url,
|
|
23872
|
+
detail,
|
|
23873
|
+
...evidence ? { evidence } : {}
|
|
23874
|
+
};
|
|
23875
|
+
}
|
|
23876
|
+
function agentReadyIssues(input) {
|
|
23877
|
+
const { ai, startUrl } = input;
|
|
23878
|
+
const issues = [];
|
|
23879
|
+
const blocked = (ai.robotsTxt?.botAccess ?? []).filter((bot) => bot.purpose === "search" && bot.allowed === false);
|
|
23880
|
+
if (blocked.length && ai.robotsTxt) issues.push(issue("ai_search_bot_blocked", ai.robotsTxt.url, `robots.txt blocks ${blocked.map((bot) => bot.userAgent).join(", ")} from the start URL.`, { blocked: blocked.map((bot) => bot.userAgent) }));
|
|
23881
|
+
if (ai.llmsTxt?.availability === "absent") issues.push(issue("llms_txt_missing", ai.llmsTxt.url, `/llms.txt returned ${ai.llmsTxt.status === void 0 ? "no file" : `HTTP ${ai.llmsTxt.status}`}.`));
|
|
23882
|
+
for (const link of ai.llmsTxt?.links?.broken ?? []) issues.push(issue("llms_txt_broken_link", link.url, link.status === void 0 ? `The link in llms.txt could not be fetched: ${link.error ?? "no response"}.` : `The link in llms.txt returned HTTP ${link.status}.`, {
|
|
23883
|
+
llmsTxt: ai.llmsTxt?.url,
|
|
23884
|
+
...link
|
|
23885
|
+
}));
|
|
23886
|
+
const expected = input.pages.filter(expectsMarkdown);
|
|
23887
|
+
const linked = expected.filter((page) => page.markdownAlternates?.length);
|
|
23888
|
+
const negotiation = ai.markdownNegotiation;
|
|
23889
|
+
if (expected.length && !linked.length && negotiation?.availability === "available" && !negotiation.markdown) issues.push(issue("markdown_missing", startUrl, `Accept: text/markdown returned ${negotiation.contentType || "no content type"}, and none of ${expected.length} public pages links to a Markdown copy.`, { negotiation }));
|
|
23890
|
+
if (linked.length) for (const page of expected) {
|
|
23891
|
+
if (page.markdownAlternates?.length) continue;
|
|
23892
|
+
issues.push(issue("markdown_alternate_missing", page.url, `${linked.length} of ${expected.length} public pages link to a Markdown copy; this one does not.`));
|
|
23893
|
+
}
|
|
23894
|
+
for (const page of input.pages) {
|
|
23895
|
+
if (!needsSignIn(page) || !page.markdownAlternates?.length) continue;
|
|
23896
|
+
issues.push(issue("markdown_alternate_on_private_page", page.url, `The page needs a sign-in but links to a Markdown copy at ${page.markdownAlternates[0]}.`, { markdownAlternates: page.markdownAlternates }));
|
|
23897
|
+
}
|
|
23898
|
+
for (const copy of ai.markdownCopies?.broken ?? []) {
|
|
23899
|
+
if (!copy.public) continue;
|
|
23900
|
+
issues.push(issue("markdown_copy_broken", copy.page, copy.status === void 0 ? `The Markdown copy at ${copy.url} could not be fetched: ${copy.error ?? "no response"}.` : `The Markdown copy at ${copy.url} returned HTTP ${copy.status}${copy.contentType ? ` as ${copy.contentType}` : ""}.`, { ...copy }));
|
|
23901
|
+
}
|
|
23902
|
+
return issues;
|
|
23903
|
+
}
|
|
23904
|
+
//#endregion
|
|
23699
23905
|
//#region packages/core/dist/analyze/crawler/crawl-caveats.js
|
|
23700
23906
|
function crawlCaveats(input) {
|
|
23701
23907
|
const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
|
|
@@ -24133,66 +24339,137 @@ async function collectCrawlSiteChecks(input) {
|
|
|
24133
24339
|
//#endregion
|
|
24134
24340
|
//#region packages/core/dist/analyze/crawler/site-crawl-ai-signals.js
|
|
24135
24341
|
const MAX_AUXILIARY_RESPONSE_BYTES = 2 * 1024 * 1024;
|
|
24136
|
-
const
|
|
24137
|
-
|
|
24138
|
-
"OAI-SearchBot",
|
|
24139
|
-
"ChatGPT-User",
|
|
24140
|
-
"
|
|
24141
|
-
"Claude-User",
|
|
24142
|
-
"
|
|
24143
|
-
"
|
|
24144
|
-
"
|
|
24145
|
-
"
|
|
24146
|
-
"
|
|
24147
|
-
"Applebot",
|
|
24148
|
-
"
|
|
24149
|
-
"
|
|
24150
|
-
"
|
|
24151
|
-
"
|
|
24152
|
-
"
|
|
24153
|
-
"Meta-
|
|
24154
|
-
"
|
|
24155
|
-
"
|
|
24156
|
-
"
|
|
24157
|
-
"
|
|
24342
|
+
const LLMS_LINK_LIMIT = 20;
|
|
24343
|
+
const AI_BOTS = [
|
|
24344
|
+
["OAI-SearchBot", "search"],
|
|
24345
|
+
["ChatGPT-User", "search"],
|
|
24346
|
+
["Claude-SearchBot", "search"],
|
|
24347
|
+
["Claude-User", "search"],
|
|
24348
|
+
["PerplexityBot", "search"],
|
|
24349
|
+
["Perplexity-User", "search"],
|
|
24350
|
+
["DuckAssistBot", "search"],
|
|
24351
|
+
["MistralAI-User", "search"],
|
|
24352
|
+
["Meta-ExternalFetcher", "search"],
|
|
24353
|
+
["Applebot", "search"],
|
|
24354
|
+
["Bingbot", "search"],
|
|
24355
|
+
["GPTBot", "training"],
|
|
24356
|
+
["ClaudeBot", "training"],
|
|
24357
|
+
["Google-Extended", "training"],
|
|
24358
|
+
["Applebot-Extended", "training"],
|
|
24359
|
+
["Meta-ExternalAgent", "training"],
|
|
24360
|
+
["CCBot", "training"],
|
|
24361
|
+
["Bytespider", "training"],
|
|
24362
|
+
["cohere-ai", "training"],
|
|
24363
|
+
["Amazonbot", "other"],
|
|
24364
|
+
["Diffbot", "other"]
|
|
24158
24365
|
];
|
|
24159
24366
|
const AGENT_RESOURCE_PATHS = [
|
|
24160
|
-
"/.well-known/agent.json",
|
|
24161
|
-
"/
|
|
24367
|
+
"/.well-known/agent-skills/index.json",
|
|
24368
|
+
"/.well-known/mcp/server-card.json",
|
|
24162
24369
|
"/.well-known/mcp.json",
|
|
24163
|
-
"/.well-known/
|
|
24164
|
-
"
|
|
24165
|
-
"/
|
|
24370
|
+
"/.well-known/api-catalog",
|
|
24371
|
+
"/openapi.json",
|
|
24372
|
+
"/.well-known/oauth-protected-resource",
|
|
24373
|
+
"/.well-known/oauth-authorization-server",
|
|
24374
|
+
"/.well-known/http-message-signatures-directory"
|
|
24166
24375
|
];
|
|
24167
|
-
|
|
24168
|
-
|
|
24376
|
+
function availabilityOf(status) {
|
|
24377
|
+
if (status >= 200 && status < 300) return "available";
|
|
24378
|
+
if (status === 401 || status === 403) return "access-blocked";
|
|
24379
|
+
if (status === 429) return "rate-limited";
|
|
24380
|
+
if (status >= 400 && status < 500) return "absent";
|
|
24381
|
+
return "unreachable";
|
|
24382
|
+
}
|
|
24383
|
+
function errorText(error) {
|
|
24384
|
+
return error instanceof Error ? error.message : String(error);
|
|
24385
|
+
}
|
|
24386
|
+
async function get(input, url, headers, readBody = true) {
|
|
24169
24387
|
const controller = abortController({
|
|
24170
24388
|
timeoutMs: input.timeoutMs,
|
|
24171
24389
|
signal: input.signal
|
|
24172
24390
|
});
|
|
24173
24391
|
try {
|
|
24174
|
-
const response = await input.fetch(
|
|
24392
|
+
const response = await input.fetch(url, {
|
|
24175
24393
|
profile: "bot",
|
|
24176
24394
|
redirect: "follow",
|
|
24177
|
-
signal: controller.signal
|
|
24395
|
+
signal: controller.signal,
|
|
24396
|
+
...headers ? { headers } : {}
|
|
24178
24397
|
});
|
|
24179
|
-
|
|
24180
|
-
|
|
24181
|
-
await response.body?.cancel().catch(() => void 0);
|
|
24398
|
+
let text = "";
|
|
24399
|
+
if (readBody) text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, `${new URL(url).pathname} response`);
|
|
24400
|
+
else await response.body?.cancel().catch(() => void 0);
|
|
24182
24401
|
return {
|
|
24183
|
-
url: response.url ||
|
|
24184
|
-
|
|
24185
|
-
status: response.status
|
|
24402
|
+
url: response.url || url,
|
|
24403
|
+
availability: availabilityOf(response.status),
|
|
24404
|
+
status: response.status,
|
|
24405
|
+
contentType: response.headers.get("content-type") ?? "",
|
|
24406
|
+
headers: response.headers,
|
|
24407
|
+
text
|
|
24186
24408
|
};
|
|
24187
|
-
} catch {
|
|
24409
|
+
} catch (error) {
|
|
24188
24410
|
return {
|
|
24189
|
-
url
|
|
24190
|
-
|
|
24411
|
+
url,
|
|
24412
|
+
availability: "unreachable",
|
|
24413
|
+
contentType: "",
|
|
24414
|
+
text: "",
|
|
24415
|
+
error: errorText(error)
|
|
24191
24416
|
};
|
|
24192
24417
|
} finally {
|
|
24193
24418
|
controller.cleanup();
|
|
24194
24419
|
}
|
|
24195
24420
|
}
|
|
24421
|
+
const isHtml = (contentType) => /\btext\/html\b/i.test(contentType);
|
|
24422
|
+
const isMarkdown = (contentType) => /\btext\/(?:x-)?markdown\b/i.test(contentType);
|
|
24423
|
+
function brokenLink(result, expectText) {
|
|
24424
|
+
if (result.availability === "available" && !(expectText && isHtml(result.contentType))) return void 0;
|
|
24425
|
+
return {
|
|
24426
|
+
url: result.url,
|
|
24427
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
24428
|
+
...result.contentType ? { contentType: result.contentType } : {},
|
|
24429
|
+
...result.error ? { error: result.error } : {}
|
|
24430
|
+
};
|
|
24431
|
+
}
|
|
24432
|
+
function llmsLinks(text, base) {
|
|
24433
|
+
const origin = new URL(base).origin;
|
|
24434
|
+
const links = [];
|
|
24435
|
+
for (const match of text.matchAll(/\]\(\s*<?([^)\s>]+)>?/g)) try {
|
|
24436
|
+
const url = new URL(match[1] ?? "", base);
|
|
24437
|
+
url.hash = "";
|
|
24438
|
+
if (url.origin === origin && !links.includes(url.toString())) links.push(url.toString());
|
|
24439
|
+
} catch {}
|
|
24440
|
+
return links;
|
|
24441
|
+
}
|
|
24442
|
+
async function checkLlmsTxt(input) {
|
|
24443
|
+
const result = await get(input, new URL("/llms.txt", input.url).toString());
|
|
24444
|
+
const availability = result.availability === "available" && isHtml(result.contentType) ? "absent" : result.availability;
|
|
24445
|
+
const exists = availability === "available";
|
|
24446
|
+
const base = {
|
|
24447
|
+
url: result.url,
|
|
24448
|
+
exists,
|
|
24449
|
+
availability,
|
|
24450
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
24451
|
+
...result.error ? { error: result.error } : {}
|
|
24452
|
+
};
|
|
24453
|
+
if (!exists) return base;
|
|
24454
|
+
const found = llmsLinks(result.text, result.url);
|
|
24455
|
+
const broken = [];
|
|
24456
|
+
let checked = 0;
|
|
24457
|
+
for (const url of found.slice(0, LLMS_LINK_LIMIT)) {
|
|
24458
|
+
if (input.signal?.aborted) break;
|
|
24459
|
+
checked += 1;
|
|
24460
|
+
const link = brokenLink(await get(input, url, void 0, false), false);
|
|
24461
|
+
if (link) broken.push(link);
|
|
24462
|
+
}
|
|
24463
|
+
return {
|
|
24464
|
+
...base,
|
|
24465
|
+
links: {
|
|
24466
|
+
found: found.length,
|
|
24467
|
+
checked,
|
|
24468
|
+
limit: LLMS_LINK_LIMIT,
|
|
24469
|
+
broken
|
|
24470
|
+
}
|
|
24471
|
+
};
|
|
24472
|
+
}
|
|
24196
24473
|
function parseRobots(robotsUrl, text) {
|
|
24197
24474
|
return robotsParserModule(robotsUrl, text);
|
|
24198
24475
|
}
|
|
@@ -24224,118 +24501,135 @@ function sitemapUrlsFromRobots(text) {
|
|
|
24224
24501
|
return [...urls];
|
|
24225
24502
|
}
|
|
24226
24503
|
async function checkRobotsAiAccess(input) {
|
|
24227
|
-
const
|
|
24228
|
-
const
|
|
24229
|
-
|
|
24230
|
-
|
|
24231
|
-
|
|
24232
|
-
|
|
24233
|
-
|
|
24234
|
-
|
|
24235
|
-
redirect: "follow",
|
|
24236
|
-
signal: controller.signal
|
|
24237
|
-
});
|
|
24238
|
-
const text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, "robots.txt response");
|
|
24239
|
-
const available = response.status >= 200 && response.status < 300;
|
|
24240
|
-
const absent = response.status >= 400 && response.status < 500 && response.status !== 429;
|
|
24241
|
-
const availability = available ? "available" : [401, 403].includes(response.status) ? "access-blocked" : response.status === 429 ? "rate-limited" : absent ? "absent" : "unreachable";
|
|
24242
|
-
const exists = available;
|
|
24243
|
-
const declared = exists ? declaredUserAgents(text) : /* @__PURE__ */ new Set();
|
|
24244
|
-
const parsed = parseRobots(robotsUrl, exists ? text : "");
|
|
24245
|
-
return {
|
|
24246
|
-
url: response.url || robotsUrl,
|
|
24504
|
+
const result = await get(input, new URL("/robots.txt", input.url).toString());
|
|
24505
|
+
const exists = result.availability === "available";
|
|
24506
|
+
const unknown = result.availability === "rate-limited" || result.availability === "unreachable";
|
|
24507
|
+
const declared = exists ? declaredUserAgents(result.text) : /* @__PURE__ */ new Set();
|
|
24508
|
+
const parsed = parseRobots(result.url, exists ? result.text : "");
|
|
24509
|
+
return {
|
|
24510
|
+
robotsTxt: {
|
|
24511
|
+
url: result.url,
|
|
24247
24512
|
exists,
|
|
24248
|
-
availability,
|
|
24249
|
-
status:
|
|
24250
|
-
...
|
|
24251
|
-
sitemapUrls: exists ? sitemapUrlsFromRobots(text) : [],
|
|
24252
|
-
contentSignals: exists ? contentSignalsFromRobots(text) : [],
|
|
24253
|
-
botAccess:
|
|
24254
|
-
const lower = userAgent.toLowerCase();
|
|
24255
|
-
return {
|
|
24256
|
-
userAgent,
|
|
24257
|
-
allowed: availability === "rate-limited" || availability === "unreachable" ? null : parsed.isAllowed(input.url, userAgent) ?? true,
|
|
24258
|
-
declared: declared.has(lower),
|
|
24259
|
-
coveredByWildcard: declared.has("*")
|
|
24260
|
-
};
|
|
24261
|
-
})
|
|
24262
|
-
};
|
|
24263
|
-
} catch (error) {
|
|
24264
|
-
return {
|
|
24265
|
-
url: robotsUrl,
|
|
24266
|
-
exists: false,
|
|
24267
|
-
availability: "unreachable",
|
|
24268
|
-
error: error instanceof Error ? error.message : String(error),
|
|
24269
|
-
sitemapUrls: [],
|
|
24270
|
-
botAccess: AI_BOT_USER_AGENTS.map((userAgent) => ({
|
|
24513
|
+
availability: result.availability,
|
|
24514
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
24515
|
+
...unknown ? { error: result.error ?? `robots.txt returned HTTP ${result.status}.` } : {},
|
|
24516
|
+
sitemapUrls: exists ? sitemapUrlsFromRobots(result.text) : [],
|
|
24517
|
+
...unknown ? {} : { contentSignals: exists ? contentSignalsFromRobots(result.text) : [] },
|
|
24518
|
+
botAccess: AI_BOTS.map(([userAgent, purpose]) => ({
|
|
24271
24519
|
userAgent,
|
|
24272
|
-
|
|
24273
|
-
|
|
24274
|
-
|
|
24520
|
+
purpose,
|
|
24521
|
+
allowed: unknown ? null : parsed.isAllowed(input.url, userAgent) ?? true,
|
|
24522
|
+
declared: declared.has(userAgent.toLowerCase()),
|
|
24523
|
+
coveredByWildcard: declared.has("*")
|
|
24275
24524
|
}))
|
|
24276
|
-
}
|
|
24277
|
-
|
|
24278
|
-
|
|
24279
|
-
}
|
|
24525
|
+
},
|
|
24526
|
+
allows: (url) => unknown ? null : parsed.isAllowed(url, SEO_CRAWLER_USER_AGENT) ?? true
|
|
24527
|
+
};
|
|
24280
24528
|
}
|
|
24281
|
-
async function
|
|
24282
|
-
const
|
|
24283
|
-
const
|
|
24284
|
-
|
|
24285
|
-
|
|
24286
|
-
|
|
24287
|
-
try {
|
|
24288
|
-
const response = await input.fetch(url, {
|
|
24289
|
-
profile: "bot",
|
|
24290
|
-
redirect: "follow",
|
|
24291
|
-
signal: controller.signal
|
|
24292
|
-
});
|
|
24293
|
-
const contentType = response.headers.get("content-type") ?? "";
|
|
24294
|
-
const text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, "Agent resource response");
|
|
24295
|
-
const exists = response.status >= 200 && response.status < 300;
|
|
24529
|
+
async function checkAgentResources(input) {
|
|
24530
|
+
const resources = [];
|
|
24531
|
+
for (const path of AGENT_RESOURCE_PATHS) {
|
|
24532
|
+
if (input.signal?.aborted) break;
|
|
24533
|
+
const result = await get(input, new URL(path, input.url).toString());
|
|
24534
|
+
const exists = result.availability === "available" && !isHtml(result.contentType);
|
|
24296
24535
|
let validJson;
|
|
24297
|
-
if (exists && /\bjson\b/i.test(contentType)) try {
|
|
24298
|
-
JSON.parse(text);
|
|
24536
|
+
if (exists && /\bjson\b/i.test(result.contentType)) try {
|
|
24537
|
+
JSON.parse(result.text);
|
|
24299
24538
|
validJson = true;
|
|
24300
24539
|
} catch {
|
|
24301
24540
|
validJson = false;
|
|
24302
24541
|
}
|
|
24303
|
-
|
|
24304
|
-
url:
|
|
24542
|
+
resources.push({
|
|
24543
|
+
url: result.url,
|
|
24305
24544
|
exists,
|
|
24306
|
-
status:
|
|
24307
|
-
contentType,
|
|
24545
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
24546
|
+
...result.contentType ? { contentType: result.contentType } : {},
|
|
24308
24547
|
...validJson === void 0 ? {} : { validJson }
|
|
24309
|
-
};
|
|
24310
|
-
} catch {
|
|
24311
|
-
return {
|
|
24312
|
-
url,
|
|
24313
|
-
exists: false
|
|
24314
|
-
};
|
|
24315
|
-
} finally {
|
|
24316
|
-
controller.cleanup();
|
|
24317
|
-
}
|
|
24318
|
-
}
|
|
24319
|
-
async function checkAgentResources(input) {
|
|
24320
|
-
const resources = [];
|
|
24321
|
-
for (const path of AGENT_RESOURCE_PATHS) {
|
|
24322
|
-
if (input.signal?.aborted) break;
|
|
24323
|
-
resources.push(await checkAgentResource({
|
|
24324
|
-
baseUrl: input.url,
|
|
24325
|
-
path,
|
|
24326
|
-
timeoutMs: input.timeoutMs,
|
|
24327
|
-
fetch: input.fetch,
|
|
24328
|
-
signal: input.signal
|
|
24329
|
-
}));
|
|
24548
|
+
});
|
|
24330
24549
|
}
|
|
24331
24550
|
return resources;
|
|
24332
24551
|
}
|
|
24333
|
-
function
|
|
24334
|
-
|
|
24552
|
+
async function checkMarkdownNegotiation(input) {
|
|
24553
|
+
const result = await get(input, input.url, { accept: "text/markdown" }, false);
|
|
24554
|
+
const vary = result.headers?.get("vary") ?? "";
|
|
24555
|
+
return {
|
|
24556
|
+
url: result.url,
|
|
24557
|
+
availability: result.availability,
|
|
24558
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
24559
|
+
...result.contentType ? { contentType: result.contentType } : {},
|
|
24560
|
+
markdown: result.availability === "available" && isMarkdown(result.contentType),
|
|
24561
|
+
varyAccept: vary.split(",").some((value) => value.trim().toLowerCase() === "accept"),
|
|
24562
|
+
...result.error ? { error: result.error } : {}
|
|
24563
|
+
};
|
|
24564
|
+
}
|
|
24565
|
+
async function collectCrawlAiSignals(input) {
|
|
24566
|
+
const robots = await checkRobotsAiAccess(input);
|
|
24567
|
+
const allows = input.respectRobots ? robots.allows : () => true;
|
|
24568
|
+
const startAllowed = allows(input.url);
|
|
24569
|
+
const [llmsTxt, agentResources, markdownNegotiation] = await Promise.all([
|
|
24335
24570
|
checkLlmsTxt(input),
|
|
24336
|
-
|
|
24337
|
-
|
|
24571
|
+
checkAgentResources(input),
|
|
24572
|
+
startAllowed ? checkMarkdownNegotiation(input) : Promise.resolve({
|
|
24573
|
+
url: input.url,
|
|
24574
|
+
availability: "unreachable",
|
|
24575
|
+
markdown: false,
|
|
24576
|
+
varyAccept: false,
|
|
24577
|
+
error: startAllowed === null ? "Not requested: robots.txt could not be read." : "Not requested: robots.txt disallows the start URL."
|
|
24578
|
+
})
|
|
24338
24579
|
]);
|
|
24580
|
+
return {
|
|
24581
|
+
signals: {
|
|
24582
|
+
llmsTxt,
|
|
24583
|
+
robotsTxt: robots.robotsTxt,
|
|
24584
|
+
agentResources,
|
|
24585
|
+
markdownNegotiation
|
|
24586
|
+
},
|
|
24587
|
+
allows
|
|
24588
|
+
};
|
|
24589
|
+
}
|
|
24590
|
+
async function checkMarkdownCopies(input) {
|
|
24591
|
+
const linked = input.pages.filter((page) => page.markdownAlternates?.length).sort((a, b) => Number(a.public) - Number(b.public) || a.url.localeCompare(b.url));
|
|
24592
|
+
const broken = [];
|
|
24593
|
+
let largest;
|
|
24594
|
+
let checked = 0;
|
|
24595
|
+
for (const page of linked.slice(0, 20)) {
|
|
24596
|
+
if (input.signal?.aborted) break;
|
|
24597
|
+
const url = page.markdownAlternates?.[0];
|
|
24598
|
+
if (!url || !input.allows(url)) continue;
|
|
24599
|
+
checked += 1;
|
|
24600
|
+
const result = await get({
|
|
24601
|
+
...input,
|
|
24602
|
+
url
|
|
24603
|
+
}, url);
|
|
24604
|
+
const link = brokenLink(result, true);
|
|
24605
|
+
if (link) {
|
|
24606
|
+
broken.push({
|
|
24607
|
+
...link,
|
|
24608
|
+
page: page.url,
|
|
24609
|
+
public: page.public
|
|
24610
|
+
});
|
|
24611
|
+
continue;
|
|
24612
|
+
}
|
|
24613
|
+
const estimatedTokens = Math.ceil(result.text.length / 4);
|
|
24614
|
+
if (!largest || estimatedTokens > largest.estimatedTokens) largest = {
|
|
24615
|
+
url: result.url,
|
|
24616
|
+
estimatedTokens
|
|
24617
|
+
};
|
|
24618
|
+
}
|
|
24619
|
+
return {
|
|
24620
|
+
publicPages: input.pages.filter((page) => page.public).length,
|
|
24621
|
+
publicPagesWithLink: linked.filter((page) => page.public).length,
|
|
24622
|
+
privatePagesWithLink: linked.filter((page) => !page.public).length,
|
|
24623
|
+
checked,
|
|
24624
|
+
limit: 20,
|
|
24625
|
+
broken,
|
|
24626
|
+
...largest ? { largest } : {}
|
|
24627
|
+
};
|
|
24628
|
+
}
|
|
24629
|
+
function linkRelations(header) {
|
|
24630
|
+
const relations = /* @__PURE__ */ new Set();
|
|
24631
|
+
for (const match of (header ?? "").matchAll(/rel\s*=\s*"?([^";,]+)"?/gi)) for (const value of (match[1] ?? "").trim().toLowerCase().split(/\s+/)) if (value) relations.add(value);
|
|
24632
|
+
return [...relations].sort();
|
|
24339
24633
|
}
|
|
24340
24634
|
//#endregion
|
|
24341
24635
|
//#region packages/core/dist/analyze/crawler/site-crawl-providers.js
|
|
@@ -26743,12 +27037,11 @@ async function crawlSite(input, dependencies) {
|
|
|
26743
27037
|
url: config.url,
|
|
26744
27038
|
timeoutMs: config.timeoutMs,
|
|
26745
27039
|
fetch: deps.fetch,
|
|
27040
|
+
respectRobots: config.respectRobots,
|
|
26746
27041
|
signal
|
|
26747
27042
|
});
|
|
26748
|
-
const llmsTxt = aiSignals?.
|
|
27043
|
+
const llmsTxt = aiSignals?.signals.llmsTxt;
|
|
26749
27044
|
const healthSitemapUrls = healthStrategy && !config.sitemapUrl && !isCancelled() ? await robotsSession?.sitemapUrls(origin) : void 0;
|
|
26750
|
-
const robotsTxt = aiSignals?.[1];
|
|
26751
|
-
const agentResources = aiSignals?.[2];
|
|
26752
27045
|
cancelled = isCancelled();
|
|
26753
27046
|
const enqueue = (value, depth) => {
|
|
26754
27047
|
const normalized = normalizeUrl$1(value, config.url);
|
|
@@ -26826,7 +27119,7 @@ async function crawlSite(input, dependencies) {
|
|
|
26826
27119
|
url: config.url,
|
|
26827
27120
|
sitemapUrl: config.sitemapUrl,
|
|
26828
27121
|
maxPages: config.maxPages,
|
|
26829
|
-
declaredSitemapUrls: robotsTxt?.sitemapUrls ?? healthSitemapUrls ?? [],
|
|
27122
|
+
declaredSitemapUrls: aiSignals?.signals.robotsTxt?.sitemapUrls ?? healthSitemapUrls ?? [],
|
|
26830
27123
|
warnings,
|
|
26831
27124
|
fetchSitemapUrls: deps.fetchSitemapUrls
|
|
26832
27125
|
});
|
|
@@ -27142,6 +27435,26 @@ async function crawlSite(input, dependencies) {
|
|
|
27142
27435
|
llmsTxtStatus: llmsTxt.status
|
|
27143
27436
|
};
|
|
27144
27437
|
}
|
|
27438
|
+
const startPage = pages.find((page) => page.url === config.url || page.finalUrl === config.url);
|
|
27439
|
+
const ai = aiSignals && !cancelled ? {
|
|
27440
|
+
...aiSignals.signals,
|
|
27441
|
+
markdownCopies: await checkMarkdownCopies({
|
|
27442
|
+
allows: aiSignals.allows,
|
|
27443
|
+
pages: pages.map((page) => ({
|
|
27444
|
+
url: page.url,
|
|
27445
|
+
markdownAlternates: page.markdownAlternates,
|
|
27446
|
+
public: isPublicPage(page)
|
|
27447
|
+
})),
|
|
27448
|
+
timeoutMs: config.timeoutMs,
|
|
27449
|
+
fetch: deps.fetch,
|
|
27450
|
+
signal
|
|
27451
|
+
}),
|
|
27452
|
+
...startPage ? { startPageHeaders: {
|
|
27453
|
+
url: startPage.url,
|
|
27454
|
+
linkRelations: linkRelations(startPage.responseHeaders?.link),
|
|
27455
|
+
...startPage.responseHeaders?.["content-signal"] ? { contentSignal: startPage.responseHeaders["content-signal"] } : {}
|
|
27456
|
+
} } : {}
|
|
27457
|
+
} : aiSignals?.signals;
|
|
27145
27458
|
const report = createCrawlReport({
|
|
27146
27459
|
config,
|
|
27147
27460
|
site,
|
|
@@ -27151,17 +27464,15 @@ async function crawlSite(input, dependencies) {
|
|
|
27151
27464
|
pages,
|
|
27152
27465
|
requests,
|
|
27153
27466
|
requestEvidenceStatus,
|
|
27154
|
-
|
|
27155
|
-
...llmsTxt ? { llmsTxt } : {},
|
|
27156
|
-
...robotsTxt ? { robotsTxt } : {},
|
|
27157
|
-
...agentResources ? { agentResources } : {}
|
|
27158
|
-
} } : {},
|
|
27467
|
+
...ai ? { ai } : {},
|
|
27159
27468
|
...sitemapDiscovery ? { sitemapDiscovery } : {},
|
|
27160
27469
|
...externalLinkVerification ? { externalLinkVerification } : {},
|
|
27161
|
-
...siteCheckResult ? {
|
|
27162
|
-
|
|
27163
|
-
|
|
27164
|
-
|
|
27470
|
+
...siteCheckResult ? { siteChecks: siteCheckResult.checks } : {},
|
|
27471
|
+
additionalIssues: [...siteCheckResult?.issues ?? [], ...ai ? agentReadyIssues({
|
|
27472
|
+
startUrl: config.url,
|
|
27473
|
+
ai,
|
|
27474
|
+
pages
|
|
27475
|
+
}) : []],
|
|
27165
27476
|
status: reportStatus,
|
|
27166
27477
|
warnings,
|
|
27167
27478
|
caveats: crawlCaveats({
|
|
@@ -37585,6 +37896,257 @@ async function runAnalyticsRows(input, google, options) {
|
|
|
37585
37896
|
};
|
|
37586
37897
|
}
|
|
37587
37898
|
//#endregion
|
|
37899
|
+
//#region packages/core/dist/ga4/matching.js
|
|
37900
|
+
function hostnameFromUrl(value) {
|
|
37901
|
+
try {
|
|
37902
|
+
const url = new URL(value);
|
|
37903
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
37904
|
+
return url.hostname.toLowerCase().replace(/\.$/, "") || void 0;
|
|
37905
|
+
} catch {
|
|
37906
|
+
return;
|
|
37907
|
+
}
|
|
37908
|
+
}
|
|
37909
|
+
function domainFromProperty(value) {
|
|
37910
|
+
if (!value.startsWith("sc-domain:")) return void 0;
|
|
37911
|
+
const domain = value.slice(10).trim().toLowerCase();
|
|
37912
|
+
if (!domain || /[/:]/.test(domain)) return void 0;
|
|
37913
|
+
return domain.replace(/\.$/, "") || void 0;
|
|
37914
|
+
}
|
|
37915
|
+
function matchForSite(site, streamHostname) {
|
|
37916
|
+
const domain = domainFromProperty(site);
|
|
37917
|
+
if (domain) return streamHostname === domain || streamHostname.endsWith(`.${domain}`) ? "domain" : void 0;
|
|
37918
|
+
return hostnameFromUrl(site) === streamHostname ? "hostname" : void 0;
|
|
37919
|
+
}
|
|
37920
|
+
function matchGa4WebStreams(site, candidates) {
|
|
37921
|
+
return candidates.flatMap((candidate) => {
|
|
37922
|
+
const defaultUri = candidate.stream.webStreamData?.defaultUri;
|
|
37923
|
+
if (!defaultUri) return [];
|
|
37924
|
+
const hostname = hostnameFromUrl(defaultUri);
|
|
37925
|
+
if (!hostname) return [];
|
|
37926
|
+
const match = matchForSite(site, hostname);
|
|
37927
|
+
if (!match) return [];
|
|
37928
|
+
return [{
|
|
37929
|
+
...candidate,
|
|
37930
|
+
defaultUri,
|
|
37931
|
+
hostname,
|
|
37932
|
+
match
|
|
37933
|
+
}];
|
|
37934
|
+
});
|
|
37935
|
+
}
|
|
37936
|
+
//#endregion
|
|
37937
|
+
//#region packages/core/dist/commands/analytics-site.js
|
|
37938
|
+
const STREAM_CHECKS = 30;
|
|
37939
|
+
const AT_ONCE = 5;
|
|
37940
|
+
const TIED = 3;
|
|
37941
|
+
const hostOf$1 = (site) => {
|
|
37942
|
+
const value = site.replace(/^sc-domain:/, "");
|
|
37943
|
+
try {
|
|
37944
|
+
return new URL(value.includes("://") ? value : `https://${value}`).hostname;
|
|
37945
|
+
} catch {
|
|
37946
|
+
return value;
|
|
37947
|
+
}
|
|
37948
|
+
};
|
|
37949
|
+
const plain = (value) => value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
37950
|
+
function ranked(site, candidates) {
|
|
37951
|
+
const host = hostOf$1(site).replace(/^www\./, "");
|
|
37952
|
+
const words = [plain(host), plain(host.split(".")[0] ?? "")].filter((word) => word.length >= 3);
|
|
37953
|
+
const named = (candidate) => words.some((word) => plain(candidate.propertyName).includes(word)) ? 0 : 1;
|
|
37954
|
+
return [...candidates].sort((a, b) => named(a) - named(b) || a.property.localeCompare(b.property, "en", { numeric: true }));
|
|
37955
|
+
}
|
|
37956
|
+
function best(site, matches) {
|
|
37957
|
+
const host = hostOf$1(site).replace(/^www\./, "");
|
|
37958
|
+
const score = (match) => match.hostname === host || match.hostname === `www.${host}` ? 0 : 1;
|
|
37959
|
+
return [...matches].sort((a, b) => score(a) - score(b) || a.hostname.length - b.hostname.length || a.property.localeCompare(b.property, "en", { numeric: true }));
|
|
37960
|
+
}
|
|
37961
|
+
async function findProperty(access, site) {
|
|
37962
|
+
const accounts = await access.accounts();
|
|
37963
|
+
const listed = await Promise.all(accounts.map((account) => listAnalyticsAccounts(account.google, { signIn: access.signIn }).then((summaries) => ({
|
|
37964
|
+
account,
|
|
37965
|
+
summaries
|
|
37966
|
+
}), (error) => error instanceof Error ? error : new Error(String(error)))));
|
|
37967
|
+
const seen = /* @__PURE__ */ new Set();
|
|
37968
|
+
const candidates = [];
|
|
37969
|
+
for (const entry of listed) {
|
|
37970
|
+
if (entry instanceof Error) continue;
|
|
37971
|
+
for (const summary of entry.summaries) for (const item of summary.propertySummaries) {
|
|
37972
|
+
const property = item.property.replace(/^properties\//, "");
|
|
37973
|
+
if (seen.has(property) || !/^\d+$/.test(property)) continue;
|
|
37974
|
+
seen.add(property);
|
|
37975
|
+
candidates.push({
|
|
37976
|
+
account: summary.account,
|
|
37977
|
+
property,
|
|
37978
|
+
propertyName: item.displayName ?? property,
|
|
37979
|
+
google: entry.account.google
|
|
37980
|
+
});
|
|
37981
|
+
}
|
|
37982
|
+
}
|
|
37983
|
+
if (!candidates.length) {
|
|
37984
|
+
const failed = listed.find((entry) => entry instanceof Error);
|
|
37985
|
+
return failed ? `could not be read. ${failed.message}` : "the connected Google accounts cannot read any property.";
|
|
37986
|
+
}
|
|
37987
|
+
const order = ranked(site, candidates).slice(0, STREAM_CHECKS);
|
|
37988
|
+
let failures = 0;
|
|
37989
|
+
for (let start = 0; start < order.length; start += AT_ONCE) {
|
|
37990
|
+
const batch = order.slice(start, start + AT_ONCE);
|
|
37991
|
+
const streams = await Promise.all(batch.map((candidate) => listAnalyticsDataStreams(candidate.google, candidate.property, { signIn: access.signIn }).catch(() => {
|
|
37992
|
+
failures += 1;
|
|
37993
|
+
return [];
|
|
37994
|
+
})));
|
|
37995
|
+
const matches = best(site, batch.flatMap((candidate, index) => matchGa4WebStreams(site, (streams[index] ?? []).map((stream) => ({
|
|
37996
|
+
...candidate,
|
|
37997
|
+
stream
|
|
37998
|
+
})))));
|
|
37999
|
+
const found = /* @__PURE__ */ new Map();
|
|
38000
|
+
for (const match of matches) {
|
|
38001
|
+
const candidate = batch.find((item) => item.property === match.property);
|
|
38002
|
+
if (!candidate || found.has(match.property)) continue;
|
|
38003
|
+
found.set(match.property, {
|
|
38004
|
+
property: match.property,
|
|
38005
|
+
name: match.propertyName,
|
|
38006
|
+
stream: match.defaultUri,
|
|
38007
|
+
google: candidate.google
|
|
38008
|
+
});
|
|
38009
|
+
}
|
|
38010
|
+
if (found.size) return [...found.values()].slice(0, TIED);
|
|
38011
|
+
}
|
|
38012
|
+
return `no web stream matches ${site} in ${order.length < candidates.length ? `the first ${order.length} of ${candidates.length} properties, named like the site first` : `${candidates.length} ${candidates.length === 1 ? "property" : "properties"}`}${failures ? ` (${failures} could not be read)` : ""}.`;
|
|
38013
|
+
}
|
|
38014
|
+
const filter = (fieldName, value, matchType = "EXACT") => ({ filter: {
|
|
38015
|
+
fieldName,
|
|
38016
|
+
stringFilter: {
|
|
38017
|
+
matchType,
|
|
38018
|
+
value
|
|
38019
|
+
}
|
|
38020
|
+
} });
|
|
38021
|
+
async function totals$1(found, access, range, before, pages) {
|
|
38022
|
+
const read = (dimensions, filters) => runAnalyticsReport(found.google, found.property, {
|
|
38023
|
+
dateRanges: [{
|
|
38024
|
+
...range,
|
|
38025
|
+
name: "now"
|
|
38026
|
+
}, {
|
|
38027
|
+
...before,
|
|
38028
|
+
name: "before"
|
|
38029
|
+
}],
|
|
38030
|
+
dimensions: dimensions.map((name) => ({ name })),
|
|
38031
|
+
metrics: [{ name: "sessions" }, { name: "keyEvents" }],
|
|
38032
|
+
...filters.length ? { dimensionFilter: { andGroup: { expressions: filters } } } : {},
|
|
38033
|
+
limit: 100
|
|
38034
|
+
}, { signIn: access.signIn });
|
|
38035
|
+
const onPages = pages ? [filter("landingPage", pages, "CONTAINS")] : [];
|
|
38036
|
+
const [channels, google] = await Promise.all([read(["sessionDefaultChannelGroup"], onPages), read([], [
|
|
38037
|
+
...onPages,
|
|
38038
|
+
filter("sessionDefaultChannelGroup", "Organic Search"),
|
|
38039
|
+
filter("sessionSource", "google")
|
|
38040
|
+
])]);
|
|
38041
|
+
const sum = () => ({
|
|
38042
|
+
google: {
|
|
38043
|
+
sessions: 0,
|
|
38044
|
+
keyEvents: 0
|
|
38045
|
+
},
|
|
38046
|
+
organic: {
|
|
38047
|
+
sessions: 0,
|
|
38048
|
+
keyEvents: 0
|
|
38049
|
+
},
|
|
38050
|
+
all: {
|
|
38051
|
+
sessions: 0,
|
|
38052
|
+
keyEvents: 0
|
|
38053
|
+
}
|
|
38054
|
+
});
|
|
38055
|
+
const periods = {
|
|
38056
|
+
now: sum(),
|
|
38057
|
+
before: sum()
|
|
38058
|
+
};
|
|
38059
|
+
const each = (data, add) => {
|
|
38060
|
+
const periodAt = (data.dimensionHeaders?.map((header) => header.name) ?? []).indexOf("dateRange");
|
|
38061
|
+
for (const row of data.rows ?? []) {
|
|
38062
|
+
const values = row.dimensionValues ?? [];
|
|
38063
|
+
add((periodAt >= 0 ? values[periodAt] : values.at(-1))?.value === "before" ? periods.before : periods.now, values[0]?.value ?? "", {
|
|
38064
|
+
sessions: Number(row.metricValues?.[0]?.value ?? 0),
|
|
38065
|
+
keyEvents: Number(row.metricValues?.[1]?.value ?? 0)
|
|
38066
|
+
});
|
|
38067
|
+
}
|
|
38068
|
+
};
|
|
38069
|
+
const plus = (into, row) => {
|
|
38070
|
+
into.sessions += row.sessions;
|
|
38071
|
+
into.keyEvents += row.keyEvents;
|
|
38072
|
+
};
|
|
38073
|
+
each(channels, (period, channel, row) => {
|
|
38074
|
+
plus(period.all, row);
|
|
38075
|
+
if (channel === "Organic Search") plus(period.organic, row);
|
|
38076
|
+
});
|
|
38077
|
+
each(google, (period, _, row) => plus(period.google, row));
|
|
38078
|
+
const data = { metadata: {
|
|
38079
|
+
timeZone: channels.metadata?.timeZone,
|
|
38080
|
+
subjectToThresholding: channels.metadata?.subjectToThresholding || google.metadata?.subjectToThresholding,
|
|
38081
|
+
dataLossFromOtherRow: channels.metadata?.dataLossFromOtherRow || google.metadata?.dataLossFromOtherRow
|
|
38082
|
+
} };
|
|
38083
|
+
const notes = [];
|
|
38084
|
+
if (data.metadata?.timeZone && data.metadata.timeZone !== "America/Los_Angeles") notes.push(`Google Analytics days use the property's time zone, ${data.metadata.timeZone}; Search Console days use Pacific time.`);
|
|
38085
|
+
if (data.metadata?.subjectToThresholding) notes.push("Google Analytics applied thresholding; small rows can be hidden.");
|
|
38086
|
+
if (data.metadata?.dataLossFromOtherRow) notes.push("Google Analytics grouped some rows into (other).");
|
|
38087
|
+
return {
|
|
38088
|
+
property: found.property,
|
|
38089
|
+
propertyName: found.name,
|
|
38090
|
+
webStream: found.stream,
|
|
38091
|
+
...pages ? { landingPageContains: pages } : {},
|
|
38092
|
+
organicSearchFromGoogle: {
|
|
38093
|
+
sessions: periods.now.google.sessions,
|
|
38094
|
+
previousSessions: periods.before.google.sessions,
|
|
38095
|
+
sessionsChangePct: pct(periods.now.google.sessions, periods.before.google.sessions)
|
|
38096
|
+
},
|
|
38097
|
+
organicSearch: {
|
|
38098
|
+
sessions: periods.now.organic.sessions,
|
|
38099
|
+
previousSessions: periods.before.organic.sessions,
|
|
38100
|
+
sessionsChangePct: pct(periods.now.organic.sessions, periods.before.organic.sessions),
|
|
38101
|
+
keyEvents: periods.now.organic.keyEvents,
|
|
38102
|
+
previousKeyEvents: periods.before.organic.keyEvents
|
|
38103
|
+
},
|
|
38104
|
+
allChannels: {
|
|
38105
|
+
sessions: periods.now.all.sessions,
|
|
38106
|
+
previousSessions: periods.before.all.sessions,
|
|
38107
|
+
sessionsChangePct: pct(periods.now.all.sessions, periods.before.all.sessions)
|
|
38108
|
+
},
|
|
38109
|
+
notes
|
|
38110
|
+
};
|
|
38111
|
+
}
|
|
38112
|
+
const change$1 = (value) => value === null ? "n/a" : `${value > 0 ? "+" : ""}${value}%`;
|
|
38113
|
+
async function analyticsBeside(access, site, range, before, pages) {
|
|
38114
|
+
if (!access) return null;
|
|
38115
|
+
try {
|
|
38116
|
+
const found = await findProperty(access, site);
|
|
38117
|
+
if (typeof found === "string") return {
|
|
38118
|
+
text: `Google Analytics: ${found}`,
|
|
38119
|
+
data: { error: found }
|
|
38120
|
+
};
|
|
38121
|
+
const read = await Promise.all(found.map((item) => totals$1(item, access, range, before, pages).catch((error) => error instanceof Error ? error : new Error(String(error)))));
|
|
38122
|
+
const data = [...read.filter((item) => !(item instanceof Error))].sort((a, b) => b.allChannels.sessions - a.allChannels.sessions)[0];
|
|
38123
|
+
if (!data) throw read[0];
|
|
38124
|
+
const others = found.filter((item) => item.property !== data.property).map((item) => `${item.name} (${item.property})`);
|
|
38125
|
+
const google = data.organicSearchFromGoogle;
|
|
38126
|
+
const organic = data.organicSearch;
|
|
38127
|
+
const all = data.allChannels;
|
|
38128
|
+
const keyEvents = organic.keyEvents || organic.previousKeyEvents ? `, Organic Search key events ${organic.keyEvents} (was ${organic.previousKeyEvents})` : "";
|
|
38129
|
+
return {
|
|
38130
|
+
text: [
|
|
38131
|
+
`Google Analytics, property ${data.propertyName} (${data.property}), matched by its web stream ${data.webStream}${pages ? `, landing pages whose path contains "${pages}"` : ""}: Organic Search sessions from Google ${google.sessions} (was ${google.previousSessions}, ${change$1(google.sessionsChangePct)}), from every search engine ${organic.sessions} (was ${organic.previousSessions}, ${change$1(organic.sessionsChangePct)})${keyEvents}, sessions from all channels ${all.sessions} (was ${all.previousSessions}, ${change$1(all.sessionsChangePct)}).`,
|
|
38132
|
+
"Sessions are Google Analytics’ own count, not Search Console clicks.",
|
|
38133
|
+
...data.notes,
|
|
38134
|
+
...others.length ? [`Other properties with a matching web stream, with fewer sessions: ${others.join(", ")}.`] : []
|
|
38135
|
+
].join(" "),
|
|
38136
|
+
data: {
|
|
38137
|
+
...data,
|
|
38138
|
+
...others.length ? { otherMatches: others } : {}
|
|
38139
|
+
}
|
|
38140
|
+
};
|
|
38141
|
+
} catch (error) {
|
|
38142
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
38143
|
+
return {
|
|
38144
|
+
text: `Google Analytics: could not be read. ${message}`,
|
|
38145
|
+
data: { error: message }
|
|
38146
|
+
};
|
|
38147
|
+
}
|
|
38148
|
+
}
|
|
38149
|
+
//#endregion
|
|
37588
38150
|
//#region packages/core/dist/commands/bing.js
|
|
37589
38151
|
const BING_LIMITS = ["Bing and Google count clicks, impressions and positions their own way. Their numbers sit side by side here and are never added together.", "Bing lists only its top queries and pages, grouped by week. A query or page missing from Bing’s list is unknown, not zero."];
|
|
37590
38152
|
const CLI_LINK = "If you have a shell, link the CLI to the same account once: run `npx -y seo login`, show the person the link it prints, and after they approve it run `npx -y seo login` again. `npx -y seo` commands then use the same Google accounts, Bing and keys.";
|
|
@@ -38146,7 +38708,7 @@ function temporaryRedirectNote(pages) {
|
|
|
38146
38708
|
//#endregion
|
|
38147
38709
|
//#region packages/core/dist/commands/report.js
|
|
38148
38710
|
const SECTIONS = {
|
|
38149
|
-
overview: "Site totals (clicks, impressions, CTR, position) against the previous period, by device, with the top pages and queries and their change.",
|
|
38711
|
+
overview: "Site totals (clicks, impressions, CTR, position) against the previous period, by device, with the top pages and queries and their change, and Google Analytics sessions for the same dates from the property whose web stream matches the site.",
|
|
38150
38712
|
"quick-wins": "Query and page pairs at average positions 4 to 10 with a click-through rate below the site’s own curve for that position.",
|
|
38151
38713
|
"low-ctr": "Query and page pairs at average positions 1 to 4 with a click-through rate below the site’s own curve: titles and snippets to review with the person.",
|
|
38152
38714
|
"striking-distance": "Queries just outside the top positions, with impressions, for pages that already rank.",
|
|
@@ -38453,10 +39015,14 @@ async function allSections(input, deps) {
|
|
|
38453
39015
|
const origin = originOf(input.site);
|
|
38454
39016
|
const bing = once(() => bingFor(deps, input.site));
|
|
38455
39017
|
const ids = SECTION_IDS.filter((id) => id !== "bing" || deps.bing);
|
|
39018
|
+
const range = dateWindow(input, DEFAULT_DAYS.overview);
|
|
39019
|
+
const analytics = analyticsBeside(deps.analytics, input.site, range, earlierRange(range));
|
|
38456
39020
|
const [overview, ...rest] = await Promise.all(ids.map((id) => build(read, input.site, id, dateWindow(input, DEFAULT_DAYS[id]), now, bing).catch((error) => error instanceof Error ? error : new Error(String(error)))));
|
|
38457
39021
|
if (!overview || overview instanceof Error) throw overview ?? /* @__PURE__ */ new Error("The overview did not run.");
|
|
38458
39022
|
const parts = [overviewText(overview, `Report on ${input.site}`, origin, 5)];
|
|
38459
39023
|
parts.push(deps.bing ? await enginesLine(overview, bing) : NO_BING_HERE);
|
|
39024
|
+
const ga = await analytics;
|
|
39025
|
+
if (ga) parts.push(ga.text);
|
|
38460
39026
|
const brand = rest.find((run) => !(run instanceof Error) && brandNote(run) !== null);
|
|
38461
39027
|
if (brand) parts.push(brandNote(brand));
|
|
38462
39028
|
const sections = {};
|
|
@@ -38486,6 +39052,7 @@ async function allSections(input, deps) {
|
|
|
38486
39052
|
data: {
|
|
38487
39053
|
site: input.site,
|
|
38488
39054
|
overview,
|
|
39055
|
+
...ga ? { analytics: ga.data } : {},
|
|
38489
39056
|
sections
|
|
38490
39057
|
}
|
|
38491
39058
|
};
|
|
@@ -38499,14 +39066,25 @@ async function oneSection(input, deps) {
|
|
|
38499
39066
|
const bing = once(() => bingFor(deps, site));
|
|
38500
39067
|
let report = await build(sharedReader(deps.searchAnalytics), site, section, range, now, bing);
|
|
38501
39068
|
const next = nextFor(section, deps.call);
|
|
38502
|
-
if (section === "overview"
|
|
38503
|
-
|
|
38504
|
-
|
|
38505
|
-
|
|
38506
|
-
|
|
38507
|
-
|
|
38508
|
-
|
|
38509
|
-
|
|
39069
|
+
if (section === "overview") {
|
|
39070
|
+
const ga = await analyticsBeside(deps.analytics, site, range, earlierRange(range));
|
|
39071
|
+
if (ga) report.analytics = ga.data;
|
|
39072
|
+
if (detail !== "full") return {
|
|
39073
|
+
text: [
|
|
39074
|
+
overviewText(report, `overview of ${site}`, origin, 10),
|
|
39075
|
+
deps.bing ? await enginesLine(report, bing) : NO_BING_HERE,
|
|
39076
|
+
ga?.text,
|
|
39077
|
+
`More: ${deps.call("report", {
|
|
39078
|
+
site,
|
|
39079
|
+
section,
|
|
39080
|
+
detail: "full"
|
|
39081
|
+
})} for every row.`,
|
|
39082
|
+
next,
|
|
39083
|
+
FACTS_ONLY
|
|
39084
|
+
].filter(Boolean).join("\n\n"),
|
|
39085
|
+
data: report
|
|
39086
|
+
};
|
|
39087
|
+
}
|
|
38510
39088
|
let checked = [];
|
|
38511
39089
|
const merged = [];
|
|
38512
39090
|
if (PAGE_CHECKS.has(section)) {
|
|
@@ -38586,10 +39164,32 @@ async function research(provider, headers, input) {
|
|
|
38586
39164
|
function countryName(code) {
|
|
38587
39165
|
return new Intl.DisplayNames("en", { type: "region" }).of(code.toUpperCase()) ?? "United States";
|
|
38588
39166
|
}
|
|
39167
|
+
function mainLanguage(country) {
|
|
39168
|
+
try {
|
|
39169
|
+
return new Intl.Locale(`und-${country.toUpperCase()}`).maximize().language;
|
|
39170
|
+
} catch {
|
|
39171
|
+
return "en";
|
|
39172
|
+
}
|
|
39173
|
+
}
|
|
39174
|
+
const LABS_LANGUAGES = new Map(`
|
|
39175
|
+
ae:ar al:sq am:hy ao:pt ar:es at:de au:en az:az ba:bs bd:bn be:nl bf:fr
|
|
39176
|
+
bg:bg bh:ar bo:es br:pt ca:en ch:de ci:fr cl:es cm:fr co:es cr:es cy:el
|
|
39177
|
+
cz:cs de:de dk:da dz:ar ec:es ee:et eg:ar es:es fi:fi fr:fr gb:en gh:en
|
|
39178
|
+
gr:el gt:es hk:en hr:hr hu:hu id:id ie:en il:he in:en it:it jo:ar jp:ja
|
|
39179
|
+
ke:en kh:en kr:ko kz:ru lk:en lt:lt lv:lv ma:ar mc:fr md:ro mk:mk mm:en
|
|
39180
|
+
mt:en mx:es my:en ng:en ni:es nl:nl no:nb nz:en pa:es pe:es ph:en pk:en
|
|
39181
|
+
pl:pl pt:pt py:es ro:ro rs:sr sa:ar se:sv sg:en si:sl sk:sk sn:fr sv:es
|
|
39182
|
+
th:th tn:ar tr:tr tw:zh-TW ua:uk us:en uy:es ve:es vn:vi za:en
|
|
39183
|
+
`.trim().split(/\s+/).map((pair) => pair.split(":")));
|
|
39184
|
+
function dataForSeoLanguage(country) {
|
|
39185
|
+
return LABS_LANGUAGES.get(country.toLowerCase()) ?? null;
|
|
39186
|
+
}
|
|
38589
39187
|
async function dataForSeo([login, password], input) {
|
|
39188
|
+
const language = dataForSeoLanguage(input.country);
|
|
39189
|
+
if (!language && input.action !== "serp" && input.action !== "backlinks") throw new CommandError("INVALID_INPUT", `DataForSEO has no ${input.action} data for country "${input.country}".`);
|
|
38590
39190
|
const market = {
|
|
38591
39191
|
location_name: countryName(input.country),
|
|
38592
|
-
language_code: input.
|
|
39192
|
+
language_code: language ?? mainLanguage(input.country)
|
|
38593
39193
|
};
|
|
38594
39194
|
const post = async (path, task) => {
|
|
38595
39195
|
const result = (await (await call("dataforseo", new Request(`https://api.dataforseo.com/${path}`, {
|
|
@@ -38798,12 +39398,12 @@ async function runResearch(input, keys, options) {
|
|
|
38798
39398
|
const country = input.country ?? "us";
|
|
38799
39399
|
const target = input.target ?? "";
|
|
38800
39400
|
const offset = input.offset ?? 0;
|
|
39401
|
+
const language = input.provider === "dataforseo" ? dataForSeoLanguage(country) : null;
|
|
38801
39402
|
const found = await research(input.provider, keys, {
|
|
38802
39403
|
action: input.action,
|
|
38803
39404
|
keywords: input.keywords ?? [],
|
|
38804
39405
|
target,
|
|
38805
39406
|
country,
|
|
38806
|
-
language: "en",
|
|
38807
39407
|
limit: Math.min(1e3, Math.max(100, offset + 50))
|
|
38808
39408
|
});
|
|
38809
39409
|
const { rows, note } = input.action === "competitors" ? realCompetitors(found, target) : {
|
|
@@ -38816,7 +39416,7 @@ async function runResearch(input, keys, options) {
|
|
|
38816
39416
|
const hasTargetDr = targetDr !== void 0 && targetDr !== null;
|
|
38817
39417
|
return {
|
|
38818
39418
|
text: rowsText({
|
|
38819
|
-
title: `${input.provider} ${input.action}${target ? ` for ${target}` : ""}, ${country}.`,
|
|
39419
|
+
title: `${input.provider} ${input.action}${target ? ` for ${target}` : ""}, ${country}${language ? `, language ${language}` : ""}.`,
|
|
38820
39420
|
rows: rated.rows,
|
|
38821
39421
|
offset,
|
|
38822
39422
|
limit: 50,
|
|
@@ -38834,6 +39434,7 @@ async function runResearch(input, keys, options) {
|
|
|
38834
39434
|
action: input.action,
|
|
38835
39435
|
target: target || null,
|
|
38836
39436
|
country,
|
|
39437
|
+
language,
|
|
38837
39438
|
targetDomainRating: hasTargetDr ? targetDr : null,
|
|
38838
39439
|
attribution: rated.added || hasTargetDr ? DR_ATTRIBUTION : null,
|
|
38839
39440
|
leftOut: note,
|
|
@@ -38973,19 +39574,25 @@ async function runCompare(input, deps) {
|
|
|
38973
39574
|
startDate: shiftIsoDate(input.date, -days),
|
|
38974
39575
|
endDate: shiftIsoDate(input.date, -1)
|
|
38975
39576
|
};
|
|
38976
|
-
const [report, updates] = await Promise.all([
|
|
38977
|
-
|
|
38978
|
-
|
|
38979
|
-
|
|
39577
|
+
const [report, updates, analytics] = await Promise.all([
|
|
39578
|
+
siteOverview(sharedReader(deps.searchAnalytics), input.site, after, before, input.pages),
|
|
39579
|
+
googleUpdates({
|
|
39580
|
+
start: before.startDate,
|
|
39581
|
+
end: after.endDate
|
|
39582
|
+
}),
|
|
39583
|
+
analyticsBeside(deps.analytics, input.site, after, before, input.pages)
|
|
39584
|
+
]);
|
|
38980
39585
|
return {
|
|
38981
39586
|
text: [
|
|
38982
39587
|
overviewText(report, `Compare on ${input.site}: ${days} days after ${input.date}`, originOf(input.site), 10),
|
|
39588
|
+
analytics?.text,
|
|
38983
39589
|
listOfUpdates(updates),
|
|
38984
39590
|
`Next: ${deps.call("page")} for a page that moved.`,
|
|
38985
39591
|
FACTS_ONLY
|
|
38986
39592
|
].filter(Boolean).join("\n\n"),
|
|
38987
39593
|
data: {
|
|
38988
39594
|
...report,
|
|
39595
|
+
...analytics ? { analytics: analytics.data } : {},
|
|
38989
39596
|
googleUpdates: updates
|
|
38990
39597
|
}
|
|
38991
39598
|
};
|
|
@@ -39202,7 +39809,7 @@ const SHARED_TOOLS = [
|
|
|
39202
39809
|
name: "seo_compare",
|
|
39203
39810
|
command: "compare",
|
|
39204
39811
|
title: "Before and after a date",
|
|
39205
|
-
description: "Compares equal periods before and after a date, such as a site change or a Google update: totals, devices, and the pages and queries that moved,
|
|
39812
|
+
description: "Compares equal periods before and after a date, such as a site change or a Google update: totals, devices, and the pages and queries that moved, Google Analytics sessions from the property whose web stream matches the site, and Google updates in the window. Optional pages narrows it to URLs that contain that text, such as /blog/.",
|
|
39206
39813
|
input: z.object({
|
|
39207
39814
|
site,
|
|
39208
39815
|
date: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).describe("YYYY-MM-DD: the first day after the change."),
|
|
@@ -39330,7 +39937,7 @@ const SKILL_RESOURCE = {
|
|
|
39330
39937
|
name: "seo-skill",
|
|
39331
39938
|
uri: "skill://seo/SKILL.md",
|
|
39332
39939
|
title: "SEO Skill",
|
|
39333
|
-
description: "How to use these tools:
|
|
39940
|
+
description: "How to use these tools: what each command returns and how to reach it.",
|
|
39334
39941
|
mimeType: "text/markdown"
|
|
39335
39942
|
};
|
|
39336
39943
|
//#endregion
|
|
@@ -39527,7 +40134,11 @@ function nodeCommandDeps(options) {
|
|
|
39527
40134
|
pageMarkdown: async (url) => extractMainMarkdown(await fetchPage(url, { refresh: options.refresh })),
|
|
39528
40135
|
call: options.call,
|
|
39529
40136
|
bing: localBing,
|
|
39530
|
-
bingConnect: LOCAL_BING_CONNECT
|
|
40137
|
+
bingConnect: LOCAL_BING_CONNECT,
|
|
40138
|
+
analytics: {
|
|
40139
|
+
accounts: localAccounts,
|
|
40140
|
+
signIn: LOCAL_SIGN_IN.signIn
|
|
40141
|
+
}
|
|
39531
40142
|
};
|
|
39532
40143
|
}
|
|
39533
40144
|
const LOCAL_BING_CONNECT = "Connect Bing on your seoskill.dev account and run `seo login`, or run `seo providers bing connect` with an API key from Bing Webmaster Tools (Settings, API access).";
|