seo 0.7.1 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -345
- package/dist/cli.js +84 -84
- package/dist/cli.js.map +1 -1
- package/dist/{dist-S2OSIAML.js → dist-B5I5uxdM.js} +88 -85
- package/dist/dist-B5I5uxdM.js.map +1 -0
- package/dist/{dist-aHJsXHVF.js → dist-ulug7z2k.js} +1 -1
- package/dist/index.d.ts +941 -220
- package/dist/index.js +631 -298
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +952 -281
- package/dist/mcp.js.map +1 -1
- package/dist/{package-BaQwwfSG.js → package-BjCto4m4.js} +2 -2
- package/dist/{package-BaQwwfSG.js.map → package-BjCto4m4.js.map} +1 -1
- package/evals/crawl.json +2 -3
- package/evals/internal-links.json +2 -3
- package/evals/low-ctr.json +1 -2
- package/evals/page.json +2 -3
- package/evals/quick-wins.json +2 -3
- package/evals/seo.json +2 -2
- package/package.json +1 -1
- package/skills/README.md +4 -4
- package/skills/seo/SKILL.md +22 -95
- package/dist/dist-S2OSIAML.js.map +0 -1
package/dist/mcp.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { i as __toESM, n as load, r as __commonJSMin, t as version } from "./package-
|
|
2
|
+
import { i as __toESM, n as load, r as __commonJSMin, t as version } from "./package-BjCto4m4.js";
|
|
3
3
|
import { createHash, randomUUID, sign } from "node:crypto";
|
|
4
4
|
import { ZodOptional, z } from "zod";
|
|
5
5
|
import { arch, homedir, platform, totalmem } from "node:os";
|
|
@@ -13036,6 +13036,98 @@ const RULE_DEFINITIONS = [
|
|
|
13036
13036
|
howToFix: "Add twitter:card and matching metadata only when you need a specific X preview. Keep Open Graph metadata accurate either way.",
|
|
13037
13037
|
impactIfIgnored: "X may use fallback metadata or a simple link preview. Organic rankings are not affected.",
|
|
13038
13038
|
howToVerify: "Re-run the crawl and confirm twitterCard is present if a specific X preview is required."
|
|
13039
|
+
},
|
|
13040
|
+
{
|
|
13041
|
+
id: "ai_search_bot_blocked",
|
|
13042
|
+
title: "robots.txt blocks AI search bots",
|
|
13043
|
+
category: "agents",
|
|
13044
|
+
defaultSeverity: "medium",
|
|
13045
|
+
whyItMatters: "AI search bots fetch pages to answer a person's question and cite the source. A blocked bot cannot read the site, so its answers cannot quote or link to it. Training bots are a separate choice and are listed apart.",
|
|
13046
|
+
howToFix: "Allow the AI search and user bots in robots.txt if the site should appear in AI answers. Block training bots separately if the person wants that.",
|
|
13047
|
+
impactIfIgnored: "ChatGPT, Claude, Perplexity and similar tools can leave the site out of answers and citations.",
|
|
13048
|
+
howToVerify: "Re-run the crawl and confirm the search bots in ai.robotsTxt.botAccess show allowed: true.",
|
|
13049
|
+
review: {
|
|
13050
|
+
question: "Is the site meant to be left out of AI search answers?",
|
|
13051
|
+
changeOnlyIf: "Change robots.txt when the person wants the site to appear in AI answers.",
|
|
13052
|
+
ifNotNeeded: "Record that blocking AI search bots is a deliberate choice for this site."
|
|
13053
|
+
},
|
|
13054
|
+
agentHints: { evidenceFields: ["report.ai.robotsTxt.botAccess"] }
|
|
13055
|
+
},
|
|
13056
|
+
{
|
|
13057
|
+
id: "llms_txt_missing",
|
|
13058
|
+
title: "No llms.txt",
|
|
13059
|
+
category: "agents",
|
|
13060
|
+
defaultSeverity: "low",
|
|
13061
|
+
whyItMatters: "llms.txt gives agents a short summary of the site and links to its main pages, ideally their Markdown copies. Search engines do not use it for ranking.",
|
|
13062
|
+
howToFix: "Publish /llms.txt as plain text with a one-line summary and sections of links to the main pages.",
|
|
13063
|
+
impactIfIgnored: "Agents must find the main pages by crawling, which costs them more requests and tokens.",
|
|
13064
|
+
howToVerify: "Request /llms.txt and confirm a 200 plain text response.",
|
|
13065
|
+
review: {
|
|
13066
|
+
question: "Should agents get a reading list for this site?",
|
|
13067
|
+
changeOnlyIf: "Add it when agents are expected to read or use the site.",
|
|
13068
|
+
ifNotNeeded: "Record that the site has no llms.txt by choice."
|
|
13069
|
+
},
|
|
13070
|
+
agentHints: { evidenceFields: ["report.ai.llmsTxt"] }
|
|
13071
|
+
},
|
|
13072
|
+
{
|
|
13073
|
+
id: "llms_txt_broken_link",
|
|
13074
|
+
title: "llms.txt links to a page that does not load",
|
|
13075
|
+
category: "agents",
|
|
13076
|
+
defaultSeverity: "low",
|
|
13077
|
+
whyItMatters: "An agent that follows a link in llms.txt and gets an error loses its reading list entry.",
|
|
13078
|
+
howToFix: "Update or remove each broken link in llms.txt, or restore the page it points to.",
|
|
13079
|
+
impactIfIgnored: "Agents waste requests and can miss the page.",
|
|
13080
|
+
howToVerify: "Re-run the crawl and confirm ai.llmsTxt.links.broken is empty.",
|
|
13081
|
+
agentHints: { evidenceFields: ["report.ai.llmsTxt.links"] }
|
|
13082
|
+
},
|
|
13083
|
+
{
|
|
13084
|
+
id: "markdown_missing",
|
|
13085
|
+
title: "No Markdown copy of the pages",
|
|
13086
|
+
category: "agents",
|
|
13087
|
+
defaultSeverity: "low",
|
|
13088
|
+
whyItMatters: "A Markdown copy gives agents the page content without layout, scripts or navigation, at a fraction of the tokens. Neither a request with Accept: text/markdown nor a rel=\"alternate\" link returned one. Search engines do not use it for ranking.",
|
|
13089
|
+
howToFix: "Serve a Markdown copy of each page, linked from the page with <link rel=\"alternate\" type=\"text/markdown\">, and return it for Accept: text/markdown with Vary: Accept.",
|
|
13090
|
+
impactIfIgnored: "Agents read the full HTML, which costs more tokens and can include layout text.",
|
|
13091
|
+
howToVerify: "Request a page with Accept: text/markdown and confirm a text/markdown response.",
|
|
13092
|
+
review: {
|
|
13093
|
+
question: "Should agents get a Markdown copy of the pages?",
|
|
13094
|
+
changeOnlyIf: "Add it when agents are expected to read the site.",
|
|
13095
|
+
ifNotNeeded: "Record that the site serves HTML only by choice."
|
|
13096
|
+
},
|
|
13097
|
+
agentHints: { evidenceFields: ["report.ai.markdownNegotiation", "report.ai.markdownCopies"] }
|
|
13098
|
+
},
|
|
13099
|
+
{
|
|
13100
|
+
id: "markdown_alternate_missing",
|
|
13101
|
+
title: "Page has no link to its Markdown copy",
|
|
13102
|
+
category: "agents",
|
|
13103
|
+
defaultSeverity: "low",
|
|
13104
|
+
whyItMatters: "Other pages on the site link to a Markdown copy, but this one does not, so agents cannot find a copy for it.",
|
|
13105
|
+
howToFix: "Add <link rel=\"alternate\" type=\"text/markdown\" href=\"...\"> to the page head, pointing at its Markdown copy.",
|
|
13106
|
+
impactIfIgnored: "Agents read the full HTML for this page.",
|
|
13107
|
+
howToVerify: "Re-run the crawl and confirm the page lists a Markdown alternate.",
|
|
13108
|
+
agentHints: { evidenceFields: ["page.markdownAlternates", "report.ai.markdownCopies"] }
|
|
13109
|
+
},
|
|
13110
|
+
{
|
|
13111
|
+
id: "markdown_alternate_on_private_page",
|
|
13112
|
+
title: "Signed-in page links to a Markdown copy",
|
|
13113
|
+
category: "agents",
|
|
13114
|
+
defaultSeverity: "medium",
|
|
13115
|
+
whyItMatters: "Signed-in pages should have no Markdown copy. This page needs a sign-in, yet it links to one. A copy can show agents content that a signed-out visitor cannot see, or fail and waste their requests.",
|
|
13116
|
+
howToFix: "Remove the rel=\"alternate\" Markdown link, the Link header and any Markdown copy for signed-in routes. Public pages keep theirs, including noindex pages such as privacy and terms.",
|
|
13117
|
+
impactIfIgnored: "Agents can request a copy that fails, or read content that was meant to stay behind a sign-in.",
|
|
13118
|
+
howToVerify: "Re-run the crawl and confirm the page no longer lists a Markdown alternate.",
|
|
13119
|
+
agentHints: { evidenceFields: ["page.markdownAlternates", "report.ai.markdownCopies.broken"] }
|
|
13120
|
+
},
|
|
13121
|
+
{
|
|
13122
|
+
id: "markdown_copy_broken",
|
|
13123
|
+
title: "Markdown copy does not load",
|
|
13124
|
+
category: "agents",
|
|
13125
|
+
defaultSeverity: "medium",
|
|
13126
|
+
whyItMatters: "The page links to a Markdown copy that returns an error or HTML, so an agent that follows the link gets nothing useful. If the page needs a sign-in, remove the link instead.",
|
|
13127
|
+
howToFix: "Make the linked URL return the page as text/markdown, or fix the link.",
|
|
13128
|
+
impactIfIgnored: "Agents get an error or the HTML page instead of the copy.",
|
|
13129
|
+
howToVerify: "Request the Markdown URL and confirm a 200 text/markdown response.",
|
|
13130
|
+
agentHints: { evidenceFields: ["report.ai.markdownCopies.broken"] }
|
|
13039
13131
|
}
|
|
13040
13132
|
];
|
|
13041
13133
|
const RULE_RECOMMENDATIONS = {
|
|
@@ -13064,7 +13156,10 @@ const RULE_RECOMMENDATIONS = {
|
|
|
13064
13156
|
og_title_missing: "review",
|
|
13065
13157
|
og_description_missing: "review",
|
|
13066
13158
|
og_image_missing: "review",
|
|
13067
|
-
twitter_card_missing: "review"
|
|
13159
|
+
twitter_card_missing: "review",
|
|
13160
|
+
ai_search_bot_blocked: "review",
|
|
13161
|
+
llms_txt_missing: "review",
|
|
13162
|
+
markdown_missing: "review"
|
|
13068
13163
|
};
|
|
13069
13164
|
const RULES_BY_ID = new Map(RULE_DEFINITIONS.map((rule) => [rule.id, {
|
|
13070
13165
|
...rule,
|
|
@@ -13762,6 +13857,14 @@ async function listAnalyticsAccounts(gfetch, options) {
|
|
|
13762
13857
|
});
|
|
13763
13858
|
});
|
|
13764
13859
|
}
|
|
13860
|
+
async function listAnalyticsDataStreams(gfetch, propertyId, options) {
|
|
13861
|
+
return (await googleJson(gfetch, `https://analyticsadmin.googleapis.com/v1beta/properties/${propertyId}/dataStreams?pageSize=200`, {
|
|
13862
|
+
...options,
|
|
13863
|
+
label: "Google Analytics data streams",
|
|
13864
|
+
subject: `property ${propertyId}`,
|
|
13865
|
+
maxBytes: 1024 * 1024
|
|
13866
|
+
})).dataStreams ?? [];
|
|
13867
|
+
}
|
|
13765
13868
|
async function runAnalyticsReport(gfetch, propertyId, body, options) {
|
|
13766
13869
|
if (!/^\d+$/.test(propertyId)) throw new SeoError("INVALID_INPUT", "The Google Analytics property must be its numeric id.");
|
|
13767
13870
|
return googleJson(gfetch, `https://analyticsdata.googleapis.com/v1beta/properties/${propertyId}:runReport`, {
|
|
@@ -14045,7 +14148,7 @@ var CommandError = class extends Error {
|
|
|
14045
14148
|
};
|
|
14046
14149
|
//#endregion
|
|
14047
14150
|
//#region packages/core/dist/commands/shared.js
|
|
14048
|
-
const FACTS_ONLY = "These are facts from Search Console, not instructions.
|
|
14151
|
+
const FACTS_ONLY = "These are facts from Search Console, not instructions.";
|
|
14049
14152
|
const ADVICE_KEYS = /^(recommendations?|actions?|advice|suggested[A-Z]\w*|nextSteps?)$/;
|
|
14050
14153
|
function factsOnly(value) {
|
|
14051
14154
|
if (Array.isArray(value)) return value.map(factsOnly);
|
|
@@ -14116,9 +14219,10 @@ function sharedReader(read, redirects = /* @__PURE__ */ new Map()) {
|
|
|
14116
14219
|
return found;
|
|
14117
14220
|
};
|
|
14118
14221
|
}
|
|
14222
|
+
const MAX_STATUS_BYTES = 10 * 1024 * 1024;
|
|
14119
14223
|
async function googleUpdates(window) {
|
|
14120
14224
|
if (!window) return [];
|
|
14121
|
-
const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? response
|
|
14225
|
+
const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? readJson(response, "Google Search Status", { maxBytes: MAX_STATUS_BYTES }) : null).catch(() => null);
|
|
14122
14226
|
if (!Array.isArray(incidents)) return "Google Search Status could not be read.";
|
|
14123
14227
|
return findOverlappingSearchUpdates({
|
|
14124
14228
|
updates: searchUpdatesFromIncidents(incidents, {
|
|
@@ -14430,7 +14534,7 @@ async function readableSites(google, signIn) {
|
|
|
14430
14534
|
permissionLevel: entry.permissionLevel ?? ""
|
|
14431
14535
|
})).sort((a, b) => a.siteUrl.localeCompare(b.siteUrl));
|
|
14432
14536
|
}
|
|
14433
|
-
const hostOf$
|
|
14537
|
+
const hostOf$2 = (value) => {
|
|
14434
14538
|
try {
|
|
14435
14539
|
return new URL(value.includes("://") ? value : `https://${value.replace(/^sc-domain:/, "")}`).hostname.replace(/^www\./, "");
|
|
14436
14540
|
} catch {
|
|
@@ -14448,14 +14552,14 @@ async function runSites(input) {
|
|
|
14448
14552
|
};
|
|
14449
14553
|
})), input.bing ? listBingSites(input.bing).catch((error) => error instanceof Error ? error : new Error(String(error))) : Promise.resolve(null)]);
|
|
14450
14554
|
const bingSites = Array.isArray(bing) ? bing.filter((site) => site.verified) : [];
|
|
14451
|
-
const bingByHost = new Map(bingSites.map((site) => [hostOf$
|
|
14555
|
+
const bingByHost = new Map(bingSites.map((site) => [hostOf$2(site.url), site]));
|
|
14452
14556
|
const properties = google.flatMap((account) => account.properties instanceof Error ? [] : account.properties.map((property) => ({
|
|
14453
14557
|
...property,
|
|
14454
14558
|
...named ? { account: account.email ?? "" } : {},
|
|
14455
|
-
...Array.isArray(bing) ? { bing: bingByHost.get(hostOf$
|
|
14559
|
+
...Array.isArray(bing) ? { bing: bingByHost.get(hostOf$2(property.siteUrl))?.url ?? "" } : {}
|
|
14456
14560
|
})));
|
|
14457
|
-
const googleHosts = new Set(properties.map((property) => hostOf$
|
|
14458
|
-
const bingOnly = bingSites.filter((site) => !googleHosts.has(hostOf$
|
|
14561
|
+
const googleHosts = new Set(properties.map((property) => hostOf$2(property.siteUrl)));
|
|
14562
|
+
const bingOnly = bingSites.filter((site) => !googleHosts.has(hostOf$2(site.url)));
|
|
14459
14563
|
const failed = google.filter((account) => account.properties instanceof Error);
|
|
14460
14564
|
const analytics = google.flatMap((account) => (account.analytics ?? []).flatMap((summary) => summary.propertySummaries.map((property) => ({
|
|
14461
14565
|
property: property.property.replace("properties/", ""),
|
|
@@ -14767,7 +14871,7 @@ async function cloudCredentials(options = {}) {
|
|
|
14767
14871
|
return credentials;
|
|
14768
14872
|
}
|
|
14769
14873
|
function notSignedIn() {
|
|
14770
|
-
return "Not signed in. Run `seo login` to use your seoskill.dev account on this machine (it prints a link to approve), or `seo auth login` to sign in with Google on this machine only.";
|
|
14874
|
+
return "Not signed in. Run `seo login` to use your seoskill.dev account on this machine (it prints a link to approve), or `seo auth login` to sign in with Google on this machine only. `seo crawl --url <url>` works without sign-in.";
|
|
14771
14875
|
}
|
|
14772
14876
|
function tokenOf(item, what) {
|
|
14773
14877
|
if ("error" in item) throw new SeoError("AUTH_EXPIRED", `${what}: ${item.error}`);
|
|
@@ -21662,7 +21766,7 @@ function clientRenderedContentEvidence(page) {
|
|
|
21662
21766
|
}
|
|
21663
21767
|
};
|
|
21664
21768
|
}
|
|
21665
|
-
function issue(ruleId, page, detail, evidence) {
|
|
21769
|
+
function issue$1(ruleId, page, detail, evidence) {
|
|
21666
21770
|
const rule = explainRule(ruleId);
|
|
21667
21771
|
if (!rule) throw new Error(`Missing rule guidance for ${ruleId}`);
|
|
21668
21772
|
return {
|
|
@@ -21827,7 +21931,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
21827
21931
|
}
|
|
21828
21932
|
for (const page of pages) {
|
|
21829
21933
|
if (page.status === 0) {
|
|
21830
|
-
issues.push(issue("connection_error", page, page.error, {
|
|
21934
|
+
issues.push(issue$1("connection_error", page, page.error, {
|
|
21831
21935
|
status: page.status,
|
|
21832
21936
|
error: page.error
|
|
21833
21937
|
}));
|
|
@@ -21835,23 +21939,23 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
21835
21939
|
}
|
|
21836
21940
|
if (page.accessBlock) continue;
|
|
21837
21941
|
if (page.status >= 500) {
|
|
21838
|
-
issues.push(issue("server_error", page, String(page.status), { status: page.status }));
|
|
21839
|
-
if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue("broken_internal_link", page, String(page.status), {
|
|
21942
|
+
issues.push(issue$1("server_error", page, String(page.status), { status: page.status }));
|
|
21943
|
+
if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
21840
21944
|
status: page.status,
|
|
21841
21945
|
internalInlinkCount: page.internalInlinkCount
|
|
21842
21946
|
}));
|
|
21843
21947
|
continue;
|
|
21844
21948
|
}
|
|
21845
21949
|
if (page.status >= 400) {
|
|
21846
|
-
issues.push(issue("client_error", page, String(page.status), { status: page.status }));
|
|
21847
|
-
if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0) issues.push(issue("broken_internal_link", page, String(page.status), {
|
|
21950
|
+
issues.push(issue$1("client_error", page, String(page.status), { status: page.status }));
|
|
21951
|
+
if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
21848
21952
|
status: page.status,
|
|
21849
21953
|
internalInlinkCount: page.internalInlinkCount
|
|
21850
21954
|
}));
|
|
21851
21955
|
continue;
|
|
21852
21956
|
}
|
|
21853
21957
|
if (page.status >= 300) {
|
|
21854
|
-
issues.push(issue("redirected_url", page, `Status ${page.status}`, {
|
|
21958
|
+
issues.push(issue$1("redirected_url", page, `Status ${page.status}`, {
|
|
21855
21959
|
requestedUrl: page.url,
|
|
21856
21960
|
finalUrl: page.finalUrl,
|
|
21857
21961
|
status: page.status
|
|
@@ -21862,7 +21966,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
21862
21966
|
const redirected = isRedirectedPage(page);
|
|
21863
21967
|
if (redirected) {
|
|
21864
21968
|
const redirectTarget = page.metaRefresh?.targetUrl ?? page.finalUrl;
|
|
21865
|
-
issues.push(issue("redirected_url", page, `Final URL: ${redirectTarget}`, {
|
|
21969
|
+
issues.push(issue$1("redirected_url", page, `Final URL: ${redirectTarget}`, {
|
|
21866
21970
|
requestedUrl: page.url,
|
|
21867
21971
|
finalUrl: redirectTarget,
|
|
21868
21972
|
status: page.status,
|
|
@@ -21873,46 +21977,46 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
21873
21977
|
}));
|
|
21874
21978
|
}
|
|
21875
21979
|
const redirectChain = page.fetchDiagnostics?.redirectChain ?? [];
|
|
21876
|
-
if (redirectChain.length > 1) issues.push(issue("redirect_chain", page, `${redirectChain.length} hops`, {
|
|
21980
|
+
if (redirectChain.length > 1) issues.push(issue$1("redirect_chain", page, `${redirectChain.length} hops`, {
|
|
21877
21981
|
redirectChain,
|
|
21878
21982
|
hops: redirectChain.length
|
|
21879
21983
|
}));
|
|
21880
|
-
if ((page.responseTimeMs ?? 0) > SLOW_RESPONSE_MS) issues.push(issue("slow_response", page, `${page.responseTimeMs}ms`, {
|
|
21984
|
+
if ((page.responseTimeMs ?? 0) > SLOW_RESPONSE_MS) issues.push(issue$1("slow_response", page, `${page.responseTimeMs}ms`, {
|
|
21881
21985
|
responseTimeMs: page.responseTimeMs,
|
|
21882
21986
|
thresholdMs: SLOW_RESPONSE_MS
|
|
21883
21987
|
}));
|
|
21884
21988
|
if (page.auditScope === "status") continue;
|
|
21885
21989
|
if (redirected) continue;
|
|
21886
|
-
if (page.softAuthenticationGate) issues.push(issue("soft_authentication_gate", page, "A password form and login-specific page signals were observed.", page.softAuthenticationGate));
|
|
21990
|
+
if (page.softAuthenticationGate) issues.push(issue$1("soft_authentication_gate", page, "A password form and login-specific page signals were observed.", page.softAuthenticationGate));
|
|
21887
21991
|
const renderedContent = clientRenderedContentEvidence(page);
|
|
21888
|
-
if (renderedContent) issues.push(issue("client_rendered_content", page, "The raw response was an empty or near-empty shell.", renderedContent));
|
|
21889
|
-
if (!renderedContent && isAuditableHtmlPage(page) && page.indexable && page.extractionStatus === "complete" && page.wordCount < NEAR_EMPTY_CONTENT_WORDS) issues.push(issue("near_empty_content", page, `${page.wordCount} extracted words`, {
|
|
21992
|
+
if (renderedContent) issues.push(issue$1("client_rendered_content", page, "The raw response was an empty or near-empty shell.", renderedContent));
|
|
21993
|
+
if (!renderedContent && isAuditableHtmlPage(page) && page.indexable && page.extractionStatus === "complete" && page.wordCount < NEAR_EMPTY_CONTENT_WORDS) issues.push(issue$1("near_empty_content", page, `${page.wordCount} extracted words`, {
|
|
21890
21994
|
wordCount: page.wordCount,
|
|
21891
21995
|
reviewBelowWords: NEAR_EMPTY_CONTENT_WORDS,
|
|
21892
21996
|
contentSample: page.contentSample,
|
|
21893
21997
|
outgoingInternalCount: page.outgoingInternalCount,
|
|
21894
21998
|
rendering: page.fetchDiagnostics?.rendering
|
|
21895
21999
|
}));
|
|
21896
|
-
if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(issue("large_html", page, `${Math.round((page.sizeBytes ?? 0) / 1024)} KB`, {
|
|
22000
|
+
if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(issue$1("large_html", page, `${Math.round((page.sizeBytes ?? 0) / 1024)} KB`, {
|
|
21897
22001
|
sizeBytes: page.sizeBytes,
|
|
21898
22002
|
thresholdBytes: LARGE_HTML_BYTES
|
|
21899
22003
|
}));
|
|
21900
|
-
if (isHtmlPage(page) && !page.compression && (page.sizeBytes ?? 0) > COMPRESSION_MIN_BYTES) issues.push(issue("no_compression", page, void 0, {
|
|
22004
|
+
if (isHtmlPage(page) && !page.compression && (page.sizeBytes ?? 0) > COMPRESSION_MIN_BYTES) issues.push(issue$1("no_compression", page, void 0, {
|
|
21901
22005
|
sizeBytes: page.sizeBytes,
|
|
21902
22006
|
thresholdBytes: COMPRESSION_MIN_BYTES,
|
|
21903
22007
|
compression: page.compression
|
|
21904
22008
|
}));
|
|
21905
|
-
if (page.isHttps === false && !isLoopbackUrl(page.finalUrl || page.url)) issues.push(issue("http_not_secure", page, void 0, { finalUrl: page.finalUrl }));
|
|
22009
|
+
if (page.isHttps === false && !isLoopbackUrl(page.finalUrl || page.url)) issues.push(issue$1("http_not_secure", page, void 0, { finalUrl: page.finalUrl }));
|
|
21906
22010
|
else if (page.isHttps === true) {
|
|
21907
|
-
if ((page.mixedContentCount ?? 0) > 0) issues.push(issue("mixed_content", page, `${page.mixedContentCount} insecure resources`, {
|
|
22011
|
+
if ((page.mixedContentCount ?? 0) > 0) issues.push(issue$1("mixed_content", page, `${page.mixedContentCount} insecure resources`, {
|
|
21908
22012
|
mixedContentCount: page.mixedContentCount,
|
|
21909
22013
|
mixedContentSamples: page.mixedContentSamples
|
|
21910
22014
|
}));
|
|
21911
|
-
if (page.hasHsts === false) issues.push(issue("hsts_missing", page, void 0, { header: "strict-transport-security" }));
|
|
22015
|
+
if (page.hasHsts === false) issues.push(issue$1("hsts_missing", page, void 0, { header: "strict-transport-security" }));
|
|
21912
22016
|
}
|
|
21913
22017
|
for (const link of page.externalLinkChecks ?? []) {
|
|
21914
22018
|
if (link.state !== "confirmed-broken") continue;
|
|
21915
|
-
issues.push(issue("broken_external_link", page, link.url, {
|
|
22019
|
+
issues.push(issue$1("broken_external_link", page, link.url, {
|
|
21916
22020
|
url: link.url,
|
|
21917
22021
|
status: link.status,
|
|
21918
22022
|
error: link.error,
|
|
@@ -21920,64 +22024,64 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
21920
22024
|
attempts: link.attempts
|
|
21921
22025
|
}));
|
|
21922
22026
|
}
|
|
21923
|
-
if (page.indexable && (page.internalInlinkCount ?? 0) === 0 && !sameUrl(page.url, opts.startUrl)) issues.push(issue("orphan_page", page, "No internal inlinks", { internalInlinkCount: page.internalInlinkCount ?? 0 }));
|
|
21924
|
-
if ((page.crawlDepth ?? 0) > DEEP_PAGE_DEPTH) issues.push(issue("deep_page", page, `Depth ${page.crawlDepth}`, {
|
|
22027
|
+
if (page.indexable && (page.internalInlinkCount ?? 0) === 0 && !sameUrl(page.url, opts.startUrl)) issues.push(issue$1("orphan_page", page, "No internal inlinks", { internalInlinkCount: page.internalInlinkCount ?? 0 }));
|
|
22028
|
+
if ((page.crawlDepth ?? 0) > DEEP_PAGE_DEPTH) issues.push(issue$1("deep_page", page, `Depth ${page.crawlDepth}`, {
|
|
21925
22029
|
crawlDepth: page.crawlDepth,
|
|
21926
22030
|
threshold: DEEP_PAGE_DEPTH
|
|
21927
22031
|
}));
|
|
21928
|
-
if (page.indexable && isValuablePage(page) && (page.internalInlinkCount ?? 0) <= WEAK_VALUABLE_INLINKS) issues.push(issue("weak_internal_links_to_valuable_page", page, `${page.internalInlinkCount ?? 0} internal inlinks`, {
|
|
22032
|
+
if (page.indexable && isValuablePage(page) && (page.internalInlinkCount ?? 0) <= WEAK_VALUABLE_INLINKS) issues.push(issue$1("weak_internal_links_to_valuable_page", page, `${page.internalInlinkCount ?? 0} internal inlinks`, {
|
|
21929
22033
|
internalInlinkCount: page.internalInlinkCount ?? 0,
|
|
21930
22034
|
searchMetrics: page.searchMetrics,
|
|
21931
22035
|
analytics: page.analytics
|
|
21932
22036
|
}));
|
|
21933
|
-
if (page.robotsTxt?.allowed === false) issues.push(issue("robots_blocked", page, page.robotsTxt.matchedLine, { robotsTxt: page.robotsTxt }));
|
|
21934
|
-
if (hasXRobotsDirective(page.xRobotsTag, "noindex")) issues.push(issue("x_robots_noindex", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
|
|
22037
|
+
if (page.robotsTxt?.allowed === false) issues.push(issue$1("robots_blocked", page, page.robotsTxt.matchedLine, { robotsTxt: page.robotsTxt }));
|
|
22038
|
+
if (hasXRobotsDirective(page.xRobotsTag, "noindex")) issues.push(issue$1("x_robots_noindex", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
|
|
21935
22039
|
const xRobotsNofollow = hasXRobotsDirective(page.xRobotsTag, "nofollow");
|
|
21936
|
-
if (xRobotsNofollow) issues.push(issue("nofollow", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
|
|
22040
|
+
if (xRobotsNofollow) issues.push(issue$1("nofollow", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
|
|
21937
22041
|
if (!isAuditableHtmlPage(page)) continue;
|
|
21938
22042
|
const title = page.title?.trim();
|
|
21939
22043
|
const titleWidth = title ? estimateSerpTitleWidth(title) : void 0;
|
|
21940
22044
|
const duplicateTitle = duplicateTitles.get(normalizedMetadata(title) ?? "");
|
|
21941
|
-
if (!title) issues.push(issue("missing_title", page));
|
|
21942
|
-
else if (titleWidth?.status === "over-reference") issues.push(issue("title_too_wide", page, `${titleWidth.estimatedPixels}px`, {
|
|
22045
|
+
if (!title) issues.push(issue$1("missing_title", page));
|
|
22046
|
+
else if (titleWidth?.status === "over-reference") issues.push(issue$1("title_too_wide", page, `${titleWidth.estimatedPixels}px`, {
|
|
21943
22047
|
title,
|
|
21944
22048
|
...titleWidth
|
|
21945
22049
|
}));
|
|
21946
|
-
if (title && duplicateTitle) issues.push(issue("title_duplicate", page, duplicateTitle.value, {
|
|
22050
|
+
if (title && duplicateTitle) issues.push(issue$1("title_duplicate", page, duplicateTitle.value, {
|
|
21947
22051
|
title,
|
|
21948
22052
|
duplicateCount: duplicateTitle.count,
|
|
21949
22053
|
sampleUrls: duplicateTitle.sampleUrls
|
|
21950
22054
|
}));
|
|
21951
22055
|
const metaDescription = page.metaDescription?.trim();
|
|
21952
22056
|
const duplicateDescription = duplicateDescriptions.get(normalizedMetadata(metaDescription) ?? "");
|
|
21953
|
-
if (!metaDescription) issues.push(issue("missing_meta_description", page));
|
|
21954
|
-
if (metaDescription && duplicateDescription) issues.push(issue("meta_description_duplicate", page, duplicateDescription.value, {
|
|
22057
|
+
if (!metaDescription) issues.push(issue$1("missing_meta_description", page));
|
|
22058
|
+
if (metaDescription && duplicateDescription) issues.push(issue$1("meta_description_duplicate", page, duplicateDescription.value, {
|
|
21955
22059
|
metaDescription,
|
|
21956
22060
|
duplicateCount: duplicateDescription.count,
|
|
21957
22061
|
sampleUrls: duplicateDescription.sampleUrls
|
|
21958
22062
|
}));
|
|
21959
22063
|
const h1Count = page.h1Count ?? (page.h1 ? 1 : 0);
|
|
21960
|
-
if (h1Count === 0) issues.push(issue("h1_missing", page, "No H1 found", { h1Count }));
|
|
21961
|
-
else if (h1Count > 1) issues.push(issue("h1_multiple", page, `${h1Count} H1 headings`, { h1Count }));
|
|
21962
|
-
if (page.canonicalStatus === "conflicting") issues.push(issue("canonical_conflict", page, void 0, { candidates: page.canonicalCandidates }));
|
|
21963
|
-
else if (page.canonicalStatus === "duplicate") issues.push(issue("canonical_multiple", page, page.canonical, {
|
|
22064
|
+
if (h1Count === 0) issues.push(issue$1("h1_missing", page, "No H1 found", { h1Count }));
|
|
22065
|
+
else if (h1Count > 1) issues.push(issue$1("h1_multiple", page, `${h1Count} H1 headings`, { h1Count }));
|
|
22066
|
+
if (page.canonicalStatus === "conflicting") issues.push(issue$1("canonical_conflict", page, void 0, { candidates: page.canonicalCandidates }));
|
|
22067
|
+
else if (page.canonicalStatus === "duplicate") issues.push(issue$1("canonical_multiple", page, page.canonical, {
|
|
21964
22068
|
canonical: page.canonical,
|
|
21965
22069
|
candidates: page.canonicalCandidates
|
|
21966
22070
|
}));
|
|
21967
|
-
else if (page.canonicalStatus === "outside-head-only") issues.push(issue("canonical_outside_head", page, page.canonicalRaw, { candidates: page.canonicalCandidates }));
|
|
21968
|
-
else if (page.canonicalStatus === "streamed-outside-head") issues.push(issue("canonical_streamed_in_body", page, page.canonical, {
|
|
22071
|
+
else if (page.canonicalStatus === "outside-head-only") issues.push(issue$1("canonical_outside_head", page, page.canonicalRaw, { candidates: page.canonicalCandidates }));
|
|
22072
|
+
else if (page.canonicalStatus === "streamed-outside-head") issues.push(issue$1("canonical_streamed_in_body", page, page.canonical, {
|
|
21969
22073
|
canonical: page.canonical,
|
|
21970
22074
|
candidates: page.canonicalCandidates
|
|
21971
22075
|
}));
|
|
21972
22076
|
if (!page.canonical) {
|
|
21973
|
-
if (!["conflicting", "outside-head-only"].includes(page.canonicalStatus ?? "")) issues.push(page.canonicalRaw ? issue("canonical_invalid", page, page.canonicalRaw, { canonicalRaw: page.canonicalRaw }) : issue("canonical_missing", page));
|
|
22077
|
+
if (!["conflicting", "outside-head-only"].includes(page.canonicalStatus ?? "")) issues.push(page.canonicalRaw ? issue$1("canonical_invalid", page, page.canonicalRaw, { canonicalRaw: page.canonicalRaw }) : issue$1("canonical_missing", page));
|
|
21974
22078
|
} else {
|
|
21975
|
-
if (page.canonicalRaw && !absoluteHttpUrl(page.canonicalRaw)) issues.push(issue("canonical_non_absolute", page, page.canonicalRaw, {
|
|
22079
|
+
if (page.canonicalRaw && !absoluteHttpUrl(page.canonicalRaw)) issues.push(issue$1("canonical_non_absolute", page, page.canonicalRaw, {
|
|
21976
22080
|
canonicalRaw: page.canonicalRaw,
|
|
21977
22081
|
canonical: page.canonical
|
|
21978
22082
|
}));
|
|
21979
22083
|
const canonicalTarget = pageByUrl.get(urlKey(page.canonical) ?? "");
|
|
21980
|
-
if (canonicalTarget?.canonical && !sameUrl(canonicalTarget.canonical, page.canonical)) issues.push(issue("canonical_chain", page, page.canonical, {
|
|
22084
|
+
if (canonicalTarget?.canonical && !sameUrl(canonicalTarget.canonical, page.canonical)) issues.push(issue$1("canonical_chain", page, page.canonical, {
|
|
21981
22085
|
canonical: page.canonical,
|
|
21982
22086
|
nextCanonical: canonicalTarget.canonical,
|
|
21983
22087
|
chain: [
|
|
@@ -21987,25 +22091,25 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
21987
22091
|
]
|
|
21988
22092
|
}));
|
|
21989
22093
|
}
|
|
21990
|
-
if (hasMetaRobotsDirective(page.metaRobots, "noindex")) issues.push(issue("noindex", page, page.indexability, { metaRobots: page.metaRobots }));
|
|
21991
|
-
if (hasMetaRobotsDirective(page.metaRobots, "nofollow") && !xRobotsNofollow) issues.push(issue("nofollow", page, page.metaRobots, { metaRobots: page.metaRobots }));
|
|
21992
|
-
if (page.canonical && !sameUrl(page.canonical, page.finalUrl) && (page.declaredIndexability === "canonical-hint-other" || page.declaredIndexability === void 0 && /canonical/i.test(page.indexability ?? ""))) issues.push(issue("canonicalized_page", page, page.canonical, {
|
|
22094
|
+
if (hasMetaRobotsDirective(page.metaRobots, "noindex")) issues.push(issue$1("noindex", page, page.indexability, { metaRobots: page.metaRobots }));
|
|
22095
|
+
if (hasMetaRobotsDirective(page.metaRobots, "nofollow") && !xRobotsNofollow) issues.push(issue$1("nofollow", page, page.metaRobots, { metaRobots: page.metaRobots }));
|
|
22096
|
+
if (page.canonical && !sameUrl(page.canonical, page.finalUrl) && (page.declaredIndexability === "canonical-hint-other" || page.declaredIndexability === void 0 && /canonical/i.test(page.indexability ?? ""))) issues.push(issue$1("canonicalized_page", page, page.canonical, {
|
|
21993
22097
|
canonical: page.canonical,
|
|
21994
22098
|
finalUrl: page.finalUrl,
|
|
21995
22099
|
indexability: page.indexability
|
|
21996
22100
|
}));
|
|
21997
|
-
else if (page.canonical && !sameUrl(page.canonical, page.finalUrl)) issues.push(issue("canonical_mismatch", page, page.canonical, {
|
|
22101
|
+
else if (page.canonical && !sameUrl(page.canonical, page.finalUrl)) issues.push(issue$1("canonical_mismatch", page, page.canonical, {
|
|
21998
22102
|
canonical: page.canonical,
|
|
21999
22103
|
finalUrl: page.finalUrl
|
|
22000
22104
|
}));
|
|
22001
22105
|
const contentDuplicate = page.mainContentHash ? duplicateContent.get(page.mainContentHash) : void 0;
|
|
22002
|
-
if (contentDuplicate) issues.push(issue("duplicate_content", page, "Duplicate main content", {
|
|
22106
|
+
if (contentDuplicate) issues.push(issue$1("duplicate_content", page, "Duplicate main content", {
|
|
22003
22107
|
mainContentHash: page.mainContentHash,
|
|
22004
22108
|
duplicateCount: contentDuplicate.count,
|
|
22005
22109
|
sampleUrls: contentDuplicate.sampleUrls
|
|
22006
22110
|
}));
|
|
22007
22111
|
const coverage = queryCoverage(page);
|
|
22008
|
-
if (coverage && coverage.coverage < QUERY_COVERAGE_MIN) issues.push(issue("query_coverage_missing", page, coverage.query, {
|
|
22112
|
+
if (coverage && coverage.coverage < QUERY_COVERAGE_MIN) issues.push(issue$1("query_coverage_missing", page, coverage.query, {
|
|
22009
22113
|
query: coverage.query,
|
|
22010
22114
|
impressions: page.topQuery?.impressions,
|
|
22011
22115
|
matchedTerms: coverage.matchedTerms,
|
|
@@ -22013,36 +22117,36 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
22013
22117
|
coverage: coverage.coverage,
|
|
22014
22118
|
threshold: QUERY_COVERAGE_MIN
|
|
22015
22119
|
}));
|
|
22016
|
-
if ((page.imagesMissingAlt ?? 0) > 0) issues.push(issue("image_missing_alt", page, `${page.imagesMissingAlt} of ${page.imagesTotal ?? 0} images`, {
|
|
22120
|
+
if ((page.imagesMissingAlt ?? 0) > 0) issues.push(issue$1("image_missing_alt", page, `${page.imagesMissingAlt} of ${page.imagesTotal ?? 0} images`, {
|
|
22017
22121
|
imagesTotal: page.imagesTotal,
|
|
22018
22122
|
imagesMissingAlt: page.imagesMissingAlt
|
|
22019
22123
|
}));
|
|
22020
|
-
if (page.oversizedImageCandidates?.length) issues.push(issue("image_oversized_candidate", page, `${page.oversizedImageCandidates.length} image candidate${page.oversizedImageCandidates.length === 1 ? "" : "s"}`, {
|
|
22124
|
+
if (page.oversizedImageCandidates?.length) issues.push(issue$1("image_oversized_candidate", page, `${page.oversizedImageCandidates.length} image candidate${page.oversizedImageCandidates.length === 1 ? "" : "s"}`, {
|
|
22021
22125
|
thresholdPx: OVERSIZED_IMAGE_CANDIDATE_LIMIT,
|
|
22022
22126
|
candidates: page.oversizedImageCandidates.slice(0, 10)
|
|
22023
22127
|
}));
|
|
22024
|
-
if (!page.hasViewport) issues.push(issue("viewport_missing", page));
|
|
22025
|
-
if (!page.lang) issues.push(issue("lang_missing", page));
|
|
22128
|
+
if (!page.hasViewport) issues.push(issue$1("viewport_missing", page));
|
|
22129
|
+
if (!page.lang) issues.push(issue$1("lang_missing", page));
|
|
22026
22130
|
const hreflang = page.hreflang ?? [];
|
|
22027
22131
|
if (hreflang.length) {
|
|
22028
22132
|
const malformed = hreflang.filter((item) => !hasRecognizableHreflangFormat(item.hreflang));
|
|
22029
|
-
if (malformed.length) issues.push(issue("hreflang_invalid", page, `${malformed.length} malformed`, { malformed }));
|
|
22133
|
+
if (malformed.length) issues.push(issue$1("hreflang_invalid", page, `${malformed.length} malformed`, { malformed }));
|
|
22030
22134
|
const duplicateCodes = [...new Set(hreflang.map((item) => item.hreflang.trim().toLowerCase()).filter((code, index, values) => values.indexOf(code) !== index))];
|
|
22031
|
-
if (duplicateCodes.length) issues.push(issue("hreflang_duplicate", page, duplicateCodes.join(", "), { duplicateCodes }));
|
|
22032
|
-
if (!hreflang.some((item) => sameUrl(item.href, page.finalUrl))) issues.push(issue("hreflang_incomplete", page, page.lang, {
|
|
22135
|
+
if (duplicateCodes.length) issues.push(issue$1("hreflang_duplicate", page, duplicateCodes.join(", "), { duplicateCodes }));
|
|
22136
|
+
if (!hreflang.some((item) => sameUrl(item.href, page.finalUrl))) issues.push(issue$1("hreflang_incomplete", page, page.lang, {
|
|
22033
22137
|
lang: page.lang,
|
|
22034
22138
|
finalUrl: page.finalUrl,
|
|
22035
22139
|
hreflang
|
|
22036
22140
|
}));
|
|
22037
22141
|
}
|
|
22038
22142
|
const structuredDataFormats = page.structuredDataFormats;
|
|
22039
|
-
if (isAuditableHtmlPage(page) && page.indexable && structuredDataFormats && structuredDataFormats.length === 0 && (page.schemaTypes?.length ?? 0) === 0 && (page.invalidJsonLdCount ?? 0) === 0 && (page.unrecognizedJsonLdTypes?.length ?? 0) === 0) issues.push(issue("structured_data_missing", page, "No structured data detected", {
|
|
22143
|
+
if (isAuditableHtmlPage(page) && page.indexable && structuredDataFormats && structuredDataFormats.length === 0 && (page.schemaTypes?.length ?? 0) === 0 && (page.invalidJsonLdCount ?? 0) === 0 && (page.unrecognizedJsonLdTypes?.length ?? 0) === 0) issues.push(issue$1("structured_data_missing", page, "No structured data detected", {
|
|
22040
22144
|
structuredDataFormats,
|
|
22041
22145
|
schemaTypes: page.schemaTypes ?? [],
|
|
22042
22146
|
invalidJsonLdCount: page.invalidJsonLdCount ?? 0,
|
|
22043
22147
|
unrecognizedJsonLdTypeCount: page.unrecognizedJsonLdTypes?.length ?? 0
|
|
22044
22148
|
}));
|
|
22045
|
-
if ((page.invalidJsonLdCount ?? 0) > 0) issues.push(issue("jsonld_invalid", page, `${page.invalidJsonLdCount} invalid`, {
|
|
22149
|
+
if ((page.invalidJsonLdCount ?? 0) > 0) issues.push(issue$1("jsonld_invalid", page, `${page.invalidJsonLdCount} invalid`, {
|
|
22046
22150
|
invalidJsonLdCount: page.invalidJsonLdCount,
|
|
22047
22151
|
invalidJsonLdSamples: page.invalidJsonLdSamples
|
|
22048
22152
|
}));
|
|
@@ -22052,15 +22156,15 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
22052
22156
|
if (incompleteRichResultCount) {
|
|
22053
22157
|
const details = incompleteRichResults.map((assessment) => `${assessment.schemaType}: ${assessment.missingRequiredProperties.join(", ")}`);
|
|
22054
22158
|
const omittedDetails = incompleteRichResultCount - details.length;
|
|
22055
|
-
issues.push(issue("rich_result_required_fields_missing", page, [...details, ...omittedDetails > 0 ? [`${omittedDetails} more incomplete assessment${omittedDetails === 1 ? "" : "s"} omitted from bounded detail`] : []].join("; "), {
|
|
22159
|
+
issues.push(issue$1("rich_result_required_fields_missing", page, [...details, ...omittedDetails > 0 ? [`${omittedDetails} more incomplete assessment${omittedDetails === 1 ? "" : "s"} omitted from bounded detail`] : []].join("; "), {
|
|
22056
22160
|
assessments: incompleteRichResults,
|
|
22057
22161
|
...page.googleRichResultsSelection ? { selection: page.googleRichResultsSelection } : {}
|
|
22058
22162
|
}));
|
|
22059
22163
|
}
|
|
22060
|
-
if (!page.openGraphTitle) issues.push(issue("og_title_missing", page));
|
|
22061
|
-
if (!page.openGraphDescription) issues.push(issue("og_description_missing", page));
|
|
22062
|
-
if (!page.openGraphImage) issues.push(issue("og_image_missing", page));
|
|
22063
|
-
if (!page.twitterCard) issues.push(issue("twitter_card_missing", page));
|
|
22164
|
+
if (!page.openGraphTitle) issues.push(issue$1("og_title_missing", page));
|
|
22165
|
+
if (!page.openGraphDescription) issues.push(issue$1("og_description_missing", page));
|
|
22166
|
+
if (!page.openGraphImage) issues.push(issue$1("og_image_missing", page));
|
|
22167
|
+
if (!page.twitterCard) issues.push(issue$1("twitter_card_missing", page));
|
|
22064
22168
|
}
|
|
22065
22169
|
return issues;
|
|
22066
22170
|
}
|
|
@@ -22150,7 +22254,7 @@ const SEVERITY_SCORE = {
|
|
|
22150
22254
|
const EFFORT_BY_CATEGORY = {
|
|
22151
22255
|
canonical: "medium",
|
|
22152
22256
|
content: "high",
|
|
22153
|
-
|
|
22257
|
+
agents: "low",
|
|
22154
22258
|
headings: "low",
|
|
22155
22259
|
images: "medium",
|
|
22156
22260
|
indexability: "medium",
|
|
@@ -22717,6 +22821,92 @@ const crawlSiteChecksSchema = z.object({ soft404: z.object({
|
|
|
22717
22821
|
complete: z.boolean()
|
|
22718
22822
|
}) });
|
|
22719
22823
|
//#endregion
|
|
22824
|
+
//#region packages/core/dist/analyze/crawler/site-crawl-ai-schema.js
|
|
22825
|
+
const crawlAiAvailabilitySchema = z.enum([
|
|
22826
|
+
"available",
|
|
22827
|
+
"absent",
|
|
22828
|
+
"access-blocked",
|
|
22829
|
+
"rate-limited",
|
|
22830
|
+
"unreachable"
|
|
22831
|
+
]);
|
|
22832
|
+
const crawlAiLinkCheckSchema = z.object({
|
|
22833
|
+
url: z.string().url(),
|
|
22834
|
+
status: z.number().int().optional(),
|
|
22835
|
+
contentType: z.string().optional(),
|
|
22836
|
+
error: z.string().optional()
|
|
22837
|
+
});
|
|
22838
|
+
const crawlAiSignalsSchema = z.object({
|
|
22839
|
+
robotsTxt: z.object({
|
|
22840
|
+
url: z.string().url(),
|
|
22841
|
+
exists: z.boolean(),
|
|
22842
|
+
availability: crawlAiAvailabilitySchema,
|
|
22843
|
+
status: z.number().int().optional(),
|
|
22844
|
+
error: z.string().optional(),
|
|
22845
|
+
sitemapUrls: z.array(z.string().url()),
|
|
22846
|
+
contentSignals: z.array(z.string()).optional(),
|
|
22847
|
+
botAccess: z.array(z.object({
|
|
22848
|
+
userAgent: z.string(),
|
|
22849
|
+
purpose: z.enum([
|
|
22850
|
+
"search",
|
|
22851
|
+
"training",
|
|
22852
|
+
"other"
|
|
22853
|
+
]),
|
|
22854
|
+
allowed: z.boolean().nullable(),
|
|
22855
|
+
declared: z.boolean(),
|
|
22856
|
+
coveredByWildcard: z.boolean()
|
|
22857
|
+
}))
|
|
22858
|
+
}).optional(),
|
|
22859
|
+
llmsTxt: z.object({
|
|
22860
|
+
url: z.string().url(),
|
|
22861
|
+
exists: z.boolean(),
|
|
22862
|
+
availability: crawlAiAvailabilitySchema.optional(),
|
|
22863
|
+
status: z.number().int().optional(),
|
|
22864
|
+
error: z.string().optional(),
|
|
22865
|
+
links: z.object({
|
|
22866
|
+
found: z.number().int().nonnegative(),
|
|
22867
|
+
checked: z.number().int().nonnegative(),
|
|
22868
|
+
limit: z.number().int().positive(),
|
|
22869
|
+
broken: z.array(crawlAiLinkCheckSchema)
|
|
22870
|
+
}).optional()
|
|
22871
|
+
}).optional(),
|
|
22872
|
+
agentResources: z.array(z.object({
|
|
22873
|
+
url: z.string().url(),
|
|
22874
|
+
exists: z.boolean(),
|
|
22875
|
+
status: z.number().int().optional(),
|
|
22876
|
+
contentType: z.string().optional(),
|
|
22877
|
+
validJson: z.boolean().optional()
|
|
22878
|
+
})).optional(),
|
|
22879
|
+
markdownNegotiation: z.object({
|
|
22880
|
+
url: z.string().url(),
|
|
22881
|
+
availability: crawlAiAvailabilitySchema,
|
|
22882
|
+
status: z.number().int().optional(),
|
|
22883
|
+
contentType: z.string().optional(),
|
|
22884
|
+
markdown: z.boolean(),
|
|
22885
|
+
varyAccept: z.boolean(),
|
|
22886
|
+
error: z.string().optional()
|
|
22887
|
+
}).optional(),
|
|
22888
|
+
markdownCopies: z.object({
|
|
22889
|
+
publicPages: z.number().int().nonnegative(),
|
|
22890
|
+
publicPagesWithLink: z.number().int().nonnegative(),
|
|
22891
|
+
privatePagesWithLink: z.number().int().nonnegative(),
|
|
22892
|
+
checked: z.number().int().nonnegative(),
|
|
22893
|
+
limit: z.number().int().positive(),
|
|
22894
|
+
broken: z.array(crawlAiLinkCheckSchema.extend({
|
|
22895
|
+
page: z.string().url(),
|
|
22896
|
+
public: z.boolean()
|
|
22897
|
+
})),
|
|
22898
|
+
largest: z.object({
|
|
22899
|
+
url: z.string().url(),
|
|
22900
|
+
estimatedTokens: z.number().int().nonnegative()
|
|
22901
|
+
}).optional()
|
|
22902
|
+
}).optional(),
|
|
22903
|
+
startPageHeaders: z.object({
|
|
22904
|
+
url: z.string().url(),
|
|
22905
|
+
linkRelations: z.array(z.string()),
|
|
22906
|
+
contentSignal: z.string().optional()
|
|
22907
|
+
}).optional()
|
|
22908
|
+
});
|
|
22909
|
+
//#endregion
|
|
22720
22910
|
//#region packages/core/dist/analyze/crawler/schemas.js
|
|
22721
22911
|
const crawlerRuleSeveritySchema = z.enum([
|
|
22722
22912
|
"low",
|
|
@@ -22739,7 +22929,7 @@ const crawlerRuleCategorySchema = z.enum([
|
|
|
22739
22929
|
"security",
|
|
22740
22930
|
"social",
|
|
22741
22931
|
"structured-data",
|
|
22742
|
-
"
|
|
22932
|
+
"agents"
|
|
22743
22933
|
]);
|
|
22744
22934
|
const crawlerRuleIdSchema = z.enum(listRules().map((rule) => rule.id));
|
|
22745
22935
|
const crawlerRuleInfoSchema = z.object({
|
|
@@ -23365,41 +23555,6 @@ const crawlIssueGroupSchema = z.object({
|
|
|
23365
23555
|
count: z.number().int(),
|
|
23366
23556
|
sampleUrls: z.array(z.string().url())
|
|
23367
23557
|
});
|
|
23368
|
-
const crawlAiSignalsSchema = z.object({
|
|
23369
|
-
robotsTxt: z.object({
|
|
23370
|
-
url: z.string().url(),
|
|
23371
|
-
exists: z.boolean(),
|
|
23372
|
-
availability: z.enum([
|
|
23373
|
-
"available",
|
|
23374
|
-
"absent",
|
|
23375
|
-
"access-blocked",
|
|
23376
|
-
"rate-limited",
|
|
23377
|
-
"unreachable"
|
|
23378
|
-
]),
|
|
23379
|
-
status: z.number().int().optional(),
|
|
23380
|
-
error: z.string().optional(),
|
|
23381
|
-
sitemapUrls: z.array(z.string().url()),
|
|
23382
|
-
contentSignals: z.array(z.string()).optional(),
|
|
23383
|
-
botAccess: z.array(z.object({
|
|
23384
|
-
userAgent: z.string(),
|
|
23385
|
-
allowed: z.boolean().nullable(),
|
|
23386
|
-
declared: z.boolean(),
|
|
23387
|
-
coveredByWildcard: z.boolean()
|
|
23388
|
-
}))
|
|
23389
|
-
}).optional(),
|
|
23390
|
-
llmsTxt: z.object({
|
|
23391
|
-
url: z.string().url(),
|
|
23392
|
-
exists: z.boolean(),
|
|
23393
|
-
status: z.number().int().optional()
|
|
23394
|
-
}).optional(),
|
|
23395
|
-
agentResources: z.array(z.object({
|
|
23396
|
-
url: z.string().url(),
|
|
23397
|
-
exists: z.boolean(),
|
|
23398
|
-
status: z.number().int().optional(),
|
|
23399
|
-
contentType: z.string().optional(),
|
|
23400
|
-
validJson: z.boolean().optional()
|
|
23401
|
-
})).optional()
|
|
23402
|
-
});
|
|
23403
23558
|
const crawlTopFixSchema = crawlIssueGroupSchema.extend({
|
|
23404
23559
|
recommendation: crawlerRuleRecommendationSchema,
|
|
23405
23560
|
score: z.number(),
|
|
@@ -23696,6 +23851,58 @@ const crawlReportSchema = z.discriminatedUnion("requestEvidenceStatus", [
|
|
|
23696
23851
|
]);
|
|
23697
23852
|
z.toJSONSchema(crawlReportSchema), z.toJSONSchema(crawlIssueGroupSchema), z.toJSONSchema(crawlTopFixSchema), z.toJSONSchema(crawlerRuleInfoSchema), z.toJSONSchema(crawlRequestObservationSchema), z.toJSONSchema(crawlPageSnapshotSchema);
|
|
23698
23853
|
//#endregion
|
|
23854
|
+
//#region packages/core/dist/analyze/crawler/agent-ready.js
|
|
23855
|
+
function needsSignIn(page) {
|
|
23856
|
+
return page.status === 401 || page.status === 403 || Boolean(page.softAuthenticationGate);
|
|
23857
|
+
}
|
|
23858
|
+
function isPublicPage(page) {
|
|
23859
|
+
return page.status >= 200 && page.status < 300 && !needsSignIn(page) && /\btext\/html\b/i.test(page.contentType ?? "");
|
|
23860
|
+
}
|
|
23861
|
+
function expectsMarkdown(page) {
|
|
23862
|
+
return isPublicPage(page) && page.indexable;
|
|
23863
|
+
}
|
|
23864
|
+
function issue(ruleId, url, detail, evidence) {
|
|
23865
|
+
const rule = explainRule(ruleId);
|
|
23866
|
+
if (!rule) throw new Error(`Missing rule guidance for ${ruleId}`);
|
|
23867
|
+
return {
|
|
23868
|
+
ruleId,
|
|
23869
|
+
title: rule.title,
|
|
23870
|
+
category: rule.category,
|
|
23871
|
+
severity: rule.defaultSeverity,
|
|
23872
|
+
url,
|
|
23873
|
+
detail,
|
|
23874
|
+
...evidence ? { evidence } : {}
|
|
23875
|
+
};
|
|
23876
|
+
}
|
|
23877
|
+
function agentReadyIssues(input) {
|
|
23878
|
+
const { ai, startUrl } = input;
|
|
23879
|
+
const issues = [];
|
|
23880
|
+
const blocked = (ai.robotsTxt?.botAccess ?? []).filter((bot) => bot.purpose === "search" && bot.allowed === false);
|
|
23881
|
+
if (blocked.length && ai.robotsTxt) issues.push(issue("ai_search_bot_blocked", ai.robotsTxt.url, `robots.txt blocks ${blocked.map((bot) => bot.userAgent).join(", ")} from the start URL.`, { blocked: blocked.map((bot) => bot.userAgent) }));
|
|
23882
|
+
if (ai.llmsTxt?.availability === "absent") issues.push(issue("llms_txt_missing", ai.llmsTxt.url, `/llms.txt returned ${ai.llmsTxt.status === void 0 ? "no file" : `HTTP ${ai.llmsTxt.status}`}.`));
|
|
23883
|
+
for (const link of ai.llmsTxt?.links?.broken ?? []) issues.push(issue("llms_txt_broken_link", link.url, link.status === void 0 ? `The link in llms.txt could not be fetched: ${link.error ?? "no response"}.` : `The link in llms.txt returned HTTP ${link.status}.`, {
|
|
23884
|
+
llmsTxt: ai.llmsTxt?.url,
|
|
23885
|
+
...link
|
|
23886
|
+
}));
|
|
23887
|
+
const expected = input.pages.filter(expectsMarkdown);
|
|
23888
|
+
const linked = expected.filter((page) => page.markdownAlternates?.length);
|
|
23889
|
+
const negotiation = ai.markdownNegotiation;
|
|
23890
|
+
if (expected.length && !linked.length && negotiation?.availability === "available" && !negotiation.markdown) issues.push(issue("markdown_missing", startUrl, `Accept: text/markdown returned ${negotiation.contentType || "no content type"}, and none of ${expected.length} public pages links to a Markdown copy.`, { negotiation }));
|
|
23891
|
+
if (linked.length) for (const page of expected) {
|
|
23892
|
+
if (page.markdownAlternates?.length) continue;
|
|
23893
|
+
issues.push(issue("markdown_alternate_missing", page.url, `${linked.length} of ${expected.length} public pages link to a Markdown copy; this one does not.`));
|
|
23894
|
+
}
|
|
23895
|
+
for (const page of input.pages) {
|
|
23896
|
+
if (!needsSignIn(page) || !page.markdownAlternates?.length) continue;
|
|
23897
|
+
issues.push(issue("markdown_alternate_on_private_page", page.url, `The page needs a sign-in but links to a Markdown copy at ${page.markdownAlternates[0]}.`, { markdownAlternates: page.markdownAlternates }));
|
|
23898
|
+
}
|
|
23899
|
+
for (const copy of ai.markdownCopies?.broken ?? []) {
|
|
23900
|
+
if (!copy.public) continue;
|
|
23901
|
+
issues.push(issue("markdown_copy_broken", copy.page, copy.status === void 0 ? `The Markdown copy at ${copy.url} could not be fetched: ${copy.error ?? "no response"}.` : `The Markdown copy at ${copy.url} returned HTTP ${copy.status}${copy.contentType ? ` as ${copy.contentType}` : ""}.`, { ...copy }));
|
|
23902
|
+
}
|
|
23903
|
+
return issues;
|
|
23904
|
+
}
|
|
23905
|
+
//#endregion
|
|
23699
23906
|
//#region packages/core/dist/analyze/crawler/crawl-caveats.js
|
|
23700
23907
|
function crawlCaveats(input) {
|
|
23701
23908
|
const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
|
|
@@ -24133,66 +24340,137 @@ async function collectCrawlSiteChecks(input) {
|
|
|
24133
24340
|
//#endregion
|
|
24134
24341
|
//#region packages/core/dist/analyze/crawler/site-crawl-ai-signals.js
|
|
24135
24342
|
const MAX_AUXILIARY_RESPONSE_BYTES = 2 * 1024 * 1024;
|
|
24136
|
-
const
|
|
24137
|
-
|
|
24138
|
-
"OAI-SearchBot",
|
|
24139
|
-
"ChatGPT-User",
|
|
24140
|
-
"
|
|
24141
|
-
"Claude-User",
|
|
24142
|
-
"
|
|
24143
|
-
"
|
|
24144
|
-
"
|
|
24145
|
-
"
|
|
24146
|
-
"
|
|
24147
|
-
"Applebot",
|
|
24148
|
-
"
|
|
24149
|
-
"
|
|
24150
|
-
"
|
|
24151
|
-
"
|
|
24152
|
-
"
|
|
24153
|
-
"Meta-
|
|
24154
|
-
"
|
|
24155
|
-
"
|
|
24156
|
-
"
|
|
24157
|
-
"
|
|
24343
|
+
const LLMS_LINK_LIMIT = 20;
|
|
24344
|
+
const AI_BOTS = [
|
|
24345
|
+
["OAI-SearchBot", "search"],
|
|
24346
|
+
["ChatGPT-User", "search"],
|
|
24347
|
+
["Claude-SearchBot", "search"],
|
|
24348
|
+
["Claude-User", "search"],
|
|
24349
|
+
["PerplexityBot", "search"],
|
|
24350
|
+
["Perplexity-User", "search"],
|
|
24351
|
+
["DuckAssistBot", "search"],
|
|
24352
|
+
["MistralAI-User", "search"],
|
|
24353
|
+
["Meta-ExternalFetcher", "search"],
|
|
24354
|
+
["Applebot", "search"],
|
|
24355
|
+
["Bingbot", "search"],
|
|
24356
|
+
["GPTBot", "training"],
|
|
24357
|
+
["ClaudeBot", "training"],
|
|
24358
|
+
["Google-Extended", "training"],
|
|
24359
|
+
["Applebot-Extended", "training"],
|
|
24360
|
+
["Meta-ExternalAgent", "training"],
|
|
24361
|
+
["CCBot", "training"],
|
|
24362
|
+
["Bytespider", "training"],
|
|
24363
|
+
["cohere-ai", "training"],
|
|
24364
|
+
["Amazonbot", "other"],
|
|
24365
|
+
["Diffbot", "other"]
|
|
24158
24366
|
];
|
|
24159
24367
|
const AGENT_RESOURCE_PATHS = [
|
|
24160
|
-
"/.well-known/agent.json",
|
|
24161
|
-
"/
|
|
24368
|
+
"/.well-known/agent-skills/index.json",
|
|
24369
|
+
"/.well-known/mcp/server-card.json",
|
|
24162
24370
|
"/.well-known/mcp.json",
|
|
24163
|
-
"/.well-known/
|
|
24164
|
-
"
|
|
24165
|
-
"/
|
|
24371
|
+
"/.well-known/api-catalog",
|
|
24372
|
+
"/openapi.json",
|
|
24373
|
+
"/.well-known/oauth-protected-resource",
|
|
24374
|
+
"/.well-known/oauth-authorization-server",
|
|
24375
|
+
"/.well-known/http-message-signatures-directory"
|
|
24166
24376
|
];
|
|
24167
|
-
|
|
24168
|
-
|
|
24377
|
+
function availabilityOf(status) {
|
|
24378
|
+
if (status >= 200 && status < 300) return "available";
|
|
24379
|
+
if (status === 401 || status === 403) return "access-blocked";
|
|
24380
|
+
if (status === 429) return "rate-limited";
|
|
24381
|
+
if (status >= 400 && status < 500) return "absent";
|
|
24382
|
+
return "unreachable";
|
|
24383
|
+
}
|
|
24384
|
+
function errorText(error) {
|
|
24385
|
+
return error instanceof Error ? error.message : String(error);
|
|
24386
|
+
}
|
|
24387
|
+
async function get(input, url, headers, readBody = true) {
|
|
24169
24388
|
const controller = abortController({
|
|
24170
24389
|
timeoutMs: input.timeoutMs,
|
|
24171
24390
|
signal: input.signal
|
|
24172
24391
|
});
|
|
24173
24392
|
try {
|
|
24174
|
-
const response = await input.fetch(
|
|
24393
|
+
const response = await input.fetch(url, {
|
|
24175
24394
|
profile: "bot",
|
|
24176
24395
|
redirect: "follow",
|
|
24177
|
-
signal: controller.signal
|
|
24396
|
+
signal: controller.signal,
|
|
24397
|
+
...headers ? { headers } : {}
|
|
24178
24398
|
});
|
|
24179
|
-
|
|
24180
|
-
|
|
24181
|
-
await response.body?.cancel().catch(() => void 0);
|
|
24399
|
+
let text = "";
|
|
24400
|
+
if (readBody) text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, `${new URL(url).pathname} response`);
|
|
24401
|
+
else await response.body?.cancel().catch(() => void 0);
|
|
24182
24402
|
return {
|
|
24183
|
-
url: response.url ||
|
|
24184
|
-
|
|
24185
|
-
status: response.status
|
|
24403
|
+
url: response.url || url,
|
|
24404
|
+
availability: availabilityOf(response.status),
|
|
24405
|
+
status: response.status,
|
|
24406
|
+
contentType: response.headers.get("content-type") ?? "",
|
|
24407
|
+
headers: response.headers,
|
|
24408
|
+
text
|
|
24186
24409
|
};
|
|
24187
|
-
} catch {
|
|
24410
|
+
} catch (error) {
|
|
24188
24411
|
return {
|
|
24189
|
-
url
|
|
24190
|
-
|
|
24412
|
+
url,
|
|
24413
|
+
availability: "unreachable",
|
|
24414
|
+
contentType: "",
|
|
24415
|
+
text: "",
|
|
24416
|
+
error: errorText(error)
|
|
24191
24417
|
};
|
|
24192
24418
|
} finally {
|
|
24193
24419
|
controller.cleanup();
|
|
24194
24420
|
}
|
|
24195
24421
|
}
|
|
24422
|
+
const isHtml = (contentType) => /\btext\/html\b/i.test(contentType);
|
|
24423
|
+
const isMarkdown = (contentType) => /\btext\/(?:x-)?markdown\b/i.test(contentType);
|
|
24424
|
+
function brokenLink(result, expectText) {
|
|
24425
|
+
if (result.availability === "available" && !(expectText && isHtml(result.contentType))) return void 0;
|
|
24426
|
+
return {
|
|
24427
|
+
url: result.url,
|
|
24428
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
24429
|
+
...result.contentType ? { contentType: result.contentType } : {},
|
|
24430
|
+
...result.error ? { error: result.error } : {}
|
|
24431
|
+
};
|
|
24432
|
+
}
|
|
24433
|
+
function llmsLinks(text, base) {
|
|
24434
|
+
const origin = new URL(base).origin;
|
|
24435
|
+
const links = [];
|
|
24436
|
+
for (const match of text.matchAll(/\]\(\s*<?([^)\s>]+)>?/g)) try {
|
|
24437
|
+
const url = new URL(match[1] ?? "", base);
|
|
24438
|
+
url.hash = "";
|
|
24439
|
+
if (url.origin === origin && !links.includes(url.toString())) links.push(url.toString());
|
|
24440
|
+
} catch {}
|
|
24441
|
+
return links;
|
|
24442
|
+
}
|
|
24443
|
+
async function checkLlmsTxt(input) {
|
|
24444
|
+
const result = await get(input, new URL("/llms.txt", input.url).toString());
|
|
24445
|
+
const availability = result.availability === "available" && isHtml(result.contentType) ? "absent" : result.availability;
|
|
24446
|
+
const exists = availability === "available";
|
|
24447
|
+
const base = {
|
|
24448
|
+
url: result.url,
|
|
24449
|
+
exists,
|
|
24450
|
+
availability,
|
|
24451
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
24452
|
+
...result.error ? { error: result.error } : {}
|
|
24453
|
+
};
|
|
24454
|
+
if (!exists) return base;
|
|
24455
|
+
const found = llmsLinks(result.text, result.url);
|
|
24456
|
+
const broken = [];
|
|
24457
|
+
let checked = 0;
|
|
24458
|
+
for (const url of found.slice(0, LLMS_LINK_LIMIT)) {
|
|
24459
|
+
if (input.signal?.aborted) break;
|
|
24460
|
+
checked += 1;
|
|
24461
|
+
const link = brokenLink(await get(input, url, void 0, false), false);
|
|
24462
|
+
if (link) broken.push(link);
|
|
24463
|
+
}
|
|
24464
|
+
return {
|
|
24465
|
+
...base,
|
|
24466
|
+
links: {
|
|
24467
|
+
found: found.length,
|
|
24468
|
+
checked,
|
|
24469
|
+
limit: LLMS_LINK_LIMIT,
|
|
24470
|
+
broken
|
|
24471
|
+
}
|
|
24472
|
+
};
|
|
24473
|
+
}
|
|
24196
24474
|
function parseRobots(robotsUrl, text) {
|
|
24197
24475
|
return robotsParserModule(robotsUrl, text);
|
|
24198
24476
|
}
|
|
@@ -24224,118 +24502,135 @@ function sitemapUrlsFromRobots(text) {
|
|
|
24224
24502
|
return [...urls];
|
|
24225
24503
|
}
|
|
24226
24504
|
async function checkRobotsAiAccess(input) {
|
|
24227
|
-
const
|
|
24228
|
-
const
|
|
24229
|
-
|
|
24230
|
-
|
|
24231
|
-
|
|
24232
|
-
|
|
24233
|
-
|
|
24234
|
-
|
|
24235
|
-
redirect: "follow",
|
|
24236
|
-
signal: controller.signal
|
|
24237
|
-
});
|
|
24238
|
-
const text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, "robots.txt response");
|
|
24239
|
-
const available = response.status >= 200 && response.status < 300;
|
|
24240
|
-
const absent = response.status >= 400 && response.status < 500 && response.status !== 429;
|
|
24241
|
-
const availability = available ? "available" : [401, 403].includes(response.status) ? "access-blocked" : response.status === 429 ? "rate-limited" : absent ? "absent" : "unreachable";
|
|
24242
|
-
const exists = available;
|
|
24243
|
-
const declared = exists ? declaredUserAgents(text) : /* @__PURE__ */ new Set();
|
|
24244
|
-
const parsed = parseRobots(robotsUrl, exists ? text : "");
|
|
24245
|
-
return {
|
|
24246
|
-
url: response.url || robotsUrl,
|
|
24505
|
+
const result = await get(input, new URL("/robots.txt", input.url).toString());
|
|
24506
|
+
const exists = result.availability === "available";
|
|
24507
|
+
const unknown = result.availability === "rate-limited" || result.availability === "unreachable";
|
|
24508
|
+
const declared = exists ? declaredUserAgents(result.text) : /* @__PURE__ */ new Set();
|
|
24509
|
+
const parsed = parseRobots(result.url, exists ? result.text : "");
|
|
24510
|
+
return {
|
|
24511
|
+
robotsTxt: {
|
|
24512
|
+
url: result.url,
|
|
24247
24513
|
exists,
|
|
24248
|
-
availability,
|
|
24249
|
-
status:
|
|
24250
|
-
...
|
|
24251
|
-
sitemapUrls: exists ? sitemapUrlsFromRobots(text) : [],
|
|
24252
|
-
contentSignals: exists ? contentSignalsFromRobots(text) : [],
|
|
24253
|
-
botAccess:
|
|
24254
|
-
const lower = userAgent.toLowerCase();
|
|
24255
|
-
return {
|
|
24256
|
-
userAgent,
|
|
24257
|
-
allowed: availability === "rate-limited" || availability === "unreachable" ? null : parsed.isAllowed(input.url, userAgent) ?? true,
|
|
24258
|
-
declared: declared.has(lower),
|
|
24259
|
-
coveredByWildcard: declared.has("*")
|
|
24260
|
-
};
|
|
24261
|
-
})
|
|
24262
|
-
};
|
|
24263
|
-
} catch (error) {
|
|
24264
|
-
return {
|
|
24265
|
-
url: robotsUrl,
|
|
24266
|
-
exists: false,
|
|
24267
|
-
availability: "unreachable",
|
|
24268
|
-
error: error instanceof Error ? error.message : String(error),
|
|
24269
|
-
sitemapUrls: [],
|
|
24270
|
-
botAccess: AI_BOT_USER_AGENTS.map((userAgent) => ({
|
|
24514
|
+
availability: result.availability,
|
|
24515
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
24516
|
+
...unknown ? { error: result.error ?? `robots.txt returned HTTP ${result.status}.` } : {},
|
|
24517
|
+
sitemapUrls: exists ? sitemapUrlsFromRobots(result.text) : [],
|
|
24518
|
+
...unknown ? {} : { contentSignals: exists ? contentSignalsFromRobots(result.text) : [] },
|
|
24519
|
+
botAccess: AI_BOTS.map(([userAgent, purpose]) => ({
|
|
24271
24520
|
userAgent,
|
|
24272
|
-
|
|
24273
|
-
|
|
24274
|
-
|
|
24521
|
+
purpose,
|
|
24522
|
+
allowed: unknown ? null : parsed.isAllowed(input.url, userAgent) ?? true,
|
|
24523
|
+
declared: declared.has(userAgent.toLowerCase()),
|
|
24524
|
+
coveredByWildcard: declared.has("*")
|
|
24275
24525
|
}))
|
|
24276
|
-
}
|
|
24277
|
-
|
|
24278
|
-
|
|
24279
|
-
}
|
|
24526
|
+
},
|
|
24527
|
+
allows: (url) => unknown ? null : parsed.isAllowed(url, SEO_CRAWLER_USER_AGENT) ?? true
|
|
24528
|
+
};
|
|
24280
24529
|
}
|
|
24281
|
-
async function
|
|
24282
|
-
const
|
|
24283
|
-
const
|
|
24284
|
-
|
|
24285
|
-
|
|
24286
|
-
|
|
24287
|
-
try {
|
|
24288
|
-
const response = await input.fetch(url, {
|
|
24289
|
-
profile: "bot",
|
|
24290
|
-
redirect: "follow",
|
|
24291
|
-
signal: controller.signal
|
|
24292
|
-
});
|
|
24293
|
-
const contentType = response.headers.get("content-type") ?? "";
|
|
24294
|
-
const text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, "Agent resource response");
|
|
24295
|
-
const exists = response.status >= 200 && response.status < 300;
|
|
24530
|
+
async function checkAgentResources(input) {
|
|
24531
|
+
const resources = [];
|
|
24532
|
+
for (const path of AGENT_RESOURCE_PATHS) {
|
|
24533
|
+
if (input.signal?.aborted) break;
|
|
24534
|
+
const result = await get(input, new URL(path, input.url).toString());
|
|
24535
|
+
const exists = result.availability === "available" && !isHtml(result.contentType);
|
|
24296
24536
|
let validJson;
|
|
24297
|
-
if (exists && /\bjson\b/i.test(contentType)) try {
|
|
24298
|
-
JSON.parse(text);
|
|
24537
|
+
if (exists && /\bjson\b/i.test(result.contentType)) try {
|
|
24538
|
+
JSON.parse(result.text);
|
|
24299
24539
|
validJson = true;
|
|
24300
24540
|
} catch {
|
|
24301
24541
|
validJson = false;
|
|
24302
24542
|
}
|
|
24303
|
-
|
|
24304
|
-
url:
|
|
24543
|
+
resources.push({
|
|
24544
|
+
url: result.url,
|
|
24305
24545
|
exists,
|
|
24306
|
-
status:
|
|
24307
|
-
contentType,
|
|
24546
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
24547
|
+
...result.contentType ? { contentType: result.contentType } : {},
|
|
24308
24548
|
...validJson === void 0 ? {} : { validJson }
|
|
24309
|
-
};
|
|
24310
|
-
} catch {
|
|
24311
|
-
return {
|
|
24312
|
-
url,
|
|
24313
|
-
exists: false
|
|
24314
|
-
};
|
|
24315
|
-
} finally {
|
|
24316
|
-
controller.cleanup();
|
|
24317
|
-
}
|
|
24318
|
-
}
|
|
24319
|
-
async function checkAgentResources(input) {
|
|
24320
|
-
const resources = [];
|
|
24321
|
-
for (const path of AGENT_RESOURCE_PATHS) {
|
|
24322
|
-
if (input.signal?.aborted) break;
|
|
24323
|
-
resources.push(await checkAgentResource({
|
|
24324
|
-
baseUrl: input.url,
|
|
24325
|
-
path,
|
|
24326
|
-
timeoutMs: input.timeoutMs,
|
|
24327
|
-
fetch: input.fetch,
|
|
24328
|
-
signal: input.signal
|
|
24329
|
-
}));
|
|
24549
|
+
});
|
|
24330
24550
|
}
|
|
24331
24551
|
return resources;
|
|
24332
24552
|
}
|
|
24333
|
-
function
|
|
24334
|
-
|
|
24553
|
+
async function checkMarkdownNegotiation(input) {
|
|
24554
|
+
const result = await get(input, input.url, { accept: "text/markdown" }, false);
|
|
24555
|
+
const vary = result.headers?.get("vary") ?? "";
|
|
24556
|
+
return {
|
|
24557
|
+
url: result.url,
|
|
24558
|
+
availability: result.availability,
|
|
24559
|
+
...result.status === void 0 ? {} : { status: result.status },
|
|
24560
|
+
...result.contentType ? { contentType: result.contentType } : {},
|
|
24561
|
+
markdown: result.availability === "available" && isMarkdown(result.contentType),
|
|
24562
|
+
varyAccept: vary.split(",").some((value) => value.trim().toLowerCase() === "accept"),
|
|
24563
|
+
...result.error ? { error: result.error } : {}
|
|
24564
|
+
};
|
|
24565
|
+
}
|
|
24566
|
+
async function collectCrawlAiSignals(input) {
|
|
24567
|
+
const robots = await checkRobotsAiAccess(input);
|
|
24568
|
+
const allows = input.respectRobots ? robots.allows : () => true;
|
|
24569
|
+
const startAllowed = allows(input.url);
|
|
24570
|
+
const [llmsTxt, agentResources, markdownNegotiation] = await Promise.all([
|
|
24335
24571
|
checkLlmsTxt(input),
|
|
24336
|
-
|
|
24337
|
-
|
|
24572
|
+
checkAgentResources(input),
|
|
24573
|
+
startAllowed ? checkMarkdownNegotiation(input) : Promise.resolve({
|
|
24574
|
+
url: input.url,
|
|
24575
|
+
availability: "unreachable",
|
|
24576
|
+
markdown: false,
|
|
24577
|
+
varyAccept: false,
|
|
24578
|
+
error: startAllowed === null ? "Not requested: robots.txt could not be read." : "Not requested: robots.txt disallows the start URL."
|
|
24579
|
+
})
|
|
24338
24580
|
]);
|
|
24581
|
+
return {
|
|
24582
|
+
signals: {
|
|
24583
|
+
llmsTxt,
|
|
24584
|
+
robotsTxt: robots.robotsTxt,
|
|
24585
|
+
agentResources,
|
|
24586
|
+
markdownNegotiation
|
|
24587
|
+
},
|
|
24588
|
+
allows
|
|
24589
|
+
};
|
|
24590
|
+
}
|
|
24591
|
+
async function checkMarkdownCopies(input) {
|
|
24592
|
+
const linked = input.pages.filter((page) => page.markdownAlternates?.length).sort((a, b) => Number(a.public) - Number(b.public) || a.url.localeCompare(b.url));
|
|
24593
|
+
const broken = [];
|
|
24594
|
+
let largest;
|
|
24595
|
+
let checked = 0;
|
|
24596
|
+
for (const page of linked.slice(0, 20)) {
|
|
24597
|
+
if (input.signal?.aborted) break;
|
|
24598
|
+
const url = page.markdownAlternates?.[0];
|
|
24599
|
+
if (!url || !input.allows(url)) continue;
|
|
24600
|
+
checked += 1;
|
|
24601
|
+
const result = await get({
|
|
24602
|
+
...input,
|
|
24603
|
+
url
|
|
24604
|
+
}, url);
|
|
24605
|
+
const link = brokenLink(result, true);
|
|
24606
|
+
if (link) {
|
|
24607
|
+
broken.push({
|
|
24608
|
+
...link,
|
|
24609
|
+
page: page.url,
|
|
24610
|
+
public: page.public
|
|
24611
|
+
});
|
|
24612
|
+
continue;
|
|
24613
|
+
}
|
|
24614
|
+
const estimatedTokens = Math.ceil(result.text.length / 4);
|
|
24615
|
+
if (!largest || estimatedTokens > largest.estimatedTokens) largest = {
|
|
24616
|
+
url: result.url,
|
|
24617
|
+
estimatedTokens
|
|
24618
|
+
};
|
|
24619
|
+
}
|
|
24620
|
+
return {
|
|
24621
|
+
publicPages: input.pages.filter((page) => page.public).length,
|
|
24622
|
+
publicPagesWithLink: linked.filter((page) => page.public).length,
|
|
24623
|
+
privatePagesWithLink: linked.filter((page) => !page.public).length,
|
|
24624
|
+
checked,
|
|
24625
|
+
limit: 20,
|
|
24626
|
+
broken,
|
|
24627
|
+
...largest ? { largest } : {}
|
|
24628
|
+
};
|
|
24629
|
+
}
|
|
24630
|
+
function linkRelations(header) {
|
|
24631
|
+
const relations = /* @__PURE__ */ new Set();
|
|
24632
|
+
for (const match of (header ?? "").matchAll(/rel\s*=\s*"?([^";,]+)"?/gi)) for (const value of (match[1] ?? "").trim().toLowerCase().split(/\s+/)) if (value) relations.add(value);
|
|
24633
|
+
return [...relations].sort();
|
|
24339
24634
|
}
|
|
24340
24635
|
//#endregion
|
|
24341
24636
|
//#region packages/core/dist/analyze/crawler/site-crawl-providers.js
|
|
@@ -26743,12 +27038,11 @@ async function crawlSite(input, dependencies) {
|
|
|
26743
27038
|
url: config.url,
|
|
26744
27039
|
timeoutMs: config.timeoutMs,
|
|
26745
27040
|
fetch: deps.fetch,
|
|
27041
|
+
respectRobots: config.respectRobots,
|
|
26746
27042
|
signal
|
|
26747
27043
|
});
|
|
26748
|
-
const llmsTxt = aiSignals?.
|
|
27044
|
+
const llmsTxt = aiSignals?.signals.llmsTxt;
|
|
26749
27045
|
const healthSitemapUrls = healthStrategy && !config.sitemapUrl && !isCancelled() ? await robotsSession?.sitemapUrls(origin) : void 0;
|
|
26750
|
-
const robotsTxt = aiSignals?.[1];
|
|
26751
|
-
const agentResources = aiSignals?.[2];
|
|
26752
27046
|
cancelled = isCancelled();
|
|
26753
27047
|
const enqueue = (value, depth) => {
|
|
26754
27048
|
const normalized = normalizeUrl$1(value, config.url);
|
|
@@ -26826,7 +27120,7 @@ async function crawlSite(input, dependencies) {
|
|
|
26826
27120
|
url: config.url,
|
|
26827
27121
|
sitemapUrl: config.sitemapUrl,
|
|
26828
27122
|
maxPages: config.maxPages,
|
|
26829
|
-
declaredSitemapUrls: robotsTxt?.sitemapUrls ?? healthSitemapUrls ?? [],
|
|
27123
|
+
declaredSitemapUrls: aiSignals?.signals.robotsTxt?.sitemapUrls ?? healthSitemapUrls ?? [],
|
|
26830
27124
|
warnings,
|
|
26831
27125
|
fetchSitemapUrls: deps.fetchSitemapUrls
|
|
26832
27126
|
});
|
|
@@ -27142,6 +27436,26 @@ async function crawlSite(input, dependencies) {
|
|
|
27142
27436
|
llmsTxtStatus: llmsTxt.status
|
|
27143
27437
|
};
|
|
27144
27438
|
}
|
|
27439
|
+
const startPage = pages.find((page) => page.url === config.url || page.finalUrl === config.url);
|
|
27440
|
+
const ai = aiSignals && !cancelled ? {
|
|
27441
|
+
...aiSignals.signals,
|
|
27442
|
+
markdownCopies: await checkMarkdownCopies({
|
|
27443
|
+
allows: aiSignals.allows,
|
|
27444
|
+
pages: pages.map((page) => ({
|
|
27445
|
+
url: page.url,
|
|
27446
|
+
markdownAlternates: page.markdownAlternates,
|
|
27447
|
+
public: isPublicPage(page)
|
|
27448
|
+
})),
|
|
27449
|
+
timeoutMs: config.timeoutMs,
|
|
27450
|
+
fetch: deps.fetch,
|
|
27451
|
+
signal
|
|
27452
|
+
}),
|
|
27453
|
+
...startPage ? { startPageHeaders: {
|
|
27454
|
+
url: startPage.url,
|
|
27455
|
+
linkRelations: linkRelations(startPage.responseHeaders?.link),
|
|
27456
|
+
...startPage.responseHeaders?.["content-signal"] ? { contentSignal: startPage.responseHeaders["content-signal"] } : {}
|
|
27457
|
+
} } : {}
|
|
27458
|
+
} : aiSignals?.signals;
|
|
27145
27459
|
const report = createCrawlReport({
|
|
27146
27460
|
config,
|
|
27147
27461
|
site,
|
|
@@ -27151,17 +27465,15 @@ async function crawlSite(input, dependencies) {
|
|
|
27151
27465
|
pages,
|
|
27152
27466
|
requests,
|
|
27153
27467
|
requestEvidenceStatus,
|
|
27154
|
-
|
|
27155
|
-
...llmsTxt ? { llmsTxt } : {},
|
|
27156
|
-
...robotsTxt ? { robotsTxt } : {},
|
|
27157
|
-
...agentResources ? { agentResources } : {}
|
|
27158
|
-
} } : {},
|
|
27468
|
+
...ai ? { ai } : {},
|
|
27159
27469
|
...sitemapDiscovery ? { sitemapDiscovery } : {},
|
|
27160
27470
|
...externalLinkVerification ? { externalLinkVerification } : {},
|
|
27161
|
-
...siteCheckResult ? {
|
|
27162
|
-
|
|
27163
|
-
|
|
27164
|
-
|
|
27471
|
+
...siteCheckResult ? { siteChecks: siteCheckResult.checks } : {},
|
|
27472
|
+
additionalIssues: [...siteCheckResult?.issues ?? [], ...ai ? agentReadyIssues({
|
|
27473
|
+
startUrl: config.url,
|
|
27474
|
+
ai,
|
|
27475
|
+
pages
|
|
27476
|
+
}) : []],
|
|
27165
27477
|
status: reportStatus,
|
|
27166
27478
|
warnings,
|
|
27167
27479
|
caveats: crawlCaveats({
|
|
@@ -31571,9 +31883,9 @@ function strikingDistance(input, dependencies = defaultDependencies$5) {
|
|
|
31571
31883
|
//#region packages/core/dist/updates/search-status.js
|
|
31572
31884
|
const INCIDENTS_URL = "https://status.search.google.com/incidents.json";
|
|
31573
31885
|
async function fetchSearchStatusIncidents() {
|
|
31574
|
-
const response = await fetch$1(INCIDENTS_URL);
|
|
31886
|
+
const response = await fetch$1(INCIDENTS_URL, { signal: AbortSignal.timeout(8e3) });
|
|
31575
31887
|
if (!response.ok) throw new SeoError("OPTIONAL_PROVIDER_UNAVAILABLE", `Google Search Status fetch failed with ${response.status}.`);
|
|
31576
|
-
return
|
|
31888
|
+
return readJson(response, "Google Search Status", { maxBytes: 10 * 1024 * 1024 });
|
|
31577
31889
|
}
|
|
31578
31890
|
async function listSearchUpdates(input = {}) {
|
|
31579
31891
|
return searchUpdatesFromIncidents(await fetchSearchStatusIncidents(), input);
|
|
@@ -34856,7 +35168,7 @@ async function fetchCruxFieldData(input) {
|
|
|
34856
35168
|
headers: { "content-type": "application/json" },
|
|
34857
35169
|
signal: controller.signal
|
|
34858
35170
|
});
|
|
34859
|
-
const json = await response
|
|
35171
|
+
const json = await readJson(response, "CrUX").catch(() => ({}));
|
|
34860
35172
|
if (response.status === 404) return {
|
|
34861
35173
|
response,
|
|
34862
35174
|
json
|
|
@@ -37585,6 +37897,340 @@ async function runAnalyticsRows(input, google, options) {
|
|
|
37585
37897
|
};
|
|
37586
37898
|
}
|
|
37587
37899
|
//#endregion
|
|
37900
|
+
//#region packages/core/dist/ga4/matching.js
|
|
37901
|
+
function hostnameFromUrl(value) {
|
|
37902
|
+
try {
|
|
37903
|
+
const url = new URL(value);
|
|
37904
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
37905
|
+
return url.hostname.toLowerCase().replace(/\.$/, "") || void 0;
|
|
37906
|
+
} catch {
|
|
37907
|
+
return;
|
|
37908
|
+
}
|
|
37909
|
+
}
|
|
37910
|
+
function domainFromProperty(value) {
|
|
37911
|
+
if (!value.startsWith("sc-domain:")) return void 0;
|
|
37912
|
+
const domain = value.slice(10).trim().toLowerCase();
|
|
37913
|
+
if (!domain || /[/:]/.test(domain)) return void 0;
|
|
37914
|
+
return domain.replace(/\.$/, "") || void 0;
|
|
37915
|
+
}
|
|
37916
|
+
function matchForSite(site, streamHostname) {
|
|
37917
|
+
const domain = domainFromProperty(site);
|
|
37918
|
+
if (domain) return streamHostname === domain || streamHostname.endsWith(`.${domain}`) ? "domain" : void 0;
|
|
37919
|
+
return hostnameFromUrl(site) === streamHostname ? "hostname" : void 0;
|
|
37920
|
+
}
|
|
37921
|
+
function matchGa4WebStreams(site, candidates) {
|
|
37922
|
+
return candidates.flatMap((candidate) => {
|
|
37923
|
+
const defaultUri = candidate.stream.webStreamData?.defaultUri;
|
|
37924
|
+
if (!defaultUri) return [];
|
|
37925
|
+
const hostname = hostnameFromUrl(defaultUri);
|
|
37926
|
+
if (!hostname) return [];
|
|
37927
|
+
const match = matchForSite(site, hostname);
|
|
37928
|
+
if (!match) return [];
|
|
37929
|
+
return [{
|
|
37930
|
+
...candidate,
|
|
37931
|
+
defaultUri,
|
|
37932
|
+
hostname,
|
|
37933
|
+
match
|
|
37934
|
+
}];
|
|
37935
|
+
});
|
|
37936
|
+
}
|
|
37937
|
+
//#endregion
|
|
37938
|
+
//#region packages/core/dist/commands/analytics-site.js
|
|
37939
|
+
const STREAM_CHECKS = 30;
|
|
37940
|
+
const AT_ONCE = 5;
|
|
37941
|
+
const TIED = 3;
|
|
37942
|
+
const hostOf$1 = (site) => {
|
|
37943
|
+
const value = site.replace(/^sc-domain:/, "");
|
|
37944
|
+
try {
|
|
37945
|
+
return new URL(value.includes("://") ? value : `https://${value}`).hostname;
|
|
37946
|
+
} catch {
|
|
37947
|
+
return value;
|
|
37948
|
+
}
|
|
37949
|
+
};
|
|
37950
|
+
const plain = (value) => value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
37951
|
+
function ranked(site, candidates) {
|
|
37952
|
+
const host = hostOf$1(site).replace(/^www\./, "");
|
|
37953
|
+
const words = [plain(host), plain(host.split(".")[0] ?? "")].filter((word) => word.length >= 3);
|
|
37954
|
+
const named = (candidate) => words.some((word) => plain(candidate.propertyName).includes(word)) ? 0 : 1;
|
|
37955
|
+
return [...candidates].sort((a, b) => named(a) - named(b) || a.property.localeCompare(b.property, "en", { numeric: true }));
|
|
37956
|
+
}
|
|
37957
|
+
function best(site, matches) {
|
|
37958
|
+
const host = hostOf$1(site).replace(/^www\./, "");
|
|
37959
|
+
const score = (match) => match.hostname === host || match.hostname === `www.${host}` ? 0 : 1;
|
|
37960
|
+
return [...matches].sort((a, b) => score(a) - score(b) || a.hostname.length - b.hostname.length || a.property.localeCompare(b.property, "en", { numeric: true }));
|
|
37961
|
+
}
|
|
37962
|
+
async function findProperty(access, site) {
|
|
37963
|
+
const accounts = await access.accounts();
|
|
37964
|
+
const listed = await Promise.all(accounts.map((account) => listAnalyticsAccounts(account.google, { signIn: access.signIn }).then((summaries) => ({
|
|
37965
|
+
account,
|
|
37966
|
+
summaries
|
|
37967
|
+
}), (error) => error instanceof Error ? error : new Error(String(error)))));
|
|
37968
|
+
const seen = /* @__PURE__ */ new Set();
|
|
37969
|
+
const candidates = [];
|
|
37970
|
+
for (const entry of listed) {
|
|
37971
|
+
if (entry instanceof Error) continue;
|
|
37972
|
+
for (const summary of entry.summaries) for (const item of summary.propertySummaries) {
|
|
37973
|
+
const property = item.property.replace(/^properties\//, "");
|
|
37974
|
+
if (seen.has(property) || !/^\d+$/.test(property)) continue;
|
|
37975
|
+
seen.add(property);
|
|
37976
|
+
candidates.push({
|
|
37977
|
+
account: summary.account,
|
|
37978
|
+
property,
|
|
37979
|
+
propertyName: item.displayName ?? property,
|
|
37980
|
+
google: entry.account.google
|
|
37981
|
+
});
|
|
37982
|
+
}
|
|
37983
|
+
}
|
|
37984
|
+
if (!candidates.length) {
|
|
37985
|
+
const failed = listed.find((entry) => entry instanceof Error);
|
|
37986
|
+
return failed ? `could not be read. ${failed.message}` : "the connected Google accounts cannot read any property.";
|
|
37987
|
+
}
|
|
37988
|
+
const order = ranked(site, candidates).slice(0, STREAM_CHECKS);
|
|
37989
|
+
let failures = 0;
|
|
37990
|
+
for (let start = 0; start < order.length; start += AT_ONCE) {
|
|
37991
|
+
const batch = order.slice(start, start + AT_ONCE);
|
|
37992
|
+
const streams = await Promise.all(batch.map((candidate) => listAnalyticsDataStreams(candidate.google, candidate.property, { signIn: access.signIn }).catch(() => {
|
|
37993
|
+
failures += 1;
|
|
37994
|
+
return [];
|
|
37995
|
+
})));
|
|
37996
|
+
const matches = best(site, batch.flatMap((candidate, index) => matchGa4WebStreams(site, (streams[index] ?? []).map((stream) => ({
|
|
37997
|
+
...candidate,
|
|
37998
|
+
stream
|
|
37999
|
+
})))));
|
|
38000
|
+
const found = /* @__PURE__ */ new Map();
|
|
38001
|
+
for (const match of matches) {
|
|
38002
|
+
const candidate = batch.find((item) => item.property === match.property);
|
|
38003
|
+
if (!candidate || found.has(match.property)) continue;
|
|
38004
|
+
found.set(match.property, {
|
|
38005
|
+
property: match.property,
|
|
38006
|
+
name: match.propertyName,
|
|
38007
|
+
stream: match.defaultUri,
|
|
38008
|
+
google: candidate.google
|
|
38009
|
+
});
|
|
38010
|
+
}
|
|
38011
|
+
if (found.size) return [...found.values()].slice(0, TIED);
|
|
38012
|
+
}
|
|
38013
|
+
return `no web stream matches ${site} in ${order.length < candidates.length ? `the first ${order.length} of ${candidates.length} properties, named like the site first` : `${candidates.length} ${candidates.length === 1 ? "property" : "properties"}`}${failures ? ` (${failures} could not be read)` : ""}.`;
|
|
38014
|
+
}
|
|
38015
|
+
const AI_SOURCES = [
|
|
38016
|
+
"chatgpt.com",
|
|
38017
|
+
"chatgpt",
|
|
38018
|
+
"chat.openai.com",
|
|
38019
|
+
"perplexity.ai",
|
|
38020
|
+
"perplexity",
|
|
38021
|
+
"claude.ai",
|
|
38022
|
+
"gemini.google.com",
|
|
38023
|
+
"copilot.microsoft.com",
|
|
38024
|
+
"copilot.com",
|
|
38025
|
+
"chat.deepseek.com",
|
|
38026
|
+
"meta.ai",
|
|
38027
|
+
"grok.com",
|
|
38028
|
+
"chat.mistral.ai",
|
|
38029
|
+
"you.com",
|
|
38030
|
+
"phind.com",
|
|
38031
|
+
"poe.com"
|
|
38032
|
+
];
|
|
38033
|
+
const AI_SOURCE_PATTERN = `(www\\.)?(${AI_SOURCES.map((source) => source.replaceAll(".", "\\.")).join("|")})`;
|
|
38034
|
+
const AI_ROWS = 5;
|
|
38035
|
+
const filter = (fieldName, value, matchType = "EXACT") => ({ filter: {
|
|
38036
|
+
fieldName,
|
|
38037
|
+
stringFilter: {
|
|
38038
|
+
matchType,
|
|
38039
|
+
value
|
|
38040
|
+
}
|
|
38041
|
+
} });
|
|
38042
|
+
async function totals$1(found, access, range, before, pages) {
|
|
38043
|
+
const read = (dimensions, filters) => runAnalyticsReport(found.google, found.property, {
|
|
38044
|
+
dateRanges: [{
|
|
38045
|
+
...range,
|
|
38046
|
+
name: "now"
|
|
38047
|
+
}, {
|
|
38048
|
+
...before,
|
|
38049
|
+
name: "before"
|
|
38050
|
+
}],
|
|
38051
|
+
dimensions: dimensions.map((name) => ({ name })),
|
|
38052
|
+
metrics: [{ name: "sessions" }, { name: "keyEvents" }],
|
|
38053
|
+
...filters.length ? { dimensionFilter: { andGroup: { expressions: filters } } } : {},
|
|
38054
|
+
limit: 100
|
|
38055
|
+
}, { signIn: access.signIn });
|
|
38056
|
+
const onPages = pages ? [filter("landingPage", pages, "CONTAINS")] : [];
|
|
38057
|
+
const fromAi = [...onPages, filter("sessionSource", AI_SOURCE_PATTERN, "FULL_REGEXP")];
|
|
38058
|
+
const [channels, google, aiSources, aiPages] = await Promise.all([
|
|
38059
|
+
read(["sessionDefaultChannelGroup"], onPages),
|
|
38060
|
+
read([], [
|
|
38061
|
+
...onPages,
|
|
38062
|
+
filter("sessionDefaultChannelGroup", "Organic Search"),
|
|
38063
|
+
filter("sessionSource", "google")
|
|
38064
|
+
]),
|
|
38065
|
+
read(["sessionSource"], fromAi),
|
|
38066
|
+
read(["landingPage"], fromAi)
|
|
38067
|
+
]);
|
|
38068
|
+
const sum = () => ({
|
|
38069
|
+
google: {
|
|
38070
|
+
sessions: 0,
|
|
38071
|
+
keyEvents: 0
|
|
38072
|
+
},
|
|
38073
|
+
organic: {
|
|
38074
|
+
sessions: 0,
|
|
38075
|
+
keyEvents: 0
|
|
38076
|
+
},
|
|
38077
|
+
all: {
|
|
38078
|
+
sessions: 0,
|
|
38079
|
+
keyEvents: 0
|
|
38080
|
+
}
|
|
38081
|
+
});
|
|
38082
|
+
const periods = {
|
|
38083
|
+
now: sum(),
|
|
38084
|
+
before: sum()
|
|
38085
|
+
};
|
|
38086
|
+
const each = (data, add) => {
|
|
38087
|
+
const periodAt = (data.dimensionHeaders?.map((header) => header.name) ?? []).indexOf("dateRange");
|
|
38088
|
+
for (const row of data.rows ?? []) {
|
|
38089
|
+
const values = row.dimensionValues ?? [];
|
|
38090
|
+
add((periodAt >= 0 ? values[periodAt] : values.at(-1))?.value === "before" ? periods.before : periods.now, values[0]?.value ?? "", {
|
|
38091
|
+
sessions: Number(row.metricValues?.[0]?.value ?? 0),
|
|
38092
|
+
keyEvents: Number(row.metricValues?.[1]?.value ?? 0)
|
|
38093
|
+
});
|
|
38094
|
+
}
|
|
38095
|
+
};
|
|
38096
|
+
const plus = (into, row) => {
|
|
38097
|
+
into.sessions += row.sessions;
|
|
38098
|
+
into.keyEvents += row.keyEvents;
|
|
38099
|
+
};
|
|
38100
|
+
each(channels, (period, channel, row) => {
|
|
38101
|
+
plus(period.all, row);
|
|
38102
|
+
if (channel === "Organic Search") plus(period.organic, row);
|
|
38103
|
+
});
|
|
38104
|
+
each(google, (period, _, row) => plus(period.google, row));
|
|
38105
|
+
const byName = (data, key) => {
|
|
38106
|
+
const rows = /* @__PURE__ */ new Map();
|
|
38107
|
+
const periodAt = (data.dimensionHeaders?.map((header) => header.name) ?? []).indexOf("dateRange");
|
|
38108
|
+
for (const row of data.rows ?? []) {
|
|
38109
|
+
const values = row.dimensionValues ?? [];
|
|
38110
|
+
const name = values[0]?.value ?? "";
|
|
38111
|
+
const before = (periodAt >= 0 ? values[periodAt] : values.at(-1))?.value === "before";
|
|
38112
|
+
const entry = rows.get(name) ?? {
|
|
38113
|
+
now: 0,
|
|
38114
|
+
before: 0,
|
|
38115
|
+
keyEvents: 0
|
|
38116
|
+
};
|
|
38117
|
+
const sessions = Number(row.metricValues?.[0]?.value ?? 0);
|
|
38118
|
+
if (before) entry.before += sessions;
|
|
38119
|
+
else {
|
|
38120
|
+
entry.now += sessions;
|
|
38121
|
+
entry.keyEvents += Number(row.metricValues?.[1]?.value ?? 0);
|
|
38122
|
+
}
|
|
38123
|
+
rows.set(name, entry);
|
|
38124
|
+
}
|
|
38125
|
+
return [...rows].map(([name, entry]) => ({
|
|
38126
|
+
[key]: name,
|
|
38127
|
+
sessions: entry.now,
|
|
38128
|
+
previousSessions: entry.before,
|
|
38129
|
+
keyEvents: entry.keyEvents
|
|
38130
|
+
})).sort((a, b) => b.sessions - a.sessions || b.previousSessions - a.previousSessions || String(a[key]).localeCompare(String(b[key])));
|
|
38131
|
+
};
|
|
38132
|
+
const bySource = byName(aiSources, "source");
|
|
38133
|
+
const aiNow = bySource.reduce((total, row) => total + row.sessions, 0);
|
|
38134
|
+
const aiBefore = bySource.reduce((total, row) => total + row.previousSessions, 0);
|
|
38135
|
+
const data = { metadata: {
|
|
38136
|
+
timeZone: channels.metadata?.timeZone,
|
|
38137
|
+
subjectToThresholding: [
|
|
38138
|
+
channels,
|
|
38139
|
+
google,
|
|
38140
|
+
aiSources,
|
|
38141
|
+
aiPages
|
|
38142
|
+
].some((data) => data.metadata?.subjectToThresholding),
|
|
38143
|
+
dataLossFromOtherRow: [
|
|
38144
|
+
channels,
|
|
38145
|
+
google,
|
|
38146
|
+
aiSources,
|
|
38147
|
+
aiPages
|
|
38148
|
+
].some((data) => data.metadata?.dataLossFromOtherRow)
|
|
38149
|
+
} };
|
|
38150
|
+
const notes = [];
|
|
38151
|
+
if (data.metadata?.timeZone && data.metadata.timeZone !== "America/Los_Angeles") notes.push(`Google Analytics days use the property's time zone, ${data.metadata.timeZone}; Search Console days use Pacific time.`);
|
|
38152
|
+
if (data.metadata?.subjectToThresholding) notes.push("Google Analytics applied thresholding; small rows can be hidden.");
|
|
38153
|
+
if (data.metadata?.dataLossFromOtherRow) notes.push("Google Analytics grouped some rows into (other).");
|
|
38154
|
+
return {
|
|
38155
|
+
property: found.property,
|
|
38156
|
+
propertyName: found.name,
|
|
38157
|
+
webStream: found.stream,
|
|
38158
|
+
...pages ? { landingPageContains: pages } : {},
|
|
38159
|
+
organicSearchFromGoogle: {
|
|
38160
|
+
sessions: periods.now.google.sessions,
|
|
38161
|
+
previousSessions: periods.before.google.sessions,
|
|
38162
|
+
sessionsChangePct: pct(periods.now.google.sessions, periods.before.google.sessions)
|
|
38163
|
+
},
|
|
38164
|
+
organicSearch: {
|
|
38165
|
+
sessions: periods.now.organic.sessions,
|
|
38166
|
+
previousSessions: periods.before.organic.sessions,
|
|
38167
|
+
sessionsChangePct: pct(periods.now.organic.sessions, periods.before.organic.sessions),
|
|
38168
|
+
keyEvents: periods.now.organic.keyEvents,
|
|
38169
|
+
previousKeyEvents: periods.before.organic.keyEvents
|
|
38170
|
+
},
|
|
38171
|
+
allChannels: {
|
|
38172
|
+
sessions: periods.now.all.sessions,
|
|
38173
|
+
previousSessions: periods.before.all.sessions,
|
|
38174
|
+
sessionsChangePct: pct(periods.now.all.sessions, periods.before.all.sessions)
|
|
38175
|
+
},
|
|
38176
|
+
aiAssistants: {
|
|
38177
|
+
sessions: aiNow,
|
|
38178
|
+
previousSessions: aiBefore,
|
|
38179
|
+
sessionsChangePct: pct(aiNow, aiBefore),
|
|
38180
|
+
bySource,
|
|
38181
|
+
topLandingPages: byName(aiPages, "page").slice(0, 20),
|
|
38182
|
+
sources: AI_SOURCES
|
|
38183
|
+
},
|
|
38184
|
+
notes
|
|
38185
|
+
};
|
|
38186
|
+
}
|
|
38187
|
+
const change$1 = (value) => value === null ? "n/a" : `${value > 0 ? "+" : ""}${value}%`;
|
|
38188
|
+
function aiText(ai) {
|
|
38189
|
+
const list = (rows, key) => rows.slice(0, AI_ROWS).map((row) => `${row[key]} ${row.sessions} (was ${row.previousSessions})`).join(", ");
|
|
38190
|
+
if (!ai.sessions && !ai.previousSessions) return "Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): none in either period.";
|
|
38191
|
+
return [
|
|
38192
|
+
`Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): ${ai.sessions} sessions (was ${ai.previousSessions}, ${change$1(ai.sessionsChangePct)}). By source: ${list(ai.bySource, "source")}.`,
|
|
38193
|
+
ai.topLandingPages.length ? `Top landing pages from AI assistants: ${list(ai.topLandingPages, "page")}.` : "",
|
|
38194
|
+
"Only visits that pass a referrer or utm_source count; AI apps that send none show as Direct."
|
|
38195
|
+
].filter(Boolean).join(" ");
|
|
38196
|
+
}
|
|
38197
|
+
async function analyticsBeside(access, site, range, before, pages) {
|
|
38198
|
+
if (!access) return null;
|
|
38199
|
+
try {
|
|
38200
|
+
const found = await findProperty(access, site);
|
|
38201
|
+
if (typeof found === "string") return {
|
|
38202
|
+
text: `Google Analytics: ${found}`,
|
|
38203
|
+
data: { error: found }
|
|
38204
|
+
};
|
|
38205
|
+
const read = await Promise.all(found.map((item) => totals$1(item, access, range, before, pages).catch((error) => error instanceof Error ? error : new Error(String(error)))));
|
|
38206
|
+
const data = [...read.filter((item) => !(item instanceof Error))].sort((a, b) => b.allChannels.sessions - a.allChannels.sessions)[0];
|
|
38207
|
+
if (!data) throw read[0];
|
|
38208
|
+
const others = found.filter((item) => item.property !== data.property).map((item) => `${item.name} (${item.property})`);
|
|
38209
|
+
const google = data.organicSearchFromGoogle;
|
|
38210
|
+
const organic = data.organicSearch;
|
|
38211
|
+
const all = data.allChannels;
|
|
38212
|
+
const keyEvents = organic.keyEvents || organic.previousKeyEvents ? `, Organic Search key events ${organic.keyEvents} (was ${organic.previousKeyEvents})` : "";
|
|
38213
|
+
return {
|
|
38214
|
+
text: [
|
|
38215
|
+
`Google Analytics, property ${data.propertyName} (${data.property}), matched by its web stream ${data.webStream}${pages ? `, landing pages whose path contains "${pages}"` : ""}: Organic Search sessions from Google ${google.sessions} (was ${google.previousSessions}, ${change$1(google.sessionsChangePct)}), from every search engine ${organic.sessions} (was ${organic.previousSessions}, ${change$1(organic.sessionsChangePct)})${keyEvents}, sessions from all channels ${all.sessions} (was ${all.previousSessions}, ${change$1(all.sessionsChangePct)}).`,
|
|
38216
|
+
"Sessions are Google Analytics’ own count, not Search Console clicks.",
|
|
38217
|
+
...data.notes,
|
|
38218
|
+
...others.length ? [`Other properties with a matching web stream, with fewer sessions: ${others.join(", ")}.`] : []
|
|
38219
|
+
].join(" ").concat("\n\n", aiText(data.aiAssistants)),
|
|
38220
|
+
data: {
|
|
38221
|
+
...data,
|
|
38222
|
+
...others.length ? { otherMatches: others } : {}
|
|
38223
|
+
}
|
|
38224
|
+
};
|
|
38225
|
+
} catch (error) {
|
|
38226
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
38227
|
+
return {
|
|
38228
|
+
text: `Google Analytics: could not be read. ${message}`,
|
|
38229
|
+
data: { error: message }
|
|
38230
|
+
};
|
|
38231
|
+
}
|
|
38232
|
+
}
|
|
38233
|
+
//#endregion
|
|
37588
38234
|
//#region packages/core/dist/commands/bing.js
|
|
37589
38235
|
const BING_LIMITS = ["Bing and Google count clicks, impressions and positions their own way. Their numbers sit side by side here and are never added together.", "Bing lists only its top queries and pages, grouped by week. A query or page missing from Bing’s list is unknown, not zero."];
|
|
37590
38236
|
const CLI_LINK = "If you have a shell, link the CLI to the same account once: run `npx -y seo login`, show the person the link it prints, and after they approve it run `npx -y seo login` again. `npx -y seo` commands then use the same Google accounts, Bing and keys.";
|
|
@@ -37987,7 +38633,7 @@ async function lookup(domain, key) {
|
|
|
37987
38633
|
signal: AbortSignal.timeout(5e3)
|
|
37988
38634
|
});
|
|
37989
38635
|
if (!response.ok) return null;
|
|
37990
|
-
const body = await response.clone()
|
|
38636
|
+
const body = await readJson(response.clone(), "Ahrefs Domain Rating");
|
|
37991
38637
|
if (cache && !cached) {
|
|
37992
38638
|
const copy = new Response(response.body, response);
|
|
37993
38639
|
copy.headers.set("cache-control", `max-age=${TTL_SECONDS}`);
|
|
@@ -38146,7 +38792,7 @@ function temporaryRedirectNote(pages) {
|
|
|
38146
38792
|
//#endregion
|
|
38147
38793
|
//#region packages/core/dist/commands/report.js
|
|
38148
38794
|
const SECTIONS = {
|
|
38149
|
-
overview: "Site totals (clicks, impressions, CTR, position) against the previous period, by device, with the top pages and queries and their change.",
|
|
38795
|
+
overview: "Site totals (clicks, impressions, CTR, position) against the previous period, by device, with the top pages and queries and their change, and Google Analytics sessions for the same dates from the property whose web stream matches the site.",
|
|
38150
38796
|
"quick-wins": "Query and page pairs at average positions 4 to 10 with a click-through rate below the site’s own curve for that position.",
|
|
38151
38797
|
"low-ctr": "Query and page pairs at average positions 1 to 4 with a click-through rate below the site’s own curve: titles and snippets to review with the person.",
|
|
38152
38798
|
"striking-distance": "Queries just outside the top positions, with impressions, for pages that already rank.",
|
|
@@ -38453,10 +39099,14 @@ async function allSections(input, deps) {
|
|
|
38453
39099
|
const origin = originOf(input.site);
|
|
38454
39100
|
const bing = once(() => bingFor(deps, input.site));
|
|
38455
39101
|
const ids = SECTION_IDS.filter((id) => id !== "bing" || deps.bing);
|
|
39102
|
+
const range = dateWindow(input, DEFAULT_DAYS.overview);
|
|
39103
|
+
const analytics = analyticsBeside(deps.analytics, input.site, range, earlierRange(range));
|
|
38456
39104
|
const [overview, ...rest] = await Promise.all(ids.map((id) => build(read, input.site, id, dateWindow(input, DEFAULT_DAYS[id]), now, bing).catch((error) => error instanceof Error ? error : new Error(String(error)))));
|
|
38457
39105
|
if (!overview || overview instanceof Error) throw overview ?? /* @__PURE__ */ new Error("The overview did not run.");
|
|
38458
39106
|
const parts = [overviewText(overview, `Report on ${input.site}`, origin, 5)];
|
|
38459
39107
|
parts.push(deps.bing ? await enginesLine(overview, bing) : NO_BING_HERE);
|
|
39108
|
+
const ga = await analytics;
|
|
39109
|
+
if (ga) parts.push(ga.text);
|
|
38460
39110
|
const brand = rest.find((run) => !(run instanceof Error) && brandNote(run) !== null);
|
|
38461
39111
|
if (brand) parts.push(brandNote(brand));
|
|
38462
39112
|
const sections = {};
|
|
@@ -38486,6 +39136,7 @@ async function allSections(input, deps) {
|
|
|
38486
39136
|
data: {
|
|
38487
39137
|
site: input.site,
|
|
38488
39138
|
overview,
|
|
39139
|
+
...ga ? { analytics: ga.data } : {},
|
|
38489
39140
|
sections
|
|
38490
39141
|
}
|
|
38491
39142
|
};
|
|
@@ -38499,14 +39150,25 @@ async function oneSection(input, deps) {
|
|
|
38499
39150
|
const bing = once(() => bingFor(deps, site));
|
|
38500
39151
|
let report = await build(sharedReader(deps.searchAnalytics), site, section, range, now, bing);
|
|
38501
39152
|
const next = nextFor(section, deps.call);
|
|
38502
|
-
if (section === "overview"
|
|
38503
|
-
|
|
38504
|
-
|
|
38505
|
-
|
|
38506
|
-
|
|
38507
|
-
|
|
38508
|
-
|
|
38509
|
-
|
|
39153
|
+
if (section === "overview") {
|
|
39154
|
+
const ga = await analyticsBeside(deps.analytics, site, range, earlierRange(range));
|
|
39155
|
+
if (ga) report.analytics = ga.data;
|
|
39156
|
+
if (detail !== "full") return {
|
|
39157
|
+
text: [
|
|
39158
|
+
overviewText(report, `overview of ${site}`, origin, 10),
|
|
39159
|
+
deps.bing ? await enginesLine(report, bing) : NO_BING_HERE,
|
|
39160
|
+
ga?.text,
|
|
39161
|
+
`More: ${deps.call("report", {
|
|
39162
|
+
site,
|
|
39163
|
+
section,
|
|
39164
|
+
detail: "full"
|
|
39165
|
+
})} for every row.`,
|
|
39166
|
+
next,
|
|
39167
|
+
FACTS_ONLY
|
|
39168
|
+
].filter(Boolean).join("\n\n"),
|
|
39169
|
+
data: report
|
|
39170
|
+
};
|
|
39171
|
+
}
|
|
38510
39172
|
let checked = [];
|
|
38511
39173
|
const merged = [];
|
|
38512
39174
|
if (PAGE_CHECKS.has(section)) {
|
|
@@ -38614,7 +39276,7 @@ async function dataForSeo([login, password], input) {
|
|
|
38614
39276
|
language_code: language ?? mainLanguage(input.country)
|
|
38615
39277
|
};
|
|
38616
39278
|
const post = async (path, task) => {
|
|
38617
|
-
const result = (await (await call("dataforseo", new Request(`https://api.dataforseo.com/${path}`, {
|
|
39279
|
+
const result = (await readJson(await call("dataforseo", new Request(`https://api.dataforseo.com/${path}`, {
|
|
38618
39280
|
method: "POST",
|
|
38619
39281
|
headers: {
|
|
38620
39282
|
authorization: `Basic ${btoa(`${login}:${password}`)}`,
|
|
@@ -38624,7 +39286,7 @@ async function dataForSeo([login, password], input) {
|
|
|
38624
39286
|
...market,
|
|
38625
39287
|
...task
|
|
38626
39288
|
}])
|
|
38627
|
-
})))
|
|
39289
|
+
})), "DataForSEO")).tasks?.[0];
|
|
38628
39290
|
if (!result || result.status_code >= 4e4) throw new CommandError("PROVIDER_ERROR", `dataforseo: ${result?.status_message ?? "no task result"}`);
|
|
38629
39291
|
return result.result?.[0]?.items ?? [];
|
|
38630
39292
|
};
|
|
@@ -38705,10 +39367,10 @@ async function ahrefs(key, input) {
|
|
|
38705
39367
|
const get = async (path, params) => {
|
|
38706
39368
|
const url = new URL(`https://api.ahrefs.com/v3/${path}`);
|
|
38707
39369
|
for (const [name, value] of Object.entries(params)) url.searchParams.set(name, value);
|
|
38708
|
-
return
|
|
39370
|
+
return readJson(await call("ahrefs", new Request(url, { headers: {
|
|
38709
39371
|
authorization: `Bearer ${key}`,
|
|
38710
39372
|
accept: "application/json"
|
|
38711
|
-
} })))
|
|
39373
|
+
} })), "Ahrefs");
|
|
38712
39374
|
};
|
|
38713
39375
|
const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
38714
39376
|
if (input.action === "keyword_metrics") return ((await get("keywords-explorer/overview", {
|
|
@@ -38996,19 +39658,25 @@ async function runCompare(input, deps) {
|
|
|
38996
39658
|
startDate: shiftIsoDate(input.date, -days),
|
|
38997
39659
|
endDate: shiftIsoDate(input.date, -1)
|
|
38998
39660
|
};
|
|
38999
|
-
const [report, updates] = await Promise.all([
|
|
39000
|
-
|
|
39001
|
-
|
|
39002
|
-
|
|
39661
|
+
const [report, updates, analytics] = await Promise.all([
|
|
39662
|
+
siteOverview(sharedReader(deps.searchAnalytics), input.site, after, before, input.pages),
|
|
39663
|
+
googleUpdates({
|
|
39664
|
+
start: before.startDate,
|
|
39665
|
+
end: after.endDate
|
|
39666
|
+
}),
|
|
39667
|
+
analyticsBeside(deps.analytics, input.site, after, before, input.pages)
|
|
39668
|
+
]);
|
|
39003
39669
|
return {
|
|
39004
39670
|
text: [
|
|
39005
39671
|
overviewText(report, `Compare on ${input.site}: ${days} days after ${input.date}`, originOf(input.site), 10),
|
|
39672
|
+
analytics?.text,
|
|
39006
39673
|
listOfUpdates(updates),
|
|
39007
39674
|
`Next: ${deps.call("page")} for a page that moved.`,
|
|
39008
39675
|
FACTS_ONLY
|
|
39009
39676
|
].filter(Boolean).join("\n\n"),
|
|
39010
39677
|
data: {
|
|
39011
39678
|
...report,
|
|
39679
|
+
...analytics ? { analytics: analytics.data } : {},
|
|
39012
39680
|
googleUpdates: updates
|
|
39013
39681
|
}
|
|
39014
39682
|
};
|
|
@@ -39225,7 +39893,7 @@ const SHARED_TOOLS = [
|
|
|
39225
39893
|
name: "seo_compare",
|
|
39226
39894
|
command: "compare",
|
|
39227
39895
|
title: "Before and after a date",
|
|
39228
|
-
description: "Compares equal periods before and after a date, such as a site change or a Google update: totals, devices, and the pages and queries that moved,
|
|
39896
|
+
description: "Compares equal periods before and after a date, such as a site change or a Google update: totals, devices, and the pages and queries that moved, Google Analytics sessions from the property whose web stream matches the site, and Google updates in the window. Optional pages narrows it to URLs that contain that text, such as /blog/.",
|
|
39229
39897
|
input: z.object({
|
|
39230
39898
|
site,
|
|
39231
39899
|
date: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).describe("YYYY-MM-DD: the first day after the change."),
|
|
@@ -39342,8 +40010,7 @@ async function runSharedTool(tool, surface, input) {
|
|
|
39342
40010
|
const MCP_INSTRUCTIONS = `SEO Skill reads the person's own Search Console, Google Analytics, Bing Webmaster Tools, Clicky and optional keyword accounts, and returns facts with their dates and limits. It gives no advice.
|
|
39343
40011
|
1. seo_sites: the properties the connected accounts can read.
|
|
39344
40012
|
2. seo_report: the main report. Run it without a section first; it shows the overview and every section's count and top rows. Then ask for the one section you need.
|
|
39345
|
-
3. seo_page for one URL, seo_compare for before and after a date, seo_index for pages that lost their clicks, seo_research for keyword, competitor and backlink data from the DataForSEO or Ahrefs key the person saved, seo_data for raw rows, including Google Analytics, Bing and Clicky
|
|
39346
|
-
Show the person the facts that matter, for example in a table or a small dashboard, and decide with them what to change. Do not edit pages without their approval.`;
|
|
40013
|
+
3. seo_page for one URL, seo_compare for before and after a date, seo_index for pages that lost their clicks, seo_research for keyword, competitor and backlink data from the DataForSEO or Ahrefs key the person saved, seo_data for raw rows, including Google Analytics, Bing and Clicky.`;
|
|
39347
40014
|
const mcpCall = (command, args) => {
|
|
39348
40015
|
const name = `seo_${command.replace("-", "_")}`;
|
|
39349
40016
|
const shown = Object.entries(args ?? {}).filter(([, value]) => value !== void 0);
|
|
@@ -39353,7 +40020,7 @@ const SKILL_RESOURCE = {
|
|
|
39353
40020
|
name: "seo-skill",
|
|
39354
40021
|
uri: "skill://seo/SKILL.md",
|
|
39355
40022
|
title: "SEO Skill",
|
|
39356
|
-
description: "How to use these tools:
|
|
40023
|
+
description: "How to use these tools: what each command returns and how to reach it.",
|
|
39357
40024
|
mimeType: "text/markdown"
|
|
39358
40025
|
};
|
|
39359
40026
|
//#endregion
|
|
@@ -39550,7 +40217,11 @@ function nodeCommandDeps(options) {
|
|
|
39550
40217
|
pageMarkdown: async (url) => extractMainMarkdown(await fetchPage(url, { refresh: options.refresh })),
|
|
39551
40218
|
call: options.call,
|
|
39552
40219
|
bing: localBing,
|
|
39553
|
-
bingConnect: LOCAL_BING_CONNECT
|
|
40220
|
+
bingConnect: LOCAL_BING_CONNECT,
|
|
40221
|
+
analytics: {
|
|
40222
|
+
accounts: localAccounts,
|
|
40223
|
+
signIn: LOCAL_SIGN_IN.signIn
|
|
40224
|
+
}
|
|
39554
40225
|
};
|
|
39555
40226
|
}
|
|
39556
40227
|
const LOCAL_BING_CONNECT = "Connect Bing on your seoskill.dev account and run `seo login`, or run `seo providers bing connect` with an API key from Bing Webmaster Tools (Settings, API access).";
|