seo 0.7.1 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/mcp.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
2
- import { i as __toESM, n as load, r as __commonJSMin, t as version } from "./package-BaQwwfSG.js";
2
+ import { i as __toESM, n as load, r as __commonJSMin, t as version } from "./package-BjCto4m4.js";
3
3
  import { createHash, randomUUID, sign } from "node:crypto";
4
4
  import { ZodOptional, z } from "zod";
5
5
  import { arch, homedir, platform, totalmem } from "node:os";
@@ -13036,6 +13036,98 @@ const RULE_DEFINITIONS = [
13036
13036
  howToFix: "Add twitter:card and matching metadata only when you need a specific X preview. Keep Open Graph metadata accurate either way.",
13037
13037
  impactIfIgnored: "X may use fallback metadata or a simple link preview. Organic rankings are not affected.",
13038
13038
  howToVerify: "Re-run the crawl and confirm twitterCard is present if a specific X preview is required."
13039
+ },
13040
+ {
13041
+ id: "ai_search_bot_blocked",
13042
+ title: "robots.txt blocks AI search bots",
13043
+ category: "agents",
13044
+ defaultSeverity: "medium",
13045
+ whyItMatters: "AI search bots fetch pages to answer a person's question and cite the source. A blocked bot cannot read the site, so its answers cannot quote or link to it. Training bots are a separate choice and are listed apart.",
13046
+ howToFix: "Allow the AI search and user bots in robots.txt if the site should appear in AI answers. Block training bots separately if the person wants that.",
13047
+ impactIfIgnored: "ChatGPT, Claude, Perplexity and similar tools can leave the site out of answers and citations.",
13048
+ howToVerify: "Re-run the crawl and confirm the search bots in ai.robotsTxt.botAccess show allowed: true.",
13049
+ review: {
13050
+ question: "Is the site meant to be left out of AI search answers?",
13051
+ changeOnlyIf: "Change robots.txt when the person wants the site to appear in AI answers.",
13052
+ ifNotNeeded: "Record that blocking AI search bots is a deliberate choice for this site."
13053
+ },
13054
+ agentHints: { evidenceFields: ["report.ai.robotsTxt.botAccess"] }
13055
+ },
13056
+ {
13057
+ id: "llms_txt_missing",
13058
+ title: "No llms.txt",
13059
+ category: "agents",
13060
+ defaultSeverity: "low",
13061
+ whyItMatters: "llms.txt gives agents a short summary of the site and links to its main pages, ideally their Markdown copies. Search engines do not use it for ranking.",
13062
+ howToFix: "Publish /llms.txt as plain text with a one-line summary and sections of links to the main pages.",
13063
+ impactIfIgnored: "Agents must find the main pages by crawling, which costs them more requests and tokens.",
13064
+ howToVerify: "Request /llms.txt and confirm a 200 plain text response.",
13065
+ review: {
13066
+ question: "Should agents get a reading list for this site?",
13067
+ changeOnlyIf: "Add it when agents are expected to read or use the site.",
13068
+ ifNotNeeded: "Record that the site has no llms.txt by choice."
13069
+ },
13070
+ agentHints: { evidenceFields: ["report.ai.llmsTxt"] }
13071
+ },
13072
+ {
13073
+ id: "llms_txt_broken_link",
13074
+ title: "llms.txt links to a page that does not load",
13075
+ category: "agents",
13076
+ defaultSeverity: "low",
13077
+ whyItMatters: "An agent that follows a link in llms.txt and gets an error loses its reading list entry.",
13078
+ howToFix: "Update or remove each broken link in llms.txt, or restore the page it points to.",
13079
+ impactIfIgnored: "Agents waste requests and can miss the page.",
13080
+ howToVerify: "Re-run the crawl and confirm ai.llmsTxt.links.broken is empty.",
13081
+ agentHints: { evidenceFields: ["report.ai.llmsTxt.links"] }
13082
+ },
13083
+ {
13084
+ id: "markdown_missing",
13085
+ title: "No Markdown copy of the pages",
13086
+ category: "agents",
13087
+ defaultSeverity: "low",
13088
+ whyItMatters: "A Markdown copy gives agents the page content without layout, scripts or navigation, at a fraction of the tokens. Neither a request with Accept: text/markdown nor a rel=\"alternate\" link returned one. Search engines do not use it for ranking.",
13089
+ howToFix: "Serve a Markdown copy of each page, linked from the page with <link rel=\"alternate\" type=\"text/markdown\">, and return it for Accept: text/markdown with Vary: Accept.",
13090
+ impactIfIgnored: "Agents read the full HTML, which costs more tokens and can include layout text.",
13091
+ howToVerify: "Request a page with Accept: text/markdown and confirm a text/markdown response.",
13092
+ review: {
13093
+ question: "Should agents get a Markdown copy of the pages?",
13094
+ changeOnlyIf: "Add it when agents are expected to read the site.",
13095
+ ifNotNeeded: "Record that the site serves HTML only by choice."
13096
+ },
13097
+ agentHints: { evidenceFields: ["report.ai.markdownNegotiation", "report.ai.markdownCopies"] }
13098
+ },
13099
+ {
13100
+ id: "markdown_alternate_missing",
13101
+ title: "Page has no link to its Markdown copy",
13102
+ category: "agents",
13103
+ defaultSeverity: "low",
13104
+ whyItMatters: "Other pages on the site link to a Markdown copy, but this one does not, so agents cannot find a copy for it.",
13105
+ howToFix: "Add <link rel=\"alternate\" type=\"text/markdown\" href=\"...\"> to the page head, pointing at its Markdown copy.",
13106
+ impactIfIgnored: "Agents read the full HTML for this page.",
13107
+ howToVerify: "Re-run the crawl and confirm the page lists a Markdown alternate.",
13108
+ agentHints: { evidenceFields: ["page.markdownAlternates", "report.ai.markdownCopies"] }
13109
+ },
13110
+ {
13111
+ id: "markdown_alternate_on_private_page",
13112
+ title: "Signed-in page links to a Markdown copy",
13113
+ category: "agents",
13114
+ defaultSeverity: "medium",
13115
+ whyItMatters: "Signed-in pages should have no Markdown copy. This page needs a sign-in, yet it links to one. A copy can show agents content that a signed-out visitor cannot see, or fail and waste their requests.",
13116
+ howToFix: "Remove the rel=\"alternate\" Markdown link, the Link header and any Markdown copy for signed-in routes. Public pages keep theirs, including noindex pages such as privacy and terms.",
13117
+ impactIfIgnored: "Agents can request a copy that fails, or read content that was meant to stay behind a sign-in.",
13118
+ howToVerify: "Re-run the crawl and confirm the page no longer lists a Markdown alternate.",
13119
+ agentHints: { evidenceFields: ["page.markdownAlternates", "report.ai.markdownCopies.broken"] }
13120
+ },
13121
+ {
13122
+ id: "markdown_copy_broken",
13123
+ title: "Markdown copy does not load",
13124
+ category: "agents",
13125
+ defaultSeverity: "medium",
13126
+ whyItMatters: "The page links to a Markdown copy that returns an error or HTML, so an agent that follows the link gets nothing useful. If the page needs a sign-in, remove the link instead.",
13127
+ howToFix: "Make the linked URL return the page as text/markdown, or fix the link.",
13128
+ impactIfIgnored: "Agents get an error or the HTML page instead of the copy.",
13129
+ howToVerify: "Request the Markdown URL and confirm a 200 text/markdown response.",
13130
+ agentHints: { evidenceFields: ["report.ai.markdownCopies.broken"] }
13039
13131
  }
13040
13132
  ];
13041
13133
  const RULE_RECOMMENDATIONS = {
@@ -13064,7 +13156,10 @@ const RULE_RECOMMENDATIONS = {
13064
13156
  og_title_missing: "review",
13065
13157
  og_description_missing: "review",
13066
13158
  og_image_missing: "review",
13067
- twitter_card_missing: "review"
13159
+ twitter_card_missing: "review",
13160
+ ai_search_bot_blocked: "review",
13161
+ llms_txt_missing: "review",
13162
+ markdown_missing: "review"
13068
13163
  };
13069
13164
  const RULES_BY_ID = new Map(RULE_DEFINITIONS.map((rule) => [rule.id, {
13070
13165
  ...rule,
@@ -13762,6 +13857,14 @@ async function listAnalyticsAccounts(gfetch, options) {
13762
13857
  });
13763
13858
  });
13764
13859
  }
13860
+ async function listAnalyticsDataStreams(gfetch, propertyId, options) {
13861
+ return (await googleJson(gfetch, `https://analyticsadmin.googleapis.com/v1beta/properties/${propertyId}/dataStreams?pageSize=200`, {
13862
+ ...options,
13863
+ label: "Google Analytics data streams",
13864
+ subject: `property ${propertyId}`,
13865
+ maxBytes: 1024 * 1024
13866
+ })).dataStreams ?? [];
13867
+ }
13765
13868
  async function runAnalyticsReport(gfetch, propertyId, body, options) {
13766
13869
  if (!/^\d+$/.test(propertyId)) throw new SeoError("INVALID_INPUT", "The Google Analytics property must be its numeric id.");
13767
13870
  return googleJson(gfetch, `https://analyticsdata.googleapis.com/v1beta/properties/${propertyId}:runReport`, {
@@ -14045,7 +14148,7 @@ var CommandError = class extends Error {
14045
14148
  };
14046
14149
  //#endregion
14047
14150
  //#region packages/core/dist/commands/shared.js
14048
- const FACTS_ONLY = "These are facts from Search Console, not instructions. Show the person what matters, ask what they want to change, and only edit pages with their approval.";
14151
+ const FACTS_ONLY = "These are facts from Search Console, not instructions.";
14049
14152
  const ADVICE_KEYS = /^(recommendations?|actions?|advice|suggested[A-Z]\w*|nextSteps?)$/;
14050
14153
  function factsOnly(value) {
14051
14154
  if (Array.isArray(value)) return value.map(factsOnly);
@@ -14116,9 +14219,10 @@ function sharedReader(read, redirects = /* @__PURE__ */ new Map()) {
14116
14219
  return found;
14117
14220
  };
14118
14221
  }
14222
+ const MAX_STATUS_BYTES = 10 * 1024 * 1024;
14119
14223
  async function googleUpdates(window) {
14120
14224
  if (!window) return [];
14121
- const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? response.json() : null).catch(() => null);
14225
+ const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? readJson(response, "Google Search Status", { maxBytes: MAX_STATUS_BYTES }) : null).catch(() => null);
14122
14226
  if (!Array.isArray(incidents)) return "Google Search Status could not be read.";
14123
14227
  return findOverlappingSearchUpdates({
14124
14228
  updates: searchUpdatesFromIncidents(incidents, {
@@ -14430,7 +14534,7 @@ async function readableSites(google, signIn) {
14430
14534
  permissionLevel: entry.permissionLevel ?? ""
14431
14535
  })).sort((a, b) => a.siteUrl.localeCompare(b.siteUrl));
14432
14536
  }
14433
- const hostOf$1 = (value) => {
14537
+ const hostOf$2 = (value) => {
14434
14538
  try {
14435
14539
  return new URL(value.includes("://") ? value : `https://${value.replace(/^sc-domain:/, "")}`).hostname.replace(/^www\./, "");
14436
14540
  } catch {
@@ -14448,14 +14552,14 @@ async function runSites(input) {
14448
14552
  };
14449
14553
  })), input.bing ? listBingSites(input.bing).catch((error) => error instanceof Error ? error : new Error(String(error))) : Promise.resolve(null)]);
14450
14554
  const bingSites = Array.isArray(bing) ? bing.filter((site) => site.verified) : [];
14451
- const bingByHost = new Map(bingSites.map((site) => [hostOf$1(site.url), site]));
14555
+ const bingByHost = new Map(bingSites.map((site) => [hostOf$2(site.url), site]));
14452
14556
  const properties = google.flatMap((account) => account.properties instanceof Error ? [] : account.properties.map((property) => ({
14453
14557
  ...property,
14454
14558
  ...named ? { account: account.email ?? "" } : {},
14455
- ...Array.isArray(bing) ? { bing: bingByHost.get(hostOf$1(property.siteUrl))?.url ?? "" } : {}
14559
+ ...Array.isArray(bing) ? { bing: bingByHost.get(hostOf$2(property.siteUrl))?.url ?? "" } : {}
14456
14560
  })));
14457
- const googleHosts = new Set(properties.map((property) => hostOf$1(property.siteUrl)));
14458
- const bingOnly = bingSites.filter((site) => !googleHosts.has(hostOf$1(site.url)));
14561
+ const googleHosts = new Set(properties.map((property) => hostOf$2(property.siteUrl)));
14562
+ const bingOnly = bingSites.filter((site) => !googleHosts.has(hostOf$2(site.url)));
14459
14563
  const failed = google.filter((account) => account.properties instanceof Error);
14460
14564
  const analytics = google.flatMap((account) => (account.analytics ?? []).flatMap((summary) => summary.propertySummaries.map((property) => ({
14461
14565
  property: property.property.replace("properties/", ""),
@@ -14767,7 +14871,7 @@ async function cloudCredentials(options = {}) {
14767
14871
  return credentials;
14768
14872
  }
14769
14873
  function notSignedIn() {
14770
- return "Not signed in. Run `seo login` to use your seoskill.dev account on this machine (it prints a link to approve), or `seo auth login` to sign in with Google on this machine only.";
14874
+ return "Not signed in. Run `seo login` to use your seoskill.dev account on this machine (it prints a link to approve), or `seo auth login` to sign in with Google on this machine only. `seo crawl --url <url>` works without sign-in.";
14771
14875
  }
14772
14876
  function tokenOf(item, what) {
14773
14877
  if ("error" in item) throw new SeoError("AUTH_EXPIRED", `${what}: ${item.error}`);
@@ -21662,7 +21766,7 @@ function clientRenderedContentEvidence(page) {
21662
21766
  }
21663
21767
  };
21664
21768
  }
21665
- function issue(ruleId, page, detail, evidence) {
21769
+ function issue$1(ruleId, page, detail, evidence) {
21666
21770
  const rule = explainRule(ruleId);
21667
21771
  if (!rule) throw new Error(`Missing rule guidance for ${ruleId}`);
21668
21772
  return {
@@ -21827,7 +21931,7 @@ function auditCrawlPages(pages, opts = {}) {
21827
21931
  }
21828
21932
  for (const page of pages) {
21829
21933
  if (page.status === 0) {
21830
- issues.push(issue("connection_error", page, page.error, {
21934
+ issues.push(issue$1("connection_error", page, page.error, {
21831
21935
  status: page.status,
21832
21936
  error: page.error
21833
21937
  }));
@@ -21835,23 +21939,23 @@ function auditCrawlPages(pages, opts = {}) {
21835
21939
  }
21836
21940
  if (page.accessBlock) continue;
21837
21941
  if (page.status >= 500) {
21838
- issues.push(issue("server_error", page, String(page.status), { status: page.status }));
21839
- if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue("broken_internal_link", page, String(page.status), {
21942
+ issues.push(issue$1("server_error", page, String(page.status), { status: page.status }));
21943
+ if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
21840
21944
  status: page.status,
21841
21945
  internalInlinkCount: page.internalInlinkCount
21842
21946
  }));
21843
21947
  continue;
21844
21948
  }
21845
21949
  if (page.status >= 400) {
21846
- issues.push(issue("client_error", page, String(page.status), { status: page.status }));
21847
- if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0) issues.push(issue("broken_internal_link", page, String(page.status), {
21950
+ issues.push(issue$1("client_error", page, String(page.status), { status: page.status }));
21951
+ if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
21848
21952
  status: page.status,
21849
21953
  internalInlinkCount: page.internalInlinkCount
21850
21954
  }));
21851
21955
  continue;
21852
21956
  }
21853
21957
  if (page.status >= 300) {
21854
- issues.push(issue("redirected_url", page, `Status ${page.status}`, {
21958
+ issues.push(issue$1("redirected_url", page, `Status ${page.status}`, {
21855
21959
  requestedUrl: page.url,
21856
21960
  finalUrl: page.finalUrl,
21857
21961
  status: page.status
@@ -21862,7 +21966,7 @@ function auditCrawlPages(pages, opts = {}) {
21862
21966
  const redirected = isRedirectedPage(page);
21863
21967
  if (redirected) {
21864
21968
  const redirectTarget = page.metaRefresh?.targetUrl ?? page.finalUrl;
21865
- issues.push(issue("redirected_url", page, `Final URL: ${redirectTarget}`, {
21969
+ issues.push(issue$1("redirected_url", page, `Final URL: ${redirectTarget}`, {
21866
21970
  requestedUrl: page.url,
21867
21971
  finalUrl: redirectTarget,
21868
21972
  status: page.status,
@@ -21873,46 +21977,46 @@ function auditCrawlPages(pages, opts = {}) {
21873
21977
  }));
21874
21978
  }
21875
21979
  const redirectChain = page.fetchDiagnostics?.redirectChain ?? [];
21876
- if (redirectChain.length > 1) issues.push(issue("redirect_chain", page, `${redirectChain.length} hops`, {
21980
+ if (redirectChain.length > 1) issues.push(issue$1("redirect_chain", page, `${redirectChain.length} hops`, {
21877
21981
  redirectChain,
21878
21982
  hops: redirectChain.length
21879
21983
  }));
21880
- if ((page.responseTimeMs ?? 0) > SLOW_RESPONSE_MS) issues.push(issue("slow_response", page, `${page.responseTimeMs}ms`, {
21984
+ if ((page.responseTimeMs ?? 0) > SLOW_RESPONSE_MS) issues.push(issue$1("slow_response", page, `${page.responseTimeMs}ms`, {
21881
21985
  responseTimeMs: page.responseTimeMs,
21882
21986
  thresholdMs: SLOW_RESPONSE_MS
21883
21987
  }));
21884
21988
  if (page.auditScope === "status") continue;
21885
21989
  if (redirected) continue;
21886
- if (page.softAuthenticationGate) issues.push(issue("soft_authentication_gate", page, "A password form and login-specific page signals were observed.", page.softAuthenticationGate));
21990
+ if (page.softAuthenticationGate) issues.push(issue$1("soft_authentication_gate", page, "A password form and login-specific page signals were observed.", page.softAuthenticationGate));
21887
21991
  const renderedContent = clientRenderedContentEvidence(page);
21888
- if (renderedContent) issues.push(issue("client_rendered_content", page, "The raw response was an empty or near-empty shell.", renderedContent));
21889
- if (!renderedContent && isAuditableHtmlPage(page) && page.indexable && page.extractionStatus === "complete" && page.wordCount < NEAR_EMPTY_CONTENT_WORDS) issues.push(issue("near_empty_content", page, `${page.wordCount} extracted words`, {
21992
+ if (renderedContent) issues.push(issue$1("client_rendered_content", page, "The raw response was an empty or near-empty shell.", renderedContent));
21993
+ if (!renderedContent && isAuditableHtmlPage(page) && page.indexable && page.extractionStatus === "complete" && page.wordCount < NEAR_EMPTY_CONTENT_WORDS) issues.push(issue$1("near_empty_content", page, `${page.wordCount} extracted words`, {
21890
21994
  wordCount: page.wordCount,
21891
21995
  reviewBelowWords: NEAR_EMPTY_CONTENT_WORDS,
21892
21996
  contentSample: page.contentSample,
21893
21997
  outgoingInternalCount: page.outgoingInternalCount,
21894
21998
  rendering: page.fetchDiagnostics?.rendering
21895
21999
  }));
21896
- if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(issue("large_html", page, `${Math.round((page.sizeBytes ?? 0) / 1024)} KB`, {
22000
+ if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(issue$1("large_html", page, `${Math.round((page.sizeBytes ?? 0) / 1024)} KB`, {
21897
22001
  sizeBytes: page.sizeBytes,
21898
22002
  thresholdBytes: LARGE_HTML_BYTES
21899
22003
  }));
21900
- if (isHtmlPage(page) && !page.compression && (page.sizeBytes ?? 0) > COMPRESSION_MIN_BYTES) issues.push(issue("no_compression", page, void 0, {
22004
+ if (isHtmlPage(page) && !page.compression && (page.sizeBytes ?? 0) > COMPRESSION_MIN_BYTES) issues.push(issue$1("no_compression", page, void 0, {
21901
22005
  sizeBytes: page.sizeBytes,
21902
22006
  thresholdBytes: COMPRESSION_MIN_BYTES,
21903
22007
  compression: page.compression
21904
22008
  }));
21905
- if (page.isHttps === false && !isLoopbackUrl(page.finalUrl || page.url)) issues.push(issue("http_not_secure", page, void 0, { finalUrl: page.finalUrl }));
22009
+ if (page.isHttps === false && !isLoopbackUrl(page.finalUrl || page.url)) issues.push(issue$1("http_not_secure", page, void 0, { finalUrl: page.finalUrl }));
21906
22010
  else if (page.isHttps === true) {
21907
- if ((page.mixedContentCount ?? 0) > 0) issues.push(issue("mixed_content", page, `${page.mixedContentCount} insecure resources`, {
22011
+ if ((page.mixedContentCount ?? 0) > 0) issues.push(issue$1("mixed_content", page, `${page.mixedContentCount} insecure resources`, {
21908
22012
  mixedContentCount: page.mixedContentCount,
21909
22013
  mixedContentSamples: page.mixedContentSamples
21910
22014
  }));
21911
- if (page.hasHsts === false) issues.push(issue("hsts_missing", page, void 0, { header: "strict-transport-security" }));
22015
+ if (page.hasHsts === false) issues.push(issue$1("hsts_missing", page, void 0, { header: "strict-transport-security" }));
21912
22016
  }
21913
22017
  for (const link of page.externalLinkChecks ?? []) {
21914
22018
  if (link.state !== "confirmed-broken") continue;
21915
- issues.push(issue("broken_external_link", page, link.url, {
22019
+ issues.push(issue$1("broken_external_link", page, link.url, {
21916
22020
  url: link.url,
21917
22021
  status: link.status,
21918
22022
  error: link.error,
@@ -21920,64 +22024,64 @@ function auditCrawlPages(pages, opts = {}) {
21920
22024
  attempts: link.attempts
21921
22025
  }));
21922
22026
  }
21923
- if (page.indexable && (page.internalInlinkCount ?? 0) === 0 && !sameUrl(page.url, opts.startUrl)) issues.push(issue("orphan_page", page, "No internal inlinks", { internalInlinkCount: page.internalInlinkCount ?? 0 }));
21924
- if ((page.crawlDepth ?? 0) > DEEP_PAGE_DEPTH) issues.push(issue("deep_page", page, `Depth ${page.crawlDepth}`, {
22027
+ if (page.indexable && (page.internalInlinkCount ?? 0) === 0 && !sameUrl(page.url, opts.startUrl)) issues.push(issue$1("orphan_page", page, "No internal inlinks", { internalInlinkCount: page.internalInlinkCount ?? 0 }));
22028
+ if ((page.crawlDepth ?? 0) > DEEP_PAGE_DEPTH) issues.push(issue$1("deep_page", page, `Depth ${page.crawlDepth}`, {
21925
22029
  crawlDepth: page.crawlDepth,
21926
22030
  threshold: DEEP_PAGE_DEPTH
21927
22031
  }));
21928
- if (page.indexable && isValuablePage(page) && (page.internalInlinkCount ?? 0) <= WEAK_VALUABLE_INLINKS) issues.push(issue("weak_internal_links_to_valuable_page", page, `${page.internalInlinkCount ?? 0} internal inlinks`, {
22032
+ if (page.indexable && isValuablePage(page) && (page.internalInlinkCount ?? 0) <= WEAK_VALUABLE_INLINKS) issues.push(issue$1("weak_internal_links_to_valuable_page", page, `${page.internalInlinkCount ?? 0} internal inlinks`, {
21929
22033
  internalInlinkCount: page.internalInlinkCount ?? 0,
21930
22034
  searchMetrics: page.searchMetrics,
21931
22035
  analytics: page.analytics
21932
22036
  }));
21933
- if (page.robotsTxt?.allowed === false) issues.push(issue("robots_blocked", page, page.robotsTxt.matchedLine, { robotsTxt: page.robotsTxt }));
21934
- if (hasXRobotsDirective(page.xRobotsTag, "noindex")) issues.push(issue("x_robots_noindex", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
22037
+ if (page.robotsTxt?.allowed === false) issues.push(issue$1("robots_blocked", page, page.robotsTxt.matchedLine, { robotsTxt: page.robotsTxt }));
22038
+ if (hasXRobotsDirective(page.xRobotsTag, "noindex")) issues.push(issue$1("x_robots_noindex", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
21935
22039
  const xRobotsNofollow = hasXRobotsDirective(page.xRobotsTag, "nofollow");
21936
- if (xRobotsNofollow) issues.push(issue("nofollow", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
22040
+ if (xRobotsNofollow) issues.push(issue$1("nofollow", page, page.xRobotsTag, { xRobotsTag: page.xRobotsTag }));
21937
22041
  if (!isAuditableHtmlPage(page)) continue;
21938
22042
  const title = page.title?.trim();
21939
22043
  const titleWidth = title ? estimateSerpTitleWidth(title) : void 0;
21940
22044
  const duplicateTitle = duplicateTitles.get(normalizedMetadata(title) ?? "");
21941
- if (!title) issues.push(issue("missing_title", page));
21942
- else if (titleWidth?.status === "over-reference") issues.push(issue("title_too_wide", page, `${titleWidth.estimatedPixels}px`, {
22045
+ if (!title) issues.push(issue$1("missing_title", page));
22046
+ else if (titleWidth?.status === "over-reference") issues.push(issue$1("title_too_wide", page, `${titleWidth.estimatedPixels}px`, {
21943
22047
  title,
21944
22048
  ...titleWidth
21945
22049
  }));
21946
- if (title && duplicateTitle) issues.push(issue("title_duplicate", page, duplicateTitle.value, {
22050
+ if (title && duplicateTitle) issues.push(issue$1("title_duplicate", page, duplicateTitle.value, {
21947
22051
  title,
21948
22052
  duplicateCount: duplicateTitle.count,
21949
22053
  sampleUrls: duplicateTitle.sampleUrls
21950
22054
  }));
21951
22055
  const metaDescription = page.metaDescription?.trim();
21952
22056
  const duplicateDescription = duplicateDescriptions.get(normalizedMetadata(metaDescription) ?? "");
21953
- if (!metaDescription) issues.push(issue("missing_meta_description", page));
21954
- if (metaDescription && duplicateDescription) issues.push(issue("meta_description_duplicate", page, duplicateDescription.value, {
22057
+ if (!metaDescription) issues.push(issue$1("missing_meta_description", page));
22058
+ if (metaDescription && duplicateDescription) issues.push(issue$1("meta_description_duplicate", page, duplicateDescription.value, {
21955
22059
  metaDescription,
21956
22060
  duplicateCount: duplicateDescription.count,
21957
22061
  sampleUrls: duplicateDescription.sampleUrls
21958
22062
  }));
21959
22063
  const h1Count = page.h1Count ?? (page.h1 ? 1 : 0);
21960
- if (h1Count === 0) issues.push(issue("h1_missing", page, "No H1 found", { h1Count }));
21961
- else if (h1Count > 1) issues.push(issue("h1_multiple", page, `${h1Count} H1 headings`, { h1Count }));
21962
- if (page.canonicalStatus === "conflicting") issues.push(issue("canonical_conflict", page, void 0, { candidates: page.canonicalCandidates }));
21963
- else if (page.canonicalStatus === "duplicate") issues.push(issue("canonical_multiple", page, page.canonical, {
22064
+ if (h1Count === 0) issues.push(issue$1("h1_missing", page, "No H1 found", { h1Count }));
22065
+ else if (h1Count > 1) issues.push(issue$1("h1_multiple", page, `${h1Count} H1 headings`, { h1Count }));
22066
+ if (page.canonicalStatus === "conflicting") issues.push(issue$1("canonical_conflict", page, void 0, { candidates: page.canonicalCandidates }));
22067
+ else if (page.canonicalStatus === "duplicate") issues.push(issue$1("canonical_multiple", page, page.canonical, {
21964
22068
  canonical: page.canonical,
21965
22069
  candidates: page.canonicalCandidates
21966
22070
  }));
21967
- else if (page.canonicalStatus === "outside-head-only") issues.push(issue("canonical_outside_head", page, page.canonicalRaw, { candidates: page.canonicalCandidates }));
21968
- else if (page.canonicalStatus === "streamed-outside-head") issues.push(issue("canonical_streamed_in_body", page, page.canonical, {
22071
+ else if (page.canonicalStatus === "outside-head-only") issues.push(issue$1("canonical_outside_head", page, page.canonicalRaw, { candidates: page.canonicalCandidates }));
22072
+ else if (page.canonicalStatus === "streamed-outside-head") issues.push(issue$1("canonical_streamed_in_body", page, page.canonical, {
21969
22073
  canonical: page.canonical,
21970
22074
  candidates: page.canonicalCandidates
21971
22075
  }));
21972
22076
  if (!page.canonical) {
21973
- if (!["conflicting", "outside-head-only"].includes(page.canonicalStatus ?? "")) issues.push(page.canonicalRaw ? issue("canonical_invalid", page, page.canonicalRaw, { canonicalRaw: page.canonicalRaw }) : issue("canonical_missing", page));
22077
+ if (!["conflicting", "outside-head-only"].includes(page.canonicalStatus ?? "")) issues.push(page.canonicalRaw ? issue$1("canonical_invalid", page, page.canonicalRaw, { canonicalRaw: page.canonicalRaw }) : issue$1("canonical_missing", page));
21974
22078
  } else {
21975
- if (page.canonicalRaw && !absoluteHttpUrl(page.canonicalRaw)) issues.push(issue("canonical_non_absolute", page, page.canonicalRaw, {
22079
+ if (page.canonicalRaw && !absoluteHttpUrl(page.canonicalRaw)) issues.push(issue$1("canonical_non_absolute", page, page.canonicalRaw, {
21976
22080
  canonicalRaw: page.canonicalRaw,
21977
22081
  canonical: page.canonical
21978
22082
  }));
21979
22083
  const canonicalTarget = pageByUrl.get(urlKey(page.canonical) ?? "");
21980
- if (canonicalTarget?.canonical && !sameUrl(canonicalTarget.canonical, page.canonical)) issues.push(issue("canonical_chain", page, page.canonical, {
22084
+ if (canonicalTarget?.canonical && !sameUrl(canonicalTarget.canonical, page.canonical)) issues.push(issue$1("canonical_chain", page, page.canonical, {
21981
22085
  canonical: page.canonical,
21982
22086
  nextCanonical: canonicalTarget.canonical,
21983
22087
  chain: [
@@ -21987,25 +22091,25 @@ function auditCrawlPages(pages, opts = {}) {
21987
22091
  ]
21988
22092
  }));
21989
22093
  }
21990
- if (hasMetaRobotsDirective(page.metaRobots, "noindex")) issues.push(issue("noindex", page, page.indexability, { metaRobots: page.metaRobots }));
21991
- if (hasMetaRobotsDirective(page.metaRobots, "nofollow") && !xRobotsNofollow) issues.push(issue("nofollow", page, page.metaRobots, { metaRobots: page.metaRobots }));
21992
- if (page.canonical && !sameUrl(page.canonical, page.finalUrl) && (page.declaredIndexability === "canonical-hint-other" || page.declaredIndexability === void 0 && /canonical/i.test(page.indexability ?? ""))) issues.push(issue("canonicalized_page", page, page.canonical, {
22094
+ if (hasMetaRobotsDirective(page.metaRobots, "noindex")) issues.push(issue$1("noindex", page, page.indexability, { metaRobots: page.metaRobots }));
22095
+ if (hasMetaRobotsDirective(page.metaRobots, "nofollow") && !xRobotsNofollow) issues.push(issue$1("nofollow", page, page.metaRobots, { metaRobots: page.metaRobots }));
22096
+ if (page.canonical && !sameUrl(page.canonical, page.finalUrl) && (page.declaredIndexability === "canonical-hint-other" || page.declaredIndexability === void 0 && /canonical/i.test(page.indexability ?? ""))) issues.push(issue$1("canonicalized_page", page, page.canonical, {
21993
22097
  canonical: page.canonical,
21994
22098
  finalUrl: page.finalUrl,
21995
22099
  indexability: page.indexability
21996
22100
  }));
21997
- else if (page.canonical && !sameUrl(page.canonical, page.finalUrl)) issues.push(issue("canonical_mismatch", page, page.canonical, {
22101
+ else if (page.canonical && !sameUrl(page.canonical, page.finalUrl)) issues.push(issue$1("canonical_mismatch", page, page.canonical, {
21998
22102
  canonical: page.canonical,
21999
22103
  finalUrl: page.finalUrl
22000
22104
  }));
22001
22105
  const contentDuplicate = page.mainContentHash ? duplicateContent.get(page.mainContentHash) : void 0;
22002
- if (contentDuplicate) issues.push(issue("duplicate_content", page, "Duplicate main content", {
22106
+ if (contentDuplicate) issues.push(issue$1("duplicate_content", page, "Duplicate main content", {
22003
22107
  mainContentHash: page.mainContentHash,
22004
22108
  duplicateCount: contentDuplicate.count,
22005
22109
  sampleUrls: contentDuplicate.sampleUrls
22006
22110
  }));
22007
22111
  const coverage = queryCoverage(page);
22008
- if (coverage && coverage.coverage < QUERY_COVERAGE_MIN) issues.push(issue("query_coverage_missing", page, coverage.query, {
22112
+ if (coverage && coverage.coverage < QUERY_COVERAGE_MIN) issues.push(issue$1("query_coverage_missing", page, coverage.query, {
22009
22113
  query: coverage.query,
22010
22114
  impressions: page.topQuery?.impressions,
22011
22115
  matchedTerms: coverage.matchedTerms,
@@ -22013,36 +22117,36 @@ function auditCrawlPages(pages, opts = {}) {
22013
22117
  coverage: coverage.coverage,
22014
22118
  threshold: QUERY_COVERAGE_MIN
22015
22119
  }));
22016
- if ((page.imagesMissingAlt ?? 0) > 0) issues.push(issue("image_missing_alt", page, `${page.imagesMissingAlt} of ${page.imagesTotal ?? 0} images`, {
22120
+ if ((page.imagesMissingAlt ?? 0) > 0) issues.push(issue$1("image_missing_alt", page, `${page.imagesMissingAlt} of ${page.imagesTotal ?? 0} images`, {
22017
22121
  imagesTotal: page.imagesTotal,
22018
22122
  imagesMissingAlt: page.imagesMissingAlt
22019
22123
  }));
22020
- if (page.oversizedImageCandidates?.length) issues.push(issue("image_oversized_candidate", page, `${page.oversizedImageCandidates.length} image candidate${page.oversizedImageCandidates.length === 1 ? "" : "s"}`, {
22124
+ if (page.oversizedImageCandidates?.length) issues.push(issue$1("image_oversized_candidate", page, `${page.oversizedImageCandidates.length} image candidate${page.oversizedImageCandidates.length === 1 ? "" : "s"}`, {
22021
22125
  thresholdPx: OVERSIZED_IMAGE_CANDIDATE_LIMIT,
22022
22126
  candidates: page.oversizedImageCandidates.slice(0, 10)
22023
22127
  }));
22024
- if (!page.hasViewport) issues.push(issue("viewport_missing", page));
22025
- if (!page.lang) issues.push(issue("lang_missing", page));
22128
+ if (!page.hasViewport) issues.push(issue$1("viewport_missing", page));
22129
+ if (!page.lang) issues.push(issue$1("lang_missing", page));
22026
22130
  const hreflang = page.hreflang ?? [];
22027
22131
  if (hreflang.length) {
22028
22132
  const malformed = hreflang.filter((item) => !hasRecognizableHreflangFormat(item.hreflang));
22029
- if (malformed.length) issues.push(issue("hreflang_invalid", page, `${malformed.length} malformed`, { malformed }));
22133
+ if (malformed.length) issues.push(issue$1("hreflang_invalid", page, `${malformed.length} malformed`, { malformed }));
22030
22134
  const duplicateCodes = [...new Set(hreflang.map((item) => item.hreflang.trim().toLowerCase()).filter((code, index, values) => values.indexOf(code) !== index))];
22031
- if (duplicateCodes.length) issues.push(issue("hreflang_duplicate", page, duplicateCodes.join(", "), { duplicateCodes }));
22032
- if (!hreflang.some((item) => sameUrl(item.href, page.finalUrl))) issues.push(issue("hreflang_incomplete", page, page.lang, {
22135
+ if (duplicateCodes.length) issues.push(issue$1("hreflang_duplicate", page, duplicateCodes.join(", "), { duplicateCodes }));
22136
+ if (!hreflang.some((item) => sameUrl(item.href, page.finalUrl))) issues.push(issue$1("hreflang_incomplete", page, page.lang, {
22033
22137
  lang: page.lang,
22034
22138
  finalUrl: page.finalUrl,
22035
22139
  hreflang
22036
22140
  }));
22037
22141
  }
22038
22142
  const structuredDataFormats = page.structuredDataFormats;
22039
- if (isAuditableHtmlPage(page) && page.indexable && structuredDataFormats && structuredDataFormats.length === 0 && (page.schemaTypes?.length ?? 0) === 0 && (page.invalidJsonLdCount ?? 0) === 0 && (page.unrecognizedJsonLdTypes?.length ?? 0) === 0) issues.push(issue("structured_data_missing", page, "No structured data detected", {
22143
+ if (isAuditableHtmlPage(page) && page.indexable && structuredDataFormats && structuredDataFormats.length === 0 && (page.schemaTypes?.length ?? 0) === 0 && (page.invalidJsonLdCount ?? 0) === 0 && (page.unrecognizedJsonLdTypes?.length ?? 0) === 0) issues.push(issue$1("structured_data_missing", page, "No structured data detected", {
22040
22144
  structuredDataFormats,
22041
22145
  schemaTypes: page.schemaTypes ?? [],
22042
22146
  invalidJsonLdCount: page.invalidJsonLdCount ?? 0,
22043
22147
  unrecognizedJsonLdTypeCount: page.unrecognizedJsonLdTypes?.length ?? 0
22044
22148
  }));
22045
- if ((page.invalidJsonLdCount ?? 0) > 0) issues.push(issue("jsonld_invalid", page, `${page.invalidJsonLdCount} invalid`, {
22149
+ if ((page.invalidJsonLdCount ?? 0) > 0) issues.push(issue$1("jsonld_invalid", page, `${page.invalidJsonLdCount} invalid`, {
22046
22150
  invalidJsonLdCount: page.invalidJsonLdCount,
22047
22151
  invalidJsonLdSamples: page.invalidJsonLdSamples
22048
22152
  }));
@@ -22052,15 +22156,15 @@ function auditCrawlPages(pages, opts = {}) {
22052
22156
  if (incompleteRichResultCount) {
22053
22157
  const details = incompleteRichResults.map((assessment) => `${assessment.schemaType}: ${assessment.missingRequiredProperties.join(", ")}`);
22054
22158
  const omittedDetails = incompleteRichResultCount - details.length;
22055
- issues.push(issue("rich_result_required_fields_missing", page, [...details, ...omittedDetails > 0 ? [`${omittedDetails} more incomplete assessment${omittedDetails === 1 ? "" : "s"} omitted from bounded detail`] : []].join("; "), {
22159
+ issues.push(issue$1("rich_result_required_fields_missing", page, [...details, ...omittedDetails > 0 ? [`${omittedDetails} more incomplete assessment${omittedDetails === 1 ? "" : "s"} omitted from bounded detail`] : []].join("; "), {
22056
22160
  assessments: incompleteRichResults,
22057
22161
  ...page.googleRichResultsSelection ? { selection: page.googleRichResultsSelection } : {}
22058
22162
  }));
22059
22163
  }
22060
- if (!page.openGraphTitle) issues.push(issue("og_title_missing", page));
22061
- if (!page.openGraphDescription) issues.push(issue("og_description_missing", page));
22062
- if (!page.openGraphImage) issues.push(issue("og_image_missing", page));
22063
- if (!page.twitterCard) issues.push(issue("twitter_card_missing", page));
22164
+ if (!page.openGraphTitle) issues.push(issue$1("og_title_missing", page));
22165
+ if (!page.openGraphDescription) issues.push(issue$1("og_description_missing", page));
22166
+ if (!page.openGraphImage) issues.push(issue$1("og_image_missing", page));
22167
+ if (!page.twitterCard) issues.push(issue$1("twitter_card_missing", page));
22064
22168
  }
22065
22169
  return issues;
22066
22170
  }
@@ -22150,7 +22254,7 @@ const SEVERITY_SCORE = {
22150
22254
  const EFFORT_BY_CATEGORY = {
22151
22255
  canonical: "medium",
22152
22256
  content: "high",
22153
- geo: "medium",
22257
+ agents: "low",
22154
22258
  headings: "low",
22155
22259
  images: "medium",
22156
22260
  indexability: "medium",
@@ -22717,6 +22821,92 @@ const crawlSiteChecksSchema = z.object({ soft404: z.object({
22717
22821
  complete: z.boolean()
22718
22822
  }) });
22719
22823
  //#endregion
22824
+ //#region packages/core/dist/analyze/crawler/site-crawl-ai-schema.js
22825
+ const crawlAiAvailabilitySchema = z.enum([
22826
+ "available",
22827
+ "absent",
22828
+ "access-blocked",
22829
+ "rate-limited",
22830
+ "unreachable"
22831
+ ]);
22832
+ const crawlAiLinkCheckSchema = z.object({
22833
+ url: z.string().url(),
22834
+ status: z.number().int().optional(),
22835
+ contentType: z.string().optional(),
22836
+ error: z.string().optional()
22837
+ });
22838
+ const crawlAiSignalsSchema = z.object({
22839
+ robotsTxt: z.object({
22840
+ url: z.string().url(),
22841
+ exists: z.boolean(),
22842
+ availability: crawlAiAvailabilitySchema,
22843
+ status: z.number().int().optional(),
22844
+ error: z.string().optional(),
22845
+ sitemapUrls: z.array(z.string().url()),
22846
+ contentSignals: z.array(z.string()).optional(),
22847
+ botAccess: z.array(z.object({
22848
+ userAgent: z.string(),
22849
+ purpose: z.enum([
22850
+ "search",
22851
+ "training",
22852
+ "other"
22853
+ ]),
22854
+ allowed: z.boolean().nullable(),
22855
+ declared: z.boolean(),
22856
+ coveredByWildcard: z.boolean()
22857
+ }))
22858
+ }).optional(),
22859
+ llmsTxt: z.object({
22860
+ url: z.string().url(),
22861
+ exists: z.boolean(),
22862
+ availability: crawlAiAvailabilitySchema.optional(),
22863
+ status: z.number().int().optional(),
22864
+ error: z.string().optional(),
22865
+ links: z.object({
22866
+ found: z.number().int().nonnegative(),
22867
+ checked: z.number().int().nonnegative(),
22868
+ limit: z.number().int().positive(),
22869
+ broken: z.array(crawlAiLinkCheckSchema)
22870
+ }).optional()
22871
+ }).optional(),
22872
+ agentResources: z.array(z.object({
22873
+ url: z.string().url(),
22874
+ exists: z.boolean(),
22875
+ status: z.number().int().optional(),
22876
+ contentType: z.string().optional(),
22877
+ validJson: z.boolean().optional()
22878
+ })).optional(),
22879
+ markdownNegotiation: z.object({
22880
+ url: z.string().url(),
22881
+ availability: crawlAiAvailabilitySchema,
22882
+ status: z.number().int().optional(),
22883
+ contentType: z.string().optional(),
22884
+ markdown: z.boolean(),
22885
+ varyAccept: z.boolean(),
22886
+ error: z.string().optional()
22887
+ }).optional(),
22888
+ markdownCopies: z.object({
22889
+ publicPages: z.number().int().nonnegative(),
22890
+ publicPagesWithLink: z.number().int().nonnegative(),
22891
+ privatePagesWithLink: z.number().int().nonnegative(),
22892
+ checked: z.number().int().nonnegative(),
22893
+ limit: z.number().int().positive(),
22894
+ broken: z.array(crawlAiLinkCheckSchema.extend({
22895
+ page: z.string().url(),
22896
+ public: z.boolean()
22897
+ })),
22898
+ largest: z.object({
22899
+ url: z.string().url(),
22900
+ estimatedTokens: z.number().int().nonnegative()
22901
+ }).optional()
22902
+ }).optional(),
22903
+ startPageHeaders: z.object({
22904
+ url: z.string().url(),
22905
+ linkRelations: z.array(z.string()),
22906
+ contentSignal: z.string().optional()
22907
+ }).optional()
22908
+ });
22909
+ //#endregion
22720
22910
  //#region packages/core/dist/analyze/crawler/schemas.js
22721
22911
  const crawlerRuleSeveritySchema = z.enum([
22722
22912
  "low",
@@ -22739,7 +22929,7 @@ const crawlerRuleCategorySchema = z.enum([
22739
22929
  "security",
22740
22930
  "social",
22741
22931
  "structured-data",
22742
- "geo"
22932
+ "agents"
22743
22933
  ]);
22744
22934
  const crawlerRuleIdSchema = z.enum(listRules().map((rule) => rule.id));
22745
22935
  const crawlerRuleInfoSchema = z.object({
@@ -23365,41 +23555,6 @@ const crawlIssueGroupSchema = z.object({
23365
23555
  count: z.number().int(),
23366
23556
  sampleUrls: z.array(z.string().url())
23367
23557
  });
23368
- const crawlAiSignalsSchema = z.object({
23369
- robotsTxt: z.object({
23370
- url: z.string().url(),
23371
- exists: z.boolean(),
23372
- availability: z.enum([
23373
- "available",
23374
- "absent",
23375
- "access-blocked",
23376
- "rate-limited",
23377
- "unreachable"
23378
- ]),
23379
- status: z.number().int().optional(),
23380
- error: z.string().optional(),
23381
- sitemapUrls: z.array(z.string().url()),
23382
- contentSignals: z.array(z.string()).optional(),
23383
- botAccess: z.array(z.object({
23384
- userAgent: z.string(),
23385
- allowed: z.boolean().nullable(),
23386
- declared: z.boolean(),
23387
- coveredByWildcard: z.boolean()
23388
- }))
23389
- }).optional(),
23390
- llmsTxt: z.object({
23391
- url: z.string().url(),
23392
- exists: z.boolean(),
23393
- status: z.number().int().optional()
23394
- }).optional(),
23395
- agentResources: z.array(z.object({
23396
- url: z.string().url(),
23397
- exists: z.boolean(),
23398
- status: z.number().int().optional(),
23399
- contentType: z.string().optional(),
23400
- validJson: z.boolean().optional()
23401
- })).optional()
23402
- });
23403
23558
  const crawlTopFixSchema = crawlIssueGroupSchema.extend({
23404
23559
  recommendation: crawlerRuleRecommendationSchema,
23405
23560
  score: z.number(),
@@ -23696,6 +23851,58 @@ const crawlReportSchema = z.discriminatedUnion("requestEvidenceStatus", [
23696
23851
  ]);
23697
23852
  z.toJSONSchema(crawlReportSchema), z.toJSONSchema(crawlIssueGroupSchema), z.toJSONSchema(crawlTopFixSchema), z.toJSONSchema(crawlerRuleInfoSchema), z.toJSONSchema(crawlRequestObservationSchema), z.toJSONSchema(crawlPageSnapshotSchema);
23698
23853
  //#endregion
23854
+ //#region packages/core/dist/analyze/crawler/agent-ready.js
23855
+ function needsSignIn(page) {
23856
+ return page.status === 401 || page.status === 403 || Boolean(page.softAuthenticationGate);
23857
+ }
23858
+ function isPublicPage(page) {
23859
+ return page.status >= 200 && page.status < 300 && !needsSignIn(page) && /\btext\/html\b/i.test(page.contentType ?? "");
23860
+ }
23861
+ function expectsMarkdown(page) {
23862
+ return isPublicPage(page) && page.indexable;
23863
+ }
23864
+ function issue(ruleId, url, detail, evidence) {
23865
+ const rule = explainRule(ruleId);
23866
+ if (!rule) throw new Error(`Missing rule guidance for ${ruleId}`);
23867
+ return {
23868
+ ruleId,
23869
+ title: rule.title,
23870
+ category: rule.category,
23871
+ severity: rule.defaultSeverity,
23872
+ url,
23873
+ detail,
23874
+ ...evidence ? { evidence } : {}
23875
+ };
23876
+ }
23877
+ function agentReadyIssues(input) {
23878
+ const { ai, startUrl } = input;
23879
+ const issues = [];
23880
+ const blocked = (ai.robotsTxt?.botAccess ?? []).filter((bot) => bot.purpose === "search" && bot.allowed === false);
23881
+ if (blocked.length && ai.robotsTxt) issues.push(issue("ai_search_bot_blocked", ai.robotsTxt.url, `robots.txt blocks ${blocked.map((bot) => bot.userAgent).join(", ")} from the start URL.`, { blocked: blocked.map((bot) => bot.userAgent) }));
23882
+ if (ai.llmsTxt?.availability === "absent") issues.push(issue("llms_txt_missing", ai.llmsTxt.url, `/llms.txt returned ${ai.llmsTxt.status === void 0 ? "no file" : `HTTP ${ai.llmsTxt.status}`}.`));
23883
+ for (const link of ai.llmsTxt?.links?.broken ?? []) issues.push(issue("llms_txt_broken_link", link.url, link.status === void 0 ? `The link in llms.txt could not be fetched: ${link.error ?? "no response"}.` : `The link in llms.txt returned HTTP ${link.status}.`, {
23884
+ llmsTxt: ai.llmsTxt?.url,
23885
+ ...link
23886
+ }));
23887
+ const expected = input.pages.filter(expectsMarkdown);
23888
+ const linked = expected.filter((page) => page.markdownAlternates?.length);
23889
+ const negotiation = ai.markdownNegotiation;
23890
+ if (expected.length && !linked.length && negotiation?.availability === "available" && !negotiation.markdown) issues.push(issue("markdown_missing", startUrl, `Accept: text/markdown returned ${negotiation.contentType || "no content type"}, and none of ${expected.length} public pages links to a Markdown copy.`, { negotiation }));
23891
+ if (linked.length) for (const page of expected) {
23892
+ if (page.markdownAlternates?.length) continue;
23893
+ issues.push(issue("markdown_alternate_missing", page.url, `${linked.length} of ${expected.length} public pages link to a Markdown copy; this one does not.`));
23894
+ }
23895
+ for (const page of input.pages) {
23896
+ if (!needsSignIn(page) || !page.markdownAlternates?.length) continue;
23897
+ issues.push(issue("markdown_alternate_on_private_page", page.url, `The page needs a sign-in but links to a Markdown copy at ${page.markdownAlternates[0]}.`, { markdownAlternates: page.markdownAlternates }));
23898
+ }
23899
+ for (const copy of ai.markdownCopies?.broken ?? []) {
23900
+ if (!copy.public) continue;
23901
+ issues.push(issue("markdown_copy_broken", copy.page, copy.status === void 0 ? `The Markdown copy at ${copy.url} could not be fetched: ${copy.error ?? "no response"}.` : `The Markdown copy at ${copy.url} returned HTTP ${copy.status}${copy.contentType ? ` as ${copy.contentType}` : ""}.`, { ...copy }));
23902
+ }
23903
+ return issues;
23904
+ }
23905
+ //#endregion
23699
23906
  //#region packages/core/dist/analyze/crawler/crawl-caveats.js
23700
23907
  function crawlCaveats(input) {
23701
23908
  const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
@@ -24133,66 +24340,137 @@ async function collectCrawlSiteChecks(input) {
24133
24340
  //#endregion
24134
24341
  //#region packages/core/dist/analyze/crawler/site-crawl-ai-signals.js
24135
24342
  const MAX_AUXILIARY_RESPONSE_BYTES = 2 * 1024 * 1024;
24136
- const AI_BOT_USER_AGENTS = [
24137
- "GPTBot",
24138
- "OAI-SearchBot",
24139
- "ChatGPT-User",
24140
- "ClaudeBot",
24141
- "Claude-User",
24142
- "Claude-SearchBot",
24143
- "PerplexityBot",
24144
- "Perplexity-User",
24145
- "Google-Extended",
24146
- "CCBot",
24147
- "Applebot",
24148
- "Applebot-Extended",
24149
- "Bingbot",
24150
- "Amazonbot",
24151
- "Bytespider",
24152
- "Meta-ExternalAgent",
24153
- "Meta-ExternalFetcher",
24154
- "DuckAssistBot",
24155
- "MistralAI-User",
24156
- "Diffbot",
24157
- "cohere-ai"
24343
+ const LLMS_LINK_LIMIT = 20;
24344
+ const AI_BOTS = [
24345
+ ["OAI-SearchBot", "search"],
24346
+ ["ChatGPT-User", "search"],
24347
+ ["Claude-SearchBot", "search"],
24348
+ ["Claude-User", "search"],
24349
+ ["PerplexityBot", "search"],
24350
+ ["Perplexity-User", "search"],
24351
+ ["DuckAssistBot", "search"],
24352
+ ["MistralAI-User", "search"],
24353
+ ["Meta-ExternalFetcher", "search"],
24354
+ ["Applebot", "search"],
24355
+ ["Bingbot", "search"],
24356
+ ["GPTBot", "training"],
24357
+ ["ClaudeBot", "training"],
24358
+ ["Google-Extended", "training"],
24359
+ ["Applebot-Extended", "training"],
24360
+ ["Meta-ExternalAgent", "training"],
24361
+ ["CCBot", "training"],
24362
+ ["Bytespider", "training"],
24363
+ ["cohere-ai", "training"],
24364
+ ["Amazonbot", "other"],
24365
+ ["Diffbot", "other"]
24158
24366
  ];
24159
24367
  const AGENT_RESOURCE_PATHS = [
24160
- "/.well-known/agent.json",
24161
- "/agent.json",
24368
+ "/.well-known/agent-skills/index.json",
24369
+ "/.well-known/mcp/server-card.json",
24162
24370
  "/.well-known/mcp.json",
24163
- "/.well-known/ai-plugin.json",
24164
- "/.well-known/openapi.json",
24165
- "/openapi.json"
24371
+ "/.well-known/api-catalog",
24372
+ "/openapi.json",
24373
+ "/.well-known/oauth-protected-resource",
24374
+ "/.well-known/oauth-authorization-server",
24375
+ "/.well-known/http-message-signatures-directory"
24166
24376
  ];
24167
- async function checkLlmsTxt(input) {
24168
- const llmsUrl = new URL("/llms.txt", input.url).toString();
24377
+ function availabilityOf(status) {
24378
+ if (status >= 200 && status < 300) return "available";
24379
+ if (status === 401 || status === 403) return "access-blocked";
24380
+ if (status === 429) return "rate-limited";
24381
+ if (status >= 400 && status < 500) return "absent";
24382
+ return "unreachable";
24383
+ }
24384
+ function errorText(error) {
24385
+ return error instanceof Error ? error.message : String(error);
24386
+ }
24387
+ async function get(input, url, headers, readBody = true) {
24169
24388
  const controller = abortController({
24170
24389
  timeoutMs: input.timeoutMs,
24171
24390
  signal: input.signal
24172
24391
  });
24173
24392
  try {
24174
- const response = await input.fetch(llmsUrl, {
24393
+ const response = await input.fetch(url, {
24175
24394
  profile: "bot",
24176
24395
  redirect: "follow",
24177
- signal: controller.signal
24396
+ signal: controller.signal,
24397
+ ...headers ? { headers } : {}
24178
24398
  });
24179
- const contentType = response.headers.get("content-type") ?? "";
24180
- const exists = response.status >= 200 && response.status < 300 && !/\btext\/html\b/i.test(contentType);
24181
- await response.body?.cancel().catch(() => void 0);
24399
+ let text = "";
24400
+ if (readBody) text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, `${new URL(url).pathname} response`);
24401
+ else await response.body?.cancel().catch(() => void 0);
24182
24402
  return {
24183
- url: response.url || llmsUrl,
24184
- exists,
24185
- status: response.status
24403
+ url: response.url || url,
24404
+ availability: availabilityOf(response.status),
24405
+ status: response.status,
24406
+ contentType: response.headers.get("content-type") ?? "",
24407
+ headers: response.headers,
24408
+ text
24186
24409
  };
24187
- } catch {
24410
+ } catch (error) {
24188
24411
  return {
24189
- url: llmsUrl,
24190
- exists: false
24412
+ url,
24413
+ availability: "unreachable",
24414
+ contentType: "",
24415
+ text: "",
24416
+ error: errorText(error)
24191
24417
  };
24192
24418
  } finally {
24193
24419
  controller.cleanup();
24194
24420
  }
24195
24421
  }
24422
+ const isHtml = (contentType) => /\btext\/html\b/i.test(contentType);
24423
+ const isMarkdown = (contentType) => /\btext\/(?:x-)?markdown\b/i.test(contentType);
24424
+ function brokenLink(result, expectText) {
24425
+ if (result.availability === "available" && !(expectText && isHtml(result.contentType))) return void 0;
24426
+ return {
24427
+ url: result.url,
24428
+ ...result.status === void 0 ? {} : { status: result.status },
24429
+ ...result.contentType ? { contentType: result.contentType } : {},
24430
+ ...result.error ? { error: result.error } : {}
24431
+ };
24432
+ }
24433
+ function llmsLinks(text, base) {
24434
+ const origin = new URL(base).origin;
24435
+ const links = [];
24436
+ for (const match of text.matchAll(/\]\(\s*<?([^)\s>]+)>?/g)) try {
24437
+ const url = new URL(match[1] ?? "", base);
24438
+ url.hash = "";
24439
+ if (url.origin === origin && !links.includes(url.toString())) links.push(url.toString());
24440
+ } catch {}
24441
+ return links;
24442
+ }
24443
+ async function checkLlmsTxt(input) {
24444
+ const result = await get(input, new URL("/llms.txt", input.url).toString());
24445
+ const availability = result.availability === "available" && isHtml(result.contentType) ? "absent" : result.availability;
24446
+ const exists = availability === "available";
24447
+ const base = {
24448
+ url: result.url,
24449
+ exists,
24450
+ availability,
24451
+ ...result.status === void 0 ? {} : { status: result.status },
24452
+ ...result.error ? { error: result.error } : {}
24453
+ };
24454
+ if (!exists) return base;
24455
+ const found = llmsLinks(result.text, result.url);
24456
+ const broken = [];
24457
+ let checked = 0;
24458
+ for (const url of found.slice(0, LLMS_LINK_LIMIT)) {
24459
+ if (input.signal?.aborted) break;
24460
+ checked += 1;
24461
+ const link = brokenLink(await get(input, url, void 0, false), false);
24462
+ if (link) broken.push(link);
24463
+ }
24464
+ return {
24465
+ ...base,
24466
+ links: {
24467
+ found: found.length,
24468
+ checked,
24469
+ limit: LLMS_LINK_LIMIT,
24470
+ broken
24471
+ }
24472
+ };
24473
+ }
24196
24474
  function parseRobots(robotsUrl, text) {
24197
24475
  return robotsParserModule(robotsUrl, text);
24198
24476
  }
@@ -24224,118 +24502,135 @@ function sitemapUrlsFromRobots(text) {
24224
24502
  return [...urls];
24225
24503
  }
24226
24504
  async function checkRobotsAiAccess(input) {
24227
- const robotsUrl = new URL("/robots.txt", input.url).toString();
24228
- const controller = abortController({
24229
- timeoutMs: input.timeoutMs,
24230
- signal: input.signal
24231
- });
24232
- try {
24233
- const response = await input.fetch(robotsUrl, {
24234
- profile: "bot",
24235
- redirect: "follow",
24236
- signal: controller.signal
24237
- });
24238
- const text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, "robots.txt response");
24239
- const available = response.status >= 200 && response.status < 300;
24240
- const absent = response.status >= 400 && response.status < 500 && response.status !== 429;
24241
- const availability = available ? "available" : [401, 403].includes(response.status) ? "access-blocked" : response.status === 429 ? "rate-limited" : absent ? "absent" : "unreachable";
24242
- const exists = available;
24243
- const declared = exists ? declaredUserAgents(text) : /* @__PURE__ */ new Set();
24244
- const parsed = parseRobots(robotsUrl, exists ? text : "");
24245
- return {
24246
- url: response.url || robotsUrl,
24505
+ const result = await get(input, new URL("/robots.txt", input.url).toString());
24506
+ const exists = result.availability === "available";
24507
+ const unknown = result.availability === "rate-limited" || result.availability === "unreachable";
24508
+ const declared = exists ? declaredUserAgents(result.text) : /* @__PURE__ */ new Set();
24509
+ const parsed = parseRobots(result.url, exists ? result.text : "");
24510
+ return {
24511
+ robotsTxt: {
24512
+ url: result.url,
24247
24513
  exists,
24248
- availability,
24249
- status: response.status,
24250
- ...["rate-limited", "unreachable"].includes(availability) ? { error: `robots.txt returned HTTP ${response.status}.` } : {},
24251
- sitemapUrls: exists ? sitemapUrlsFromRobots(text) : [],
24252
- contentSignals: exists ? contentSignalsFromRobots(text) : [],
24253
- botAccess: AI_BOT_USER_AGENTS.map((userAgent) => {
24254
- const lower = userAgent.toLowerCase();
24255
- return {
24256
- userAgent,
24257
- allowed: availability === "rate-limited" || availability === "unreachable" ? null : parsed.isAllowed(input.url, userAgent) ?? true,
24258
- declared: declared.has(lower),
24259
- coveredByWildcard: declared.has("*")
24260
- };
24261
- })
24262
- };
24263
- } catch (error) {
24264
- return {
24265
- url: robotsUrl,
24266
- exists: false,
24267
- availability: "unreachable",
24268
- error: error instanceof Error ? error.message : String(error),
24269
- sitemapUrls: [],
24270
- botAccess: AI_BOT_USER_AGENTS.map((userAgent) => ({
24514
+ availability: result.availability,
24515
+ ...result.status === void 0 ? {} : { status: result.status },
24516
+ ...unknown ? { error: result.error ?? `robots.txt returned HTTP ${result.status}.` } : {},
24517
+ sitemapUrls: exists ? sitemapUrlsFromRobots(result.text) : [],
24518
+ ...unknown ? {} : { contentSignals: exists ? contentSignalsFromRobots(result.text) : [] },
24519
+ botAccess: AI_BOTS.map(([userAgent, purpose]) => ({
24271
24520
  userAgent,
24272
- allowed: null,
24273
- declared: false,
24274
- coveredByWildcard: false
24521
+ purpose,
24522
+ allowed: unknown ? null : parsed.isAllowed(input.url, userAgent) ?? true,
24523
+ declared: declared.has(userAgent.toLowerCase()),
24524
+ coveredByWildcard: declared.has("*")
24275
24525
  }))
24276
- };
24277
- } finally {
24278
- controller.cleanup();
24279
- }
24526
+ },
24527
+ allows: (url) => unknown ? null : parsed.isAllowed(url, SEO_CRAWLER_USER_AGENT) ?? true
24528
+ };
24280
24529
  }
24281
- async function checkAgentResource(input) {
24282
- const url = new URL(input.path, input.baseUrl).toString();
24283
- const controller = abortController({
24284
- timeoutMs: input.timeoutMs,
24285
- signal: input.signal
24286
- });
24287
- try {
24288
- const response = await input.fetch(url, {
24289
- profile: "bot",
24290
- redirect: "follow",
24291
- signal: controller.signal
24292
- });
24293
- const contentType = response.headers.get("content-type") ?? "";
24294
- const text = await readBoundedResponseText(response, MAX_AUXILIARY_RESPONSE_BYTES, "Agent resource response");
24295
- const exists = response.status >= 200 && response.status < 300;
24530
+ async function checkAgentResources(input) {
24531
+ const resources = [];
24532
+ for (const path of AGENT_RESOURCE_PATHS) {
24533
+ if (input.signal?.aborted) break;
24534
+ const result = await get(input, new URL(path, input.url).toString());
24535
+ const exists = result.availability === "available" && !isHtml(result.contentType);
24296
24536
  let validJson;
24297
- if (exists && /\bjson\b/i.test(contentType)) try {
24298
- JSON.parse(text);
24537
+ if (exists && /\bjson\b/i.test(result.contentType)) try {
24538
+ JSON.parse(result.text);
24299
24539
  validJson = true;
24300
24540
  } catch {
24301
24541
  validJson = false;
24302
24542
  }
24303
- return {
24304
- url: response.url || url,
24543
+ resources.push({
24544
+ url: result.url,
24305
24545
  exists,
24306
- status: response.status,
24307
- contentType,
24546
+ ...result.status === void 0 ? {} : { status: result.status },
24547
+ ...result.contentType ? { contentType: result.contentType } : {},
24308
24548
  ...validJson === void 0 ? {} : { validJson }
24309
- };
24310
- } catch {
24311
- return {
24312
- url,
24313
- exists: false
24314
- };
24315
- } finally {
24316
- controller.cleanup();
24317
- }
24318
- }
24319
- async function checkAgentResources(input) {
24320
- const resources = [];
24321
- for (const path of AGENT_RESOURCE_PATHS) {
24322
- if (input.signal?.aborted) break;
24323
- resources.push(await checkAgentResource({
24324
- baseUrl: input.url,
24325
- path,
24326
- timeoutMs: input.timeoutMs,
24327
- fetch: input.fetch,
24328
- signal: input.signal
24329
- }));
24549
+ });
24330
24550
  }
24331
24551
  return resources;
24332
24552
  }
24333
- function collectCrawlAiSignals(input) {
24334
- return Promise.all([
24553
+ async function checkMarkdownNegotiation(input) {
24554
+ const result = await get(input, input.url, { accept: "text/markdown" }, false);
24555
+ const vary = result.headers?.get("vary") ?? "";
24556
+ return {
24557
+ url: result.url,
24558
+ availability: result.availability,
24559
+ ...result.status === void 0 ? {} : { status: result.status },
24560
+ ...result.contentType ? { contentType: result.contentType } : {},
24561
+ markdown: result.availability === "available" && isMarkdown(result.contentType),
24562
+ varyAccept: vary.split(",").some((value) => value.trim().toLowerCase() === "accept"),
24563
+ ...result.error ? { error: result.error } : {}
24564
+ };
24565
+ }
24566
+ async function collectCrawlAiSignals(input) {
24567
+ const robots = await checkRobotsAiAccess(input);
24568
+ const allows = input.respectRobots ? robots.allows : () => true;
24569
+ const startAllowed = allows(input.url);
24570
+ const [llmsTxt, agentResources, markdownNegotiation] = await Promise.all([
24335
24571
  checkLlmsTxt(input),
24336
- checkRobotsAiAccess(input),
24337
- checkAgentResources(input)
24572
+ checkAgentResources(input),
24573
+ startAllowed ? checkMarkdownNegotiation(input) : Promise.resolve({
24574
+ url: input.url,
24575
+ availability: "unreachable",
24576
+ markdown: false,
24577
+ varyAccept: false,
24578
+ error: startAllowed === null ? "Not requested: robots.txt could not be read." : "Not requested: robots.txt disallows the start URL."
24579
+ })
24338
24580
  ]);
24581
+ return {
24582
+ signals: {
24583
+ llmsTxt,
24584
+ robotsTxt: robots.robotsTxt,
24585
+ agentResources,
24586
+ markdownNegotiation
24587
+ },
24588
+ allows
24589
+ };
24590
+ }
24591
+ async function checkMarkdownCopies(input) {
24592
+ const linked = input.pages.filter((page) => page.markdownAlternates?.length).sort((a, b) => Number(a.public) - Number(b.public) || a.url.localeCompare(b.url));
24593
+ const broken = [];
24594
+ let largest;
24595
+ let checked = 0;
24596
+ for (const page of linked.slice(0, 20)) {
24597
+ if (input.signal?.aborted) break;
24598
+ const url = page.markdownAlternates?.[0];
24599
+ if (!url || !input.allows(url)) continue;
24600
+ checked += 1;
24601
+ const result = await get({
24602
+ ...input,
24603
+ url
24604
+ }, url);
24605
+ const link = brokenLink(result, true);
24606
+ if (link) {
24607
+ broken.push({
24608
+ ...link,
24609
+ page: page.url,
24610
+ public: page.public
24611
+ });
24612
+ continue;
24613
+ }
24614
+ const estimatedTokens = Math.ceil(result.text.length / 4);
24615
+ if (!largest || estimatedTokens > largest.estimatedTokens) largest = {
24616
+ url: result.url,
24617
+ estimatedTokens
24618
+ };
24619
+ }
24620
+ return {
24621
+ publicPages: input.pages.filter((page) => page.public).length,
24622
+ publicPagesWithLink: linked.filter((page) => page.public).length,
24623
+ privatePagesWithLink: linked.filter((page) => !page.public).length,
24624
+ checked,
24625
+ limit: 20,
24626
+ broken,
24627
+ ...largest ? { largest } : {}
24628
+ };
24629
+ }
24630
+ function linkRelations(header) {
24631
+ const relations = /* @__PURE__ */ new Set();
24632
+ for (const match of (header ?? "").matchAll(/rel\s*=\s*"?([^";,]+)"?/gi)) for (const value of (match[1] ?? "").trim().toLowerCase().split(/\s+/)) if (value) relations.add(value);
24633
+ return [...relations].sort();
24339
24634
  }
24340
24635
  //#endregion
24341
24636
  //#region packages/core/dist/analyze/crawler/site-crawl-providers.js
@@ -26743,12 +27038,11 @@ async function crawlSite(input, dependencies) {
26743
27038
  url: config.url,
26744
27039
  timeoutMs: config.timeoutMs,
26745
27040
  fetch: deps.fetch,
27041
+ respectRobots: config.respectRobots,
26746
27042
  signal
26747
27043
  });
26748
- const llmsTxt = aiSignals?.[0];
27044
+ const llmsTxt = aiSignals?.signals.llmsTxt;
26749
27045
  const healthSitemapUrls = healthStrategy && !config.sitemapUrl && !isCancelled() ? await robotsSession?.sitemapUrls(origin) : void 0;
26750
- const robotsTxt = aiSignals?.[1];
26751
- const agentResources = aiSignals?.[2];
26752
27046
  cancelled = isCancelled();
26753
27047
  const enqueue = (value, depth) => {
26754
27048
  const normalized = normalizeUrl$1(value, config.url);
@@ -26826,7 +27120,7 @@ async function crawlSite(input, dependencies) {
26826
27120
  url: config.url,
26827
27121
  sitemapUrl: config.sitemapUrl,
26828
27122
  maxPages: config.maxPages,
26829
- declaredSitemapUrls: robotsTxt?.sitemapUrls ?? healthSitemapUrls ?? [],
27123
+ declaredSitemapUrls: aiSignals?.signals.robotsTxt?.sitemapUrls ?? healthSitemapUrls ?? [],
26830
27124
  warnings,
26831
27125
  fetchSitemapUrls: deps.fetchSitemapUrls
26832
27126
  });
@@ -27142,6 +27436,26 @@ async function crawlSite(input, dependencies) {
27142
27436
  llmsTxtStatus: llmsTxt.status
27143
27437
  };
27144
27438
  }
27439
+ const startPage = pages.find((page) => page.url === config.url || page.finalUrl === config.url);
27440
+ const ai = aiSignals && !cancelled ? {
27441
+ ...aiSignals.signals,
27442
+ markdownCopies: await checkMarkdownCopies({
27443
+ allows: aiSignals.allows,
27444
+ pages: pages.map((page) => ({
27445
+ url: page.url,
27446
+ markdownAlternates: page.markdownAlternates,
27447
+ public: isPublicPage(page)
27448
+ })),
27449
+ timeoutMs: config.timeoutMs,
27450
+ fetch: deps.fetch,
27451
+ signal
27452
+ }),
27453
+ ...startPage ? { startPageHeaders: {
27454
+ url: startPage.url,
27455
+ linkRelations: linkRelations(startPage.responseHeaders?.link),
27456
+ ...startPage.responseHeaders?.["content-signal"] ? { contentSignal: startPage.responseHeaders["content-signal"] } : {}
27457
+ } } : {}
27458
+ } : aiSignals?.signals;
27145
27459
  const report = createCrawlReport({
27146
27460
  config,
27147
27461
  site,
@@ -27151,17 +27465,15 @@ async function crawlSite(input, dependencies) {
27151
27465
  pages,
27152
27466
  requests,
27153
27467
  requestEvidenceStatus,
27154
- ...!healthStrategy ? { ai: {
27155
- ...llmsTxt ? { llmsTxt } : {},
27156
- ...robotsTxt ? { robotsTxt } : {},
27157
- ...agentResources ? { agentResources } : {}
27158
- } } : {},
27468
+ ...ai ? { ai } : {},
27159
27469
  ...sitemapDiscovery ? { sitemapDiscovery } : {},
27160
27470
  ...externalLinkVerification ? { externalLinkVerification } : {},
27161
- ...siteCheckResult ? {
27162
- siteChecks: siteCheckResult.checks,
27163
- additionalIssues: siteCheckResult.issues
27164
- } : {},
27471
+ ...siteCheckResult ? { siteChecks: siteCheckResult.checks } : {},
27472
+ additionalIssues: [...siteCheckResult?.issues ?? [], ...ai ? agentReadyIssues({
27473
+ startUrl: config.url,
27474
+ ai,
27475
+ pages
27476
+ }) : []],
27165
27477
  status: reportStatus,
27166
27478
  warnings,
27167
27479
  caveats: crawlCaveats({
@@ -31571,9 +31883,9 @@ function strikingDistance(input, dependencies = defaultDependencies$5) {
31571
31883
  //#region packages/core/dist/updates/search-status.js
31572
31884
  const INCIDENTS_URL = "https://status.search.google.com/incidents.json";
31573
31885
  async function fetchSearchStatusIncidents() {
31574
- const response = await fetch$1(INCIDENTS_URL);
31886
+ const response = await fetch$1(INCIDENTS_URL, { signal: AbortSignal.timeout(8e3) });
31575
31887
  if (!response.ok) throw new SeoError("OPTIONAL_PROVIDER_UNAVAILABLE", `Google Search Status fetch failed with ${response.status}.`);
31576
- return await response.json();
31888
+ return readJson(response, "Google Search Status", { maxBytes: 10 * 1024 * 1024 });
31577
31889
  }
31578
31890
  async function listSearchUpdates(input = {}) {
31579
31891
  return searchUpdatesFromIncidents(await fetchSearchStatusIncidents(), input);
@@ -34856,7 +35168,7 @@ async function fetchCruxFieldData(input) {
34856
35168
  headers: { "content-type": "application/json" },
34857
35169
  signal: controller.signal
34858
35170
  });
34859
- const json = await response.json().catch(() => ({}));
35171
+ const json = await readJson(response, "CrUX").catch(() => ({}));
34860
35172
  if (response.status === 404) return {
34861
35173
  response,
34862
35174
  json
@@ -37585,6 +37897,340 @@ async function runAnalyticsRows(input, google, options) {
37585
37897
  };
37586
37898
  }
37587
37899
  //#endregion
37900
+ //#region packages/core/dist/ga4/matching.js
37901
+ function hostnameFromUrl(value) {
37902
+ try {
37903
+ const url = new URL(value);
37904
+ if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
37905
+ return url.hostname.toLowerCase().replace(/\.$/, "") || void 0;
37906
+ } catch {
37907
+ return;
37908
+ }
37909
+ }
37910
+ function domainFromProperty(value) {
37911
+ if (!value.startsWith("sc-domain:")) return void 0;
37912
+ const domain = value.slice(10).trim().toLowerCase();
37913
+ if (!domain || /[/:]/.test(domain)) return void 0;
37914
+ return domain.replace(/\.$/, "") || void 0;
37915
+ }
37916
+ function matchForSite(site, streamHostname) {
37917
+ const domain = domainFromProperty(site);
37918
+ if (domain) return streamHostname === domain || streamHostname.endsWith(`.${domain}`) ? "domain" : void 0;
37919
+ return hostnameFromUrl(site) === streamHostname ? "hostname" : void 0;
37920
+ }
37921
+ function matchGa4WebStreams(site, candidates) {
37922
+ return candidates.flatMap((candidate) => {
37923
+ const defaultUri = candidate.stream.webStreamData?.defaultUri;
37924
+ if (!defaultUri) return [];
37925
+ const hostname = hostnameFromUrl(defaultUri);
37926
+ if (!hostname) return [];
37927
+ const match = matchForSite(site, hostname);
37928
+ if (!match) return [];
37929
+ return [{
37930
+ ...candidate,
37931
+ defaultUri,
37932
+ hostname,
37933
+ match
37934
+ }];
37935
+ });
37936
+ }
37937
+ //#endregion
37938
+ //#region packages/core/dist/commands/analytics-site.js
37939
+ const STREAM_CHECKS = 30;
37940
+ const AT_ONCE = 5;
37941
+ const TIED = 3;
37942
+ const hostOf$1 = (site) => {
37943
+ const value = site.replace(/^sc-domain:/, "");
37944
+ try {
37945
+ return new URL(value.includes("://") ? value : `https://${value}`).hostname;
37946
+ } catch {
37947
+ return value;
37948
+ }
37949
+ };
37950
+ const plain = (value) => value.toLowerCase().replace(/[^a-z0-9]/g, "");
37951
+ function ranked(site, candidates) {
37952
+ const host = hostOf$1(site).replace(/^www\./, "");
37953
+ const words = [plain(host), plain(host.split(".")[0] ?? "")].filter((word) => word.length >= 3);
37954
+ const named = (candidate) => words.some((word) => plain(candidate.propertyName).includes(word)) ? 0 : 1;
37955
+ return [...candidates].sort((a, b) => named(a) - named(b) || a.property.localeCompare(b.property, "en", { numeric: true }));
37956
+ }
37957
+ function best(site, matches) {
37958
+ const host = hostOf$1(site).replace(/^www\./, "");
37959
+ const score = (match) => match.hostname === host || match.hostname === `www.${host}` ? 0 : 1;
37960
+ return [...matches].sort((a, b) => score(a) - score(b) || a.hostname.length - b.hostname.length || a.property.localeCompare(b.property, "en", { numeric: true }));
37961
+ }
37962
+ async function findProperty(access, site) {
37963
+ const accounts = await access.accounts();
37964
+ const listed = await Promise.all(accounts.map((account) => listAnalyticsAccounts(account.google, { signIn: access.signIn }).then((summaries) => ({
37965
+ account,
37966
+ summaries
37967
+ }), (error) => error instanceof Error ? error : new Error(String(error)))));
37968
+ const seen = /* @__PURE__ */ new Set();
37969
+ const candidates = [];
37970
+ for (const entry of listed) {
37971
+ if (entry instanceof Error) continue;
37972
+ for (const summary of entry.summaries) for (const item of summary.propertySummaries) {
37973
+ const property = item.property.replace(/^properties\//, "");
37974
+ if (seen.has(property) || !/^\d+$/.test(property)) continue;
37975
+ seen.add(property);
37976
+ candidates.push({
37977
+ account: summary.account,
37978
+ property,
37979
+ propertyName: item.displayName ?? property,
37980
+ google: entry.account.google
37981
+ });
37982
+ }
37983
+ }
37984
+ if (!candidates.length) {
37985
+ const failed = listed.find((entry) => entry instanceof Error);
37986
+ return failed ? `could not be read. ${failed.message}` : "the connected Google accounts cannot read any property.";
37987
+ }
37988
+ const order = ranked(site, candidates).slice(0, STREAM_CHECKS);
37989
+ let failures = 0;
37990
+ for (let start = 0; start < order.length; start += AT_ONCE) {
37991
+ const batch = order.slice(start, start + AT_ONCE);
37992
+ const streams = await Promise.all(batch.map((candidate) => listAnalyticsDataStreams(candidate.google, candidate.property, { signIn: access.signIn }).catch(() => {
37993
+ failures += 1;
37994
+ return [];
37995
+ })));
37996
+ const matches = best(site, batch.flatMap((candidate, index) => matchGa4WebStreams(site, (streams[index] ?? []).map((stream) => ({
37997
+ ...candidate,
37998
+ stream
37999
+ })))));
38000
+ const found = /* @__PURE__ */ new Map();
38001
+ for (const match of matches) {
38002
+ const candidate = batch.find((item) => item.property === match.property);
38003
+ if (!candidate || found.has(match.property)) continue;
38004
+ found.set(match.property, {
38005
+ property: match.property,
38006
+ name: match.propertyName,
38007
+ stream: match.defaultUri,
38008
+ google: candidate.google
38009
+ });
38010
+ }
38011
+ if (found.size) return [...found.values()].slice(0, TIED);
38012
+ }
38013
+ return `no web stream matches ${site} in ${order.length < candidates.length ? `the first ${order.length} of ${candidates.length} properties, named like the site first` : `${candidates.length} ${candidates.length === 1 ? "property" : "properties"}`}${failures ? ` (${failures} could not be read)` : ""}.`;
38014
+ }
38015
+ const AI_SOURCES = [
38016
+ "chatgpt.com",
38017
+ "chatgpt",
38018
+ "chat.openai.com",
38019
+ "perplexity.ai",
38020
+ "perplexity",
38021
+ "claude.ai",
38022
+ "gemini.google.com",
38023
+ "copilot.microsoft.com",
38024
+ "copilot.com",
38025
+ "chat.deepseek.com",
38026
+ "meta.ai",
38027
+ "grok.com",
38028
+ "chat.mistral.ai",
38029
+ "you.com",
38030
+ "phind.com",
38031
+ "poe.com"
38032
+ ];
38033
+ const AI_SOURCE_PATTERN = `(www\\.)?(${AI_SOURCES.map((source) => source.replaceAll(".", "\\.")).join("|")})`;
38034
+ const AI_ROWS = 5;
38035
+ const filter = (fieldName, value, matchType = "EXACT") => ({ filter: {
38036
+ fieldName,
38037
+ stringFilter: {
38038
+ matchType,
38039
+ value
38040
+ }
38041
+ } });
38042
+ async function totals$1(found, access, range, before, pages) {
38043
+ const read = (dimensions, filters) => runAnalyticsReport(found.google, found.property, {
38044
+ dateRanges: [{
38045
+ ...range,
38046
+ name: "now"
38047
+ }, {
38048
+ ...before,
38049
+ name: "before"
38050
+ }],
38051
+ dimensions: dimensions.map((name) => ({ name })),
38052
+ metrics: [{ name: "sessions" }, { name: "keyEvents" }],
38053
+ ...filters.length ? { dimensionFilter: { andGroup: { expressions: filters } } } : {},
38054
+ limit: 100
38055
+ }, { signIn: access.signIn });
38056
+ const onPages = pages ? [filter("landingPage", pages, "CONTAINS")] : [];
38057
+ const fromAi = [...onPages, filter("sessionSource", AI_SOURCE_PATTERN, "FULL_REGEXP")];
38058
+ const [channels, google, aiSources, aiPages] = await Promise.all([
38059
+ read(["sessionDefaultChannelGroup"], onPages),
38060
+ read([], [
38061
+ ...onPages,
38062
+ filter("sessionDefaultChannelGroup", "Organic Search"),
38063
+ filter("sessionSource", "google")
38064
+ ]),
38065
+ read(["sessionSource"], fromAi),
38066
+ read(["landingPage"], fromAi)
38067
+ ]);
38068
+ const sum = () => ({
38069
+ google: {
38070
+ sessions: 0,
38071
+ keyEvents: 0
38072
+ },
38073
+ organic: {
38074
+ sessions: 0,
38075
+ keyEvents: 0
38076
+ },
38077
+ all: {
38078
+ sessions: 0,
38079
+ keyEvents: 0
38080
+ }
38081
+ });
38082
+ const periods = {
38083
+ now: sum(),
38084
+ before: sum()
38085
+ };
38086
+ const each = (data, add) => {
38087
+ const periodAt = (data.dimensionHeaders?.map((header) => header.name) ?? []).indexOf("dateRange");
38088
+ for (const row of data.rows ?? []) {
38089
+ const values = row.dimensionValues ?? [];
38090
+ add((periodAt >= 0 ? values[periodAt] : values.at(-1))?.value === "before" ? periods.before : periods.now, values[0]?.value ?? "", {
38091
+ sessions: Number(row.metricValues?.[0]?.value ?? 0),
38092
+ keyEvents: Number(row.metricValues?.[1]?.value ?? 0)
38093
+ });
38094
+ }
38095
+ };
38096
+ const plus = (into, row) => {
38097
+ into.sessions += row.sessions;
38098
+ into.keyEvents += row.keyEvents;
38099
+ };
38100
+ each(channels, (period, channel, row) => {
38101
+ plus(period.all, row);
38102
+ if (channel === "Organic Search") plus(period.organic, row);
38103
+ });
38104
+ each(google, (period, _, row) => plus(period.google, row));
38105
+ const byName = (data, key) => {
38106
+ const rows = /* @__PURE__ */ new Map();
38107
+ const periodAt = (data.dimensionHeaders?.map((header) => header.name) ?? []).indexOf("dateRange");
38108
+ for (const row of data.rows ?? []) {
38109
+ const values = row.dimensionValues ?? [];
38110
+ const name = values[0]?.value ?? "";
38111
+ const before = (periodAt >= 0 ? values[periodAt] : values.at(-1))?.value === "before";
38112
+ const entry = rows.get(name) ?? {
38113
+ now: 0,
38114
+ before: 0,
38115
+ keyEvents: 0
38116
+ };
38117
+ const sessions = Number(row.metricValues?.[0]?.value ?? 0);
38118
+ if (before) entry.before += sessions;
38119
+ else {
38120
+ entry.now += sessions;
38121
+ entry.keyEvents += Number(row.metricValues?.[1]?.value ?? 0);
38122
+ }
38123
+ rows.set(name, entry);
38124
+ }
38125
+ return [...rows].map(([name, entry]) => ({
38126
+ [key]: name,
38127
+ sessions: entry.now,
38128
+ previousSessions: entry.before,
38129
+ keyEvents: entry.keyEvents
38130
+ })).sort((a, b) => b.sessions - a.sessions || b.previousSessions - a.previousSessions || String(a[key]).localeCompare(String(b[key])));
38131
+ };
38132
+ const bySource = byName(aiSources, "source");
38133
+ const aiNow = bySource.reduce((total, row) => total + row.sessions, 0);
38134
+ const aiBefore = bySource.reduce((total, row) => total + row.previousSessions, 0);
38135
+ const data = { metadata: {
38136
+ timeZone: channels.metadata?.timeZone,
38137
+ subjectToThresholding: [
38138
+ channels,
38139
+ google,
38140
+ aiSources,
38141
+ aiPages
38142
+ ].some((data) => data.metadata?.subjectToThresholding),
38143
+ dataLossFromOtherRow: [
38144
+ channels,
38145
+ google,
38146
+ aiSources,
38147
+ aiPages
38148
+ ].some((data) => data.metadata?.dataLossFromOtherRow)
38149
+ } };
38150
+ const notes = [];
38151
+ if (data.metadata?.timeZone && data.metadata.timeZone !== "America/Los_Angeles") notes.push(`Google Analytics days use the property's time zone, ${data.metadata.timeZone}; Search Console days use Pacific time.`);
38152
+ if (data.metadata?.subjectToThresholding) notes.push("Google Analytics applied thresholding; small rows can be hidden.");
38153
+ if (data.metadata?.dataLossFromOtherRow) notes.push("Google Analytics grouped some rows into (other).");
38154
+ return {
38155
+ property: found.property,
38156
+ propertyName: found.name,
38157
+ webStream: found.stream,
38158
+ ...pages ? { landingPageContains: pages } : {},
38159
+ organicSearchFromGoogle: {
38160
+ sessions: periods.now.google.sessions,
38161
+ previousSessions: periods.before.google.sessions,
38162
+ sessionsChangePct: pct(periods.now.google.sessions, periods.before.google.sessions)
38163
+ },
38164
+ organicSearch: {
38165
+ sessions: periods.now.organic.sessions,
38166
+ previousSessions: periods.before.organic.sessions,
38167
+ sessionsChangePct: pct(periods.now.organic.sessions, periods.before.organic.sessions),
38168
+ keyEvents: periods.now.organic.keyEvents,
38169
+ previousKeyEvents: periods.before.organic.keyEvents
38170
+ },
38171
+ allChannels: {
38172
+ sessions: periods.now.all.sessions,
38173
+ previousSessions: periods.before.all.sessions,
38174
+ sessionsChangePct: pct(periods.now.all.sessions, periods.before.all.sessions)
38175
+ },
38176
+ aiAssistants: {
38177
+ sessions: aiNow,
38178
+ previousSessions: aiBefore,
38179
+ sessionsChangePct: pct(aiNow, aiBefore),
38180
+ bySource,
38181
+ topLandingPages: byName(aiPages, "page").slice(0, 20),
38182
+ sources: AI_SOURCES
38183
+ },
38184
+ notes
38185
+ };
38186
+ }
38187
+ const change$1 = (value) => value === null ? "n/a" : `${value > 0 ? "+" : ""}${value}%`;
38188
+ function aiText(ai) {
38189
+ const list = (rows, key) => rows.slice(0, AI_ROWS).map((row) => `${row[key]} ${row.sessions} (was ${row.previousSessions})`).join(", ");
38190
+ if (!ai.sessions && !ai.previousSessions) return "Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): none in either period.";
38191
+ return [
38192
+ `Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): ${ai.sessions} sessions (was ${ai.previousSessions}, ${change$1(ai.sessionsChangePct)}). By source: ${list(ai.bySource, "source")}.`,
38193
+ ai.topLandingPages.length ? `Top landing pages from AI assistants: ${list(ai.topLandingPages, "page")}.` : "",
38194
+ "Only visits that pass a referrer or utm_source count; AI apps that send none show as Direct."
38195
+ ].filter(Boolean).join(" ");
38196
+ }
38197
+ async function analyticsBeside(access, site, range, before, pages) {
38198
+ if (!access) return null;
38199
+ try {
38200
+ const found = await findProperty(access, site);
38201
+ if (typeof found === "string") return {
38202
+ text: `Google Analytics: ${found}`,
38203
+ data: { error: found }
38204
+ };
38205
+ const read = await Promise.all(found.map((item) => totals$1(item, access, range, before, pages).catch((error) => error instanceof Error ? error : new Error(String(error)))));
38206
+ const data = [...read.filter((item) => !(item instanceof Error))].sort((a, b) => b.allChannels.sessions - a.allChannels.sessions)[0];
38207
+ if (!data) throw read[0];
38208
+ const others = found.filter((item) => item.property !== data.property).map((item) => `${item.name} (${item.property})`);
38209
+ const google = data.organicSearchFromGoogle;
38210
+ const organic = data.organicSearch;
38211
+ const all = data.allChannels;
38212
+ const keyEvents = organic.keyEvents || organic.previousKeyEvents ? `, Organic Search key events ${organic.keyEvents} (was ${organic.previousKeyEvents})` : "";
38213
+ return {
38214
+ text: [
38215
+ `Google Analytics, property ${data.propertyName} (${data.property}), matched by its web stream ${data.webStream}${pages ? `, landing pages whose path contains "${pages}"` : ""}: Organic Search sessions from Google ${google.sessions} (was ${google.previousSessions}, ${change$1(google.sessionsChangePct)}), from every search engine ${organic.sessions} (was ${organic.previousSessions}, ${change$1(organic.sessionsChangePct)})${keyEvents}, sessions from all channels ${all.sessions} (was ${all.previousSessions}, ${change$1(all.sessionsChangePct)}).`,
38216
+ "Sessions are Google Analytics’ own count, not Search Console clicks.",
38217
+ ...data.notes,
38218
+ ...others.length ? [`Other properties with a matching web stream, with fewer sessions: ${others.join(", ")}.`] : []
38219
+ ].join(" ").concat("\n\n", aiText(data.aiAssistants)),
38220
+ data: {
38221
+ ...data,
38222
+ ...others.length ? { otherMatches: others } : {}
38223
+ }
38224
+ };
38225
+ } catch (error) {
38226
+ const message = error instanceof Error ? error.message : String(error);
38227
+ return {
38228
+ text: `Google Analytics: could not be read. ${message}`,
38229
+ data: { error: message }
38230
+ };
38231
+ }
38232
+ }
38233
+ //#endregion
37588
38234
  //#region packages/core/dist/commands/bing.js
37589
38235
  const BING_LIMITS = ["Bing and Google count clicks, impressions and positions their own way. Their numbers sit side by side here and are never added together.", "Bing lists only its top queries and pages, grouped by week. A query or page missing from Bing’s list is unknown, not zero."];
37590
38236
  const CLI_LINK = "If you have a shell, link the CLI to the same account once: run `npx -y seo login`, show the person the link it prints, and after they approve it run `npx -y seo login` again. `npx -y seo` commands then use the same Google accounts, Bing and keys.";
@@ -37987,7 +38633,7 @@ async function lookup(domain, key) {
37987
38633
  signal: AbortSignal.timeout(5e3)
37988
38634
  });
37989
38635
  if (!response.ok) return null;
37990
- const body = await response.clone().json();
38636
+ const body = await readJson(response.clone(), "Ahrefs Domain Rating");
37991
38637
  if (cache && !cached) {
37992
38638
  const copy = new Response(response.body, response);
37993
38639
  copy.headers.set("cache-control", `max-age=${TTL_SECONDS}`);
@@ -38146,7 +38792,7 @@ function temporaryRedirectNote(pages) {
38146
38792
  //#endregion
38147
38793
  //#region packages/core/dist/commands/report.js
38148
38794
  const SECTIONS = {
38149
- overview: "Site totals (clicks, impressions, CTR, position) against the previous period, by device, with the top pages and queries and their change.",
38795
+ overview: "Site totals (clicks, impressions, CTR, position) against the previous period, by device, with the top pages and queries and their change, and Google Analytics sessions for the same dates from the property whose web stream matches the site.",
38150
38796
  "quick-wins": "Query and page pairs at average positions 4 to 10 with a click-through rate below the site’s own curve for that position.",
38151
38797
  "low-ctr": "Query and page pairs at average positions 1 to 4 with a click-through rate below the site’s own curve: titles and snippets to review with the person.",
38152
38798
  "striking-distance": "Queries just outside the top positions, with impressions, for pages that already rank.",
@@ -38453,10 +39099,14 @@ async function allSections(input, deps) {
38453
39099
  const origin = originOf(input.site);
38454
39100
  const bing = once(() => bingFor(deps, input.site));
38455
39101
  const ids = SECTION_IDS.filter((id) => id !== "bing" || deps.bing);
39102
+ const range = dateWindow(input, DEFAULT_DAYS.overview);
39103
+ const analytics = analyticsBeside(deps.analytics, input.site, range, earlierRange(range));
38456
39104
  const [overview, ...rest] = await Promise.all(ids.map((id) => build(read, input.site, id, dateWindow(input, DEFAULT_DAYS[id]), now, bing).catch((error) => error instanceof Error ? error : new Error(String(error)))));
38457
39105
  if (!overview || overview instanceof Error) throw overview ?? /* @__PURE__ */ new Error("The overview did not run.");
38458
39106
  const parts = [overviewText(overview, `Report on ${input.site}`, origin, 5)];
38459
39107
  parts.push(deps.bing ? await enginesLine(overview, bing) : NO_BING_HERE);
39108
+ const ga = await analytics;
39109
+ if (ga) parts.push(ga.text);
38460
39110
  const brand = rest.find((run) => !(run instanceof Error) && brandNote(run) !== null);
38461
39111
  if (brand) parts.push(brandNote(brand));
38462
39112
  const sections = {};
@@ -38486,6 +39136,7 @@ async function allSections(input, deps) {
38486
39136
  data: {
38487
39137
  site: input.site,
38488
39138
  overview,
39139
+ ...ga ? { analytics: ga.data } : {},
38489
39140
  sections
38490
39141
  }
38491
39142
  };
@@ -38499,14 +39150,25 @@ async function oneSection(input, deps) {
38499
39150
  const bing = once(() => bingFor(deps, site));
38500
39151
  let report = await build(sharedReader(deps.searchAnalytics), site, section, range, now, bing);
38501
39152
  const next = nextFor(section, deps.call);
38502
- if (section === "overview" && detail !== "full") return {
38503
- text: `${overviewText(report, `overview of ${site}`, origin, 10)}\n\n${deps.bing ? await enginesLine(report, bing) : NO_BING_HERE}\n\nMore: ${deps.call("report", {
38504
- site,
38505
- section,
38506
- detail: "full"
38507
- })} for every row.\n\n${next}\n\n${FACTS_ONLY}`,
38508
- data: report
38509
- };
39153
+ if (section === "overview") {
39154
+ const ga = await analyticsBeside(deps.analytics, site, range, earlierRange(range));
39155
+ if (ga) report.analytics = ga.data;
39156
+ if (detail !== "full") return {
39157
+ text: [
39158
+ overviewText(report, `overview of ${site}`, origin, 10),
39159
+ deps.bing ? await enginesLine(report, bing) : NO_BING_HERE,
39160
+ ga?.text,
39161
+ `More: ${deps.call("report", {
39162
+ site,
39163
+ section,
39164
+ detail: "full"
39165
+ })} for every row.`,
39166
+ next,
39167
+ FACTS_ONLY
39168
+ ].filter(Boolean).join("\n\n"),
39169
+ data: report
39170
+ };
39171
+ }
38510
39172
  let checked = [];
38511
39173
  const merged = [];
38512
39174
  if (PAGE_CHECKS.has(section)) {
@@ -38614,7 +39276,7 @@ async function dataForSeo([login, password], input) {
38614
39276
  language_code: language ?? mainLanguage(input.country)
38615
39277
  };
38616
39278
  const post = async (path, task) => {
38617
- const result = (await (await call("dataforseo", new Request(`https://api.dataforseo.com/${path}`, {
39279
+ const result = (await readJson(await call("dataforseo", new Request(`https://api.dataforseo.com/${path}`, {
38618
39280
  method: "POST",
38619
39281
  headers: {
38620
39282
  authorization: `Basic ${btoa(`${login}:${password}`)}`,
@@ -38624,7 +39286,7 @@ async function dataForSeo([login, password], input) {
38624
39286
  ...market,
38625
39287
  ...task
38626
39288
  }])
38627
- }))).json()).tasks?.[0];
39289
+ })), "DataForSEO")).tasks?.[0];
38628
39290
  if (!result || result.status_code >= 4e4) throw new CommandError("PROVIDER_ERROR", `dataforseo: ${result?.status_message ?? "no task result"}`);
38629
39291
  return result.result?.[0]?.items ?? [];
38630
39292
  };
@@ -38705,10 +39367,10 @@ async function ahrefs(key, input) {
38705
39367
  const get = async (path, params) => {
38706
39368
  const url = new URL(`https://api.ahrefs.com/v3/${path}`);
38707
39369
  for (const [name, value] of Object.entries(params)) url.searchParams.set(name, value);
38708
- return await (await call("ahrefs", new Request(url, { headers: {
39370
+ return readJson(await call("ahrefs", new Request(url, { headers: {
38709
39371
  authorization: `Bearer ${key}`,
38710
39372
  accept: "application/json"
38711
- } }))).json();
39373
+ } })), "Ahrefs");
38712
39374
  };
38713
39375
  const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
38714
39376
  if (input.action === "keyword_metrics") return ((await get("keywords-explorer/overview", {
@@ -38996,19 +39658,25 @@ async function runCompare(input, deps) {
38996
39658
  startDate: shiftIsoDate(input.date, -days),
38997
39659
  endDate: shiftIsoDate(input.date, -1)
38998
39660
  };
38999
- const [report, updates] = await Promise.all([siteOverview(sharedReader(deps.searchAnalytics), input.site, after, before, input.pages), googleUpdates({
39000
- start: before.startDate,
39001
- end: after.endDate
39002
- })]);
39661
+ const [report, updates, analytics] = await Promise.all([
39662
+ siteOverview(sharedReader(deps.searchAnalytics), input.site, after, before, input.pages),
39663
+ googleUpdates({
39664
+ start: before.startDate,
39665
+ end: after.endDate
39666
+ }),
39667
+ analyticsBeside(deps.analytics, input.site, after, before, input.pages)
39668
+ ]);
39003
39669
  return {
39004
39670
  text: [
39005
39671
  overviewText(report, `Compare on ${input.site}: ${days} days after ${input.date}`, originOf(input.site), 10),
39672
+ analytics?.text,
39006
39673
  listOfUpdates(updates),
39007
39674
  `Next: ${deps.call("page")} for a page that moved.`,
39008
39675
  FACTS_ONLY
39009
39676
  ].filter(Boolean).join("\n\n"),
39010
39677
  data: {
39011
39678
  ...report,
39679
+ ...analytics ? { analytics: analytics.data } : {},
39012
39680
  googleUpdates: updates
39013
39681
  }
39014
39682
  };
@@ -39225,7 +39893,7 @@ const SHARED_TOOLS = [
39225
39893
  name: "seo_compare",
39226
39894
  command: "compare",
39227
39895
  title: "Before and after a date",
39228
- description: "Compares equal periods before and after a date, such as a site change or a Google update: totals, devices, and the pages and queries that moved, with Google updates in the window. Optional pages narrows it to URLs that contain that text, such as /blog/.",
39896
+ description: "Compares equal periods before and after a date, such as a site change or a Google update: totals, devices, and the pages and queries that moved, Google Analytics sessions from the property whose web stream matches the site, and Google updates in the window. Optional pages narrows it to URLs that contain that text, such as /blog/.",
39229
39897
  input: z.object({
39230
39898
  site,
39231
39899
  date: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).describe("YYYY-MM-DD: the first day after the change."),
@@ -39342,8 +40010,7 @@ async function runSharedTool(tool, surface, input) {
39342
40010
  const MCP_INSTRUCTIONS = `SEO Skill reads the person's own Search Console, Google Analytics, Bing Webmaster Tools, Clicky and optional keyword accounts, and returns facts with their dates and limits. It gives no advice.
39343
40011
  1. seo_sites: the properties the connected accounts can read.
39344
40012
  2. seo_report: the main report. Run it without a section first; it shows the overview and every section's count and top rows. Then ask for the one section you need.
39345
- 3. seo_page for one URL, seo_compare for before and after a date, seo_index for pages that lost their clicks, seo_research for keyword, competitor and backlink data from the DataForSEO or Ahrefs key the person saved, seo_data for raw rows, including Google Analytics, Bing and Clicky.
39346
- Show the person the facts that matter, for example in a table or a small dashboard, and decide with them what to change. Do not edit pages without their approval.`;
40013
+ 3. seo_page for one URL, seo_compare for before and after a date, seo_index for pages that lost their clicks, seo_research for keyword, competitor and backlink data from the DataForSEO or Ahrefs key the person saved, seo_data for raw rows, including Google Analytics, Bing and Clicky.`;
39347
40014
  const mcpCall = (command, args) => {
39348
40015
  const name = `seo_${command.replace("-", "_")}`;
39349
40016
  const shown = Object.entries(args ?? {}).filter(([, value]) => value !== void 0);
@@ -39353,7 +40020,7 @@ const SKILL_RESOURCE = {
39353
40020
  name: "seo-skill",
39354
40021
  uri: "skill://seo/SKILL.md",
39355
40022
  title: "SEO Skill",
39356
- description: "How to use these tools: the commands, the one-report flow and the evidence rules.",
40023
+ description: "How to use these tools: what each command returns and how to reach it.",
39357
40024
  mimeType: "text/markdown"
39358
40025
  };
39359
40026
  //#endregion
@@ -39550,7 +40217,11 @@ function nodeCommandDeps(options) {
39550
40217
  pageMarkdown: async (url) => extractMainMarkdown(await fetchPage(url, { refresh: options.refresh })),
39551
40218
  call: options.call,
39552
40219
  bing: localBing,
39553
- bingConnect: LOCAL_BING_CONNECT
40220
+ bingConnect: LOCAL_BING_CONNECT,
40221
+ analytics: {
40222
+ accounts: localAccounts,
40223
+ signIn: LOCAL_SIGN_IN.signIn
40224
+ }
39554
40225
  };
39555
40226
  }
39556
40227
  const LOCAL_BING_CONNECT = "Connect Bing on your seoskill.dev account and run `seo login`, or run `seo providers bing connect` with an API key from Bing Webmaster Tools (Settings, API access).";