seo 0.15.2 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/mcp.js CHANGED
@@ -1,11 +1,11 @@
1
1
  import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
2
- import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-Dt4JJZ0P.js";
2
+ import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-Cvmg0qhZ.js";
3
3
  import { createHash, randomUUID, sign } from "node:crypto";
4
4
  import { ZodOptional, z } from "zod";
5
5
  import { arch, homedir, platform, tmpdir, totalmem } from "node:os";
6
6
  import { dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
7
7
  import "open";
8
- import { chmodSync, createReadStream, existsSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
8
+ import { chmodSync, createReadStream, existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
9
9
  import lockfile from "proper-lockfile";
10
10
  import "node:http";
11
11
  import { Headers as Headers$1, fetch as fetch$1 } from "undici";
@@ -12461,6 +12461,24 @@ const RULE_DEFINITIONS = [
12461
12461
  suggestedCommands: ["seo crawl <url> --max-pages 1 --json"]
12462
12462
  }
12463
12463
  },
12464
+ {
12465
+ id: "crawl_single_page_reach",
12466
+ title: "Crawl could reach only the start page",
12467
+ category: "links",
12468
+ defaultSeverity: "high",
12469
+ whyItMatters: "The start page had no crawlable internal links and no sitemap returned URLs, so the crawl checked one page. Every other finding in this report covers that page only. The page probably builds its links with JavaScript.",
12470
+ howToFix: "Rerun with `--rendering on` so the crawler reads the links a browser sees, or pass the pages to check with `--mode list`. For crawlers that do not run JavaScript, put plain `<a href>` links in the initial HTML and publish a sitemap.",
12471
+ impactIfIgnored: "Crawlers and audits that do not run JavaScript can find only this page, so the rest of the site stays unchecked and harder to discover.",
12472
+ howToVerify: "Rerun the crawl and confirm it retains more than one page and the start page reports internal links.",
12473
+ agentHints: {
12474
+ evidenceFields: [
12475
+ "page.outgoingInternalCount",
12476
+ "report.sitemapDiscovery.urlsReturned",
12477
+ "page.fetchDiagnostics.rendering"
12478
+ ],
12479
+ suggestedCommands: ["seo crawl <url> --rendering on --json", "seo crawl --mode list --urls <url>,<url> --json"]
12480
+ }
12481
+ },
12464
12482
  {
12465
12483
  id: "soft_404",
12466
12484
  title: "Unknown URLs return a successful response",
@@ -12469,7 +12487,7 @@ const RULE_DEFINITIONS = [
12469
12487
  whyItMatters: "Known-nonexistent URLs returned a successful response or redirected to another successful page. Search engines and users cannot reliably distinguish missing content from a real page.",
12470
12488
  howToFix: "Return 404 or 410 for missing URLs. If a replacement exists, redirect only the specific retired URL to its closest relevant replacement rather than sending every unknown path to a generic page.",
12471
12489
  impactIfIgnored: "Missing URLs can remain in crawl queues, waste crawl attention, and create confusing search or monitoring evidence.",
12472
- howToVerify: "Request several known-nonexistent paths and confirm each returns 404 or 410 without redirecting to a generic successful page.",
12490
+ howToVerify: "Request several made-up paths, including one at the site root and one nested path, and confirm each returns 404 or 410 without redirecting to a generic successful page.",
12473
12491
  agentHints: {
12474
12492
  evidenceFields: ["report.siteChecks.soft404"],
12475
12493
  suggestedCommands: ["seo crawl <url> --json"]
@@ -13918,7 +13936,7 @@ function day$3(value) {
13918
13936
  const date = legacy ? new Date(Number(legacy[1])) : new Date(value);
13919
13937
  return Number.isNaN(date.getTime()) ? null : date.toISOString().slice(0, 10);
13920
13938
  }
13921
- const count$3 = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
13939
+ const count$4 = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
13922
13940
  function list$2(data, label) {
13923
13941
  if (!Array.isArray(data)) throw new SeoError("OPTIONAL_PROVIDER_UNAVAILABLE", `Bing Webmaster returned an invalid ${label}.`);
13924
13942
  return data.filter((item) => Boolean(item) && typeof item === "object");
@@ -13934,8 +13952,8 @@ async function bingTraffic(call, site) {
13934
13952
  let invalidRows = 0;
13935
13953
  for (const row of list$2(await call("GetRankAndTrafficStats", { siteUrl: site }), "traffic list")) {
13936
13954
  const date = day$3(row.Date);
13937
- const clicks = count$3(row.Clicks);
13938
- const impressions = count$3(row.Impressions);
13955
+ const clicks = count$4(row.Clicks);
13956
+ const impressions = count$4(row.Impressions);
13939
13957
  if (!date || clicks === null || impressions === null) {
13940
13958
  invalidRows += 1;
13941
13959
  continue;
@@ -13958,9 +13976,9 @@ async function keyed(call, method, params) {
13958
13976
  for (const row of list$2(await call(method, params), "row list")) {
13959
13977
  const date = day$3(row.Date);
13960
13978
  const key = typeof row.Query === "string" ? row.Query.trim() : "";
13961
- const clicks = count$3(row.Clicks);
13962
- const impressions = count$3(row.Impressions);
13963
- const position = count$3(row.AvgImpressionPosition);
13979
+ const clicks = count$4(row.Clicks);
13980
+ const impressions = count$4(row.Impressions);
13981
+ const position = count$4(row.AvgImpressionPosition);
13964
13982
  if (!date || !key || clicks === null || impressions === null) {
13965
13983
  invalidRows += 1;
13966
13984
  continue;
@@ -14491,6 +14509,14 @@ function sharedReader(read, redirects = /* @__PURE__ */ new Map()) {
14491
14509
  };
14492
14510
  }
14493
14511
  const MAX_STATUS_BYTES = 10485760;
14512
+ async function rankingUpdates() {
14513
+ const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? readJson(response, "Google Search Status", { maxBytes: MAX_STATUS_BYTES }) : null).catch(() => null);
14514
+ if (!Array.isArray(incidents)) return "Google Search Status could not be read.";
14515
+ return searchUpdatesFromIncidents(withRankingHistory(incidents), {
14516
+ product: "Ranking",
14517
+ limit: Number.POSITIVE_INFINITY
14518
+ });
14519
+ }
14494
14520
  async function googleUpdates(window) {
14495
14521
  if (!window) return [];
14496
14522
  const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? readJson(response, "Google Search Status", { maxBytes: MAX_STATUS_BYTES }) : null).catch(() => null);
@@ -19132,17 +19158,15 @@ function recordHostFetch(input) {
19132
19158
  const cfg = input.rate.backpressure;
19133
19159
  health.durations.push(input.durationMs);
19134
19160
  if (health.durations.length > 20) health.durations.shift();
19135
- const blocked = [
19136
- 401,
19137
- 403,
19138
- 429
19139
- ].includes(input.status);
19140
- const errored = input.status >= 500;
19161
+ const rateLimited = input.status === 429 || input.status === 503 && input.retryAfterMs !== void 0;
19162
+ const blocked = [401, 403].includes(input.status);
19163
+ const errored = input.status >= 500 && !rateLimited;
19141
19164
  const slow = input.durationMs >= cfg.slowMs;
19142
19165
  health.consecutiveBlocked = blocked ? health.consecutiveBlocked + 1 : 0;
19143
19166
  health.consecutiveErrors = errored ? health.consecutiveErrors + 1 : 0;
19144
19167
  health.consecutiveSlow = slow ? health.consecutiveSlow + 1 : 0;
19145
- if (!blocked && !errored && !slow) health.reason = void 0;
19168
+ if (!blocked && !errored && !slow && !rateLimited) health.reason = void 0;
19169
+ if (rateLimited) health.reason = `origin rate limited the crawler (HTTP ${input.status})`;
19146
19170
  const retryDelay = input.retryAfterMs === void 0 ? 0 : Math.min(input.retryAfterMs, cfg.retryAfterCapMs);
19147
19171
  if (retryDelay > 0) {
19148
19172
  health.reason = `origin sent Retry-After (${Math.round(retryDelay / 1e3)}s)`;
@@ -19543,6 +19567,56 @@ function systemBrowserCandidates(input) {
19543
19567
  product: path.includes("google-chrome") ? "Google Chrome" : "Chromium"
19544
19568
  }));
19545
19569
  }
19570
+ function listDirectorySafely(path) {
19571
+ try {
19572
+ return readdirSync(path);
19573
+ } catch {
19574
+ return [];
19575
+ }
19576
+ }
19577
+ function versionParts(name) {
19578
+ return (name.match(/\d+/g) ?? []).map(Number);
19579
+ }
19580
+ function newestFirst(left, right) {
19581
+ const a = versionParts(left);
19582
+ const b = versionParts(right);
19583
+ for (let index = 0; index < Math.max(a.length, b.length); index += 1) {
19584
+ const difference = (b[index] ?? -1) - (a[index] ?? -1);
19585
+ if (difference) return difference;
19586
+ }
19587
+ return left < right ? -1 : left > right ? 1 : 0;
19588
+ }
19589
+ const MAX_CACHED_BROWSER_VERSIONS = 20;
19590
+ function versionDirectories(root, prefix, listDirectory) {
19591
+ return listDirectory(root).filter((name) => name.startsWith(prefix)).sort(newestFirst).slice(0, MAX_CACHED_BROWSER_VERSIONS).map((name) => join(root, name));
19592
+ }
19593
+ const CHROME_FOR_TESTING_APP = join("Google Chrome for Testing.app", "Contents", "MacOS", "Google Chrome for Testing");
19594
+ /**
19595
+ * Browsers that other local tools already downloaded: agent-browser keeps
19596
+ * Chrome for Testing builds and Playwright keeps one Chromium per revision.
19597
+ */
19598
+ function cachedBrowserCandidates(input) {
19599
+ const agentBrowserExecutables = input.platform === "darwin" ? [CHROME_FOR_TESTING_APP] : input.platform === "win32" ? ["chrome.exe"] : ["chrome"];
19600
+ const agentBrowser = versionDirectories(join(input.homeDirectory, ".agent-browser", "browsers"), "chrome-", input.listDirectory).flatMap((directory) => agentBrowserExecutables.map((executable) => ({
19601
+ path: join(directory, executable),
19602
+ source: "agent-browser-cache",
19603
+ product: "agent-browser Chrome for Testing"
19604
+ })));
19605
+ const configuredRoot = input.environment.PLAYWRIGHT_BROWSERS_PATH?.trim();
19606
+ const playwrightRoot = configuredRoot && configuredRoot !== "0" ? configuredRoot : input.platform === "darwin" ? join(input.homeDirectory, "Library", "Caches", "ms-playwright") : input.platform === "win32" ? join(input.environment.LOCALAPPDATA ?? join(input.homeDirectory, "AppData", "Local"), "ms-playwright") : join(input.environment.XDG_CACHE_HOME ?? join(input.homeDirectory, ".cache"), "ms-playwright");
19607
+ const playwrightExecutables = input.platform === "darwin" ? [
19608
+ join("chrome-mac-arm64", CHROME_FOR_TESTING_APP),
19609
+ join("chrome-mac-x64", CHROME_FOR_TESTING_APP),
19610
+ join("chrome-mac", CHROME_FOR_TESTING_APP),
19611
+ join("chrome-mac", "Chromium.app", "Contents", "MacOS", "Chromium")
19612
+ ] : input.platform === "win32" ? [join("chrome-win64", "chrome.exe"), join("chrome-win", "chrome.exe")] : [join("chrome-linux64", "chrome"), join("chrome-linux", "chrome")];
19613
+ const playwright = versionDirectories(playwrightRoot, "chromium-", input.listDirectory).flatMap((directory) => playwrightExecutables.map((executable) => ({
19614
+ path: join(directory, executable),
19615
+ source: "playwright-cache",
19616
+ product: "Playwright Chromium"
19617
+ })));
19618
+ return [...agentBrowser, ...playwright];
19619
+ }
19546
19620
  /**
19547
19621
  * Resolve a browser without downloading one during normal package installation.
19548
19622
  * The explicit path wins so CI and managed workstations can select their own
@@ -19574,11 +19648,23 @@ function resolveBrowserExecutable(playwrightExecutablePath, dependencies = {}) {
19574
19648
  product: "Playwright Chromium"
19575
19649
  }
19576
19650
  };
19577
- const browser = systemBrowserCandidates({
19578
- platform: dependencies.platform ?? process.platform,
19579
- homeDirectory: dependencies.homeDirectory ?? homedir(),
19651
+ const platform = dependencies.platform ?? process.platform;
19652
+ const homeDirectory = dependencies.homeDirectory ?? homedir();
19653
+ const systemBrowser = systemBrowserCandidates({
19654
+ platform,
19655
+ homeDirectory,
19580
19656
  environment
19581
19657
  }).find((candidate) => fileExists(candidate.path));
19658
+ if (systemBrowser) return {
19659
+ status: "available",
19660
+ browser: systemBrowser
19661
+ };
19662
+ const browser = cachedBrowserCandidates({
19663
+ platform,
19664
+ homeDirectory,
19665
+ environment,
19666
+ listDirectory: dependencies.listDirectory ?? listDirectorySafely
19667
+ }).find((candidate) => fileExists(candidate.path));
19582
19668
  return browser ? {
19583
19669
  status: "available",
19584
19670
  browser
@@ -22574,6 +22660,17 @@ function isAuditableHtmlPage(page) {
22574
22660
  const contentAuditAllowed = page.contentAuditAllowed ?? (!page.blocked && page.robotsTxt?.allowed !== false);
22575
22661
  return isSuccessfulResponse(page) && isHtmlPage(page) && !isRedirectedPage(page) && contentAuditAllowed;
22576
22662
  }
22663
+ /**
22664
+ * Cloudflare serves its own endpoints under `/cdn-cgi/`, such as the email
22665
+ * obfuscation link `/cdn-cgi/l/email-protection`. They are not site pages.
22666
+ */
22667
+ function isCdnUtilityUrl(value) {
22668
+ try {
22669
+ return new URL(value).pathname.startsWith("/cdn-cgi/");
22670
+ } catch {
22671
+ return false;
22672
+ }
22673
+ }
22577
22674
  // packages/core/dist/analyze/crawler/audit.js
22578
22675
  function sameUrl(a, b) {
22579
22676
  if (!a || !b) return false;
@@ -22686,38 +22783,61 @@ function isValuablePage(page) {
22686
22783
  function normalizedMetadata(value) {
22687
22784
  return value?.trim().replace(/\s+/g, " ").toLowerCase() || void 0;
22688
22785
  }
22689
- function metadataDuplicates(pages, read) {
22690
- const counts = /* @__PURE__ */ new Map();
22786
+ /**
22787
+ * A page whose canonical names another crawled page in its group is already
22788
+ * handled. A group counts only when two or more of its pages are
22789
+ * self-canonical or declare no canonical, and only those pages are reported.
22790
+ */
22791
+ function duplicateGroups(pages, read) {
22792
+ const grouped = /* @__PURE__ */ new Map();
22691
22793
  for (const page of pages) {
22692
22794
  if (!isAuditableHtmlPage(page)) continue;
22693
- const raw = read(page)?.trim();
22694
- const key = normalizedMetadata(raw);
22695
- if (!key || !raw) continue;
22696
- const existing = counts.get(key) ?? {
22697
- count: 0,
22698
- sampleUrls: [],
22699
- value: raw
22795
+ const entry = read(page);
22796
+ if (!entry) continue;
22797
+ const group = grouped.get(entry.key) ?? {
22798
+ value: entry.value,
22799
+ pages: []
22700
22800
  };
22701
- existing.count += 1;
22702
- if (existing.sampleUrls.length < 10) existing.sampleUrls.push(page.url);
22703
- counts.set(key, existing);
22801
+ group.pages.push(page);
22802
+ grouped.set(entry.key, group);
22803
+ }
22804
+ const duplicates = /* @__PURE__ */ new Map();
22805
+ for (const [key, group] of grouped) {
22806
+ if (group.pages.length < 2) continue;
22807
+ const groupUrls = /* @__PURE__ */ new Set();
22808
+ for (const page of group.pages) for (const value of [page.url, page.finalUrl]) {
22809
+ const url = urlKey(value);
22810
+ if (url) groupUrls.add(url);
22811
+ }
22812
+ const counted = group.pages.filter((page) => {
22813
+ const canonical = urlKey(page.canonical);
22814
+ return !canonical || canonical === urlKey(page.url) || canonical === urlKey(page.finalUrl) || !groupUrls.has(canonical);
22815
+ });
22816
+ if (counted.length < 2) continue;
22817
+ duplicates.set(key, {
22818
+ count: counted.length,
22819
+ sampleUrls: counted.slice(0, 10).map((page) => page.url),
22820
+ value: group.value,
22821
+ urls: new Set(counted.map((page) => page.url))
22822
+ });
22704
22823
  }
22705
- return new Map([...counts].filter(([, value]) => value.count > 1));
22824
+ return duplicates;
22825
+ }
22826
+ function metadataDuplicates(pages, read) {
22827
+ return duplicateGroups(pages, (page) => {
22828
+ const value = read(page)?.trim();
22829
+ const key = normalizedMetadata(value);
22830
+ return key && value ? {
22831
+ key,
22832
+ value
22833
+ } : void 0;
22834
+ });
22706
22835
  }
22707
22836
  function contentDuplicates(pages) {
22708
- const counts = /* @__PURE__ */ new Map();
22709
- for (const page of pages) {
22710
- if (!isAuditableHtmlPage(page)) continue;
22711
- if (page.wordCount < 20 || !page.mainContentHash) continue;
22712
- const existing = counts.get(page.mainContentHash) ?? {
22713
- count: 0,
22714
- sampleUrls: []
22715
- };
22716
- existing.count += 1;
22717
- if (existing.sampleUrls.length < 10) existing.sampleUrls.push(page.url);
22718
- counts.set(page.mainContentHash, existing);
22719
- }
22720
- return new Map([...counts].filter(([, value]) => value.count > 1));
22837
+ return duplicateGroups(pages, (page) => page.wordCount >= 20 && page.mainContentHash ? {
22838
+ key: page.mainContentHash,
22839
+ value: page.mainContentHash
22840
+ } : void 0);
22721
22841
  }
22722
22842
  const QUERY_STOP_WORDS = /* @__PURE__ */ new Set([
22723
22843
  "a",
@@ -22820,7 +22940,7 @@ function auditCrawlPages(pages, opts = {}) {
22820
22940
  if (page.accessBlock) continue;
22821
22941
  if (page.status >= 500) {
22822
22942
  issues.push(issue$1("server_error", page, String(page.status), { status: page.status }));
22823
- if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
22943
+ if ((page.internalInlinkCount ?? 0) > 0 && !isCdnUtilityUrl(page.url)) issues.push(issue$1("broken_internal_link", page, String(page.status), {
22824
22944
  status: page.status,
22825
22945
  internalInlinkCount: page.internalInlinkCount
22826
22946
  }));
@@ -22828,7 +22948,7 @@ function auditCrawlPages(pages, opts = {}) {
22828
22948
  }
22829
22949
  if (page.status >= 400) {
22830
22950
  issues.push(issue$1("client_error", page, String(page.status), { status: page.status }));
22831
- if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
22951
+ if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0 && !isCdnUtilityUrl(page.url)) issues.push(issue$1("broken_internal_link", page, String(page.status), {
22832
22952
  status: page.status,
22833
22953
  internalInlinkCount: page.internalInlinkCount
22834
22954
  }));
@@ -22927,7 +23047,7 @@ function auditCrawlPages(pages, opts = {}) {
22927
23047
  title,
22928
23048
  ...titleWidth
22929
23049
  }));
22930
- if (title && duplicateTitle) issues.push(issue$1("title_duplicate", page, duplicateTitle.value, {
23050
+ if (title && duplicateTitle?.urls.has(page.url)) issues.push(issue$1("title_duplicate", page, duplicateTitle.value, {
22931
23051
  title,
22932
23052
  duplicateCount: duplicateTitle.count,
22933
23053
  sampleUrls: duplicateTitle.sampleUrls
@@ -22935,7 +23055,7 @@ function auditCrawlPages(pages, opts = {}) {
22935
23055
  const metaDescription = page.metaDescription?.trim();
22936
23056
  const duplicateDescription = duplicateDescriptions.get(normalizedMetadata(metaDescription) ?? "");
22937
23057
  if (!metaDescription) issues.push(issue$1("missing_meta_description", page));
22938
- if (metaDescription && duplicateDescription) issues.push(issue$1("meta_description_duplicate", page, duplicateDescription.value, {
23058
+ if (metaDescription && duplicateDescription?.urls.has(page.url)) issues.push(issue$1("meta_description_duplicate", page, duplicateDescription.value, {
22939
23059
  metaDescription,
22940
23060
  duplicateCount: duplicateDescription.count,
22941
23061
  sampleUrls: duplicateDescription.sampleUrls
@@ -22983,7 +23103,7 @@ function auditCrawlPages(pages, opts = {}) {
22983
23103
  finalUrl: page.finalUrl
22984
23104
  }));
22985
23105
  const contentDuplicate = page.mainContentHash ? duplicateContent.get(page.mainContentHash) : void 0;
22986
- if (contentDuplicate) issues.push(issue$1("duplicate_content", page, "Duplicate main content", {
23106
+ if (contentDuplicate?.urls.has(page.url)) issues.push(issue$1("duplicate_content", page, "Duplicate main content", {
22987
23107
  mainContentHash: page.mainContentHash,
22988
23108
  duplicateCount: contentDuplicate.count,
22989
23109
  sampleUrls: contentDuplicate.sampleUrls
@@ -23051,6 +23171,7 @@ function auditCrawlPages(pages, opts = {}) {
23051
23171
  // packages/core/dist/analyze/crawler/crawl-skip-reasons.js
23052
23172
  const CRAWL_SKIP_REASONS = [
23053
23173
  "asset-url",
23174
+ "cdn-utility-url",
23054
23175
  "configured-exclusion",
23055
23176
  "duplicate-normalization",
23056
23177
  "invalid-url",
@@ -23058,12 +23179,14 @@ const CRAWL_SKIP_REASONS = [
23058
23179
  "off-origin",
23059
23180
  "origin-backpressure",
23060
23181
  "queue-safety-limit",
23182
+ "query-variant-limit",
23061
23183
  "robots-disallowed",
23062
23184
  "robots-uncertain",
23063
23185
  "legacy-unclassified"
23064
23186
  ];
23065
23187
  const SKIP_IMPACT = {
23066
23188
  "asset-url": "non-impacting",
23189
+ "cdn-utility-url": "non-impacting",
23067
23190
  "configured-exclusion": "non-impacting",
23068
23191
  "duplicate-normalization": "non-impacting",
23069
23192
  "invalid-url": "non-impacting",
@@ -23071,6 +23194,7 @@ const SKIP_IMPACT = {
23071
23194
  "off-origin": "non-impacting",
23072
23195
  "origin-backpressure": "coverage-affecting",
23073
23196
  "queue-safety-limit": "coverage-affecting",
23197
+ "query-variant-limit": "coverage-affecting",
23074
23198
  "robots-disallowed": "coverage-affecting",
23075
23199
  "robots-uncertain": "coverage-affecting",
23076
23200
  "legacy-unclassified": "coverage-affecting"
@@ -23123,157 +23247,85 @@ function crawlSkipReasonCountsFromRecord(input = {}) {
23123
23247
  count
23124
23248
  })) }).skipReasons;
23125
23249
  }
23126
- // packages/core/dist/analyze/crawler/top-fixes.js
23127
- const SEVERITY_SCORE = {
23128
- high: 5e3,
23129
- medium: 1200,
23130
- low: 100
23131
- };
23132
- const EFFORT_BY_CATEGORY = {
23133
- canonical: "medium",
23134
- content: "high",
23135
- agents: "low",
23136
- headings: "low",
23137
- images: "medium",
23138
- indexability: "medium",
23139
- international: "low",
23140
- metadata: "low",
23141
- mobile: "low",
23142
- response: "medium",
23143
- security: "medium",
23144
- social: "low",
23145
- "structured-data": "medium"
23146
- };
23147
- const EFFORT_SCORE = {
23148
- low: 60,
23149
- medium: 30,
23150
- high: 0
23151
- };
23152
- const AFFECTED_URL_WEIGHT = {
23153
- high: 25,
23154
- medium: 15,
23155
- low: 2
23156
- };
23157
- const VALUE_WEIGHT = {
23158
- high: 1,
23159
- medium: .75,
23160
- low: .15
23161
- };
23162
- function matchesPattern(pattern, value) {
23163
- if (!pattern.includes("*")) return value.includes(pattern);
23164
- const parts = pattern.split("*");
23165
- let position = 0;
23166
- for (const part of parts) {
23167
- if (!part) continue;
23168
- const found = value.slice(position).indexOf(part);
23169
- if (found === -1) return false;
23170
- position += found + part.length;
23171
- }
23172
- return true;
23250
+ // packages/core/dist/analyze/crawler/crawl-url-queue.js
23251
+ function before(left, right) {
23252
+ return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
23173
23253
  }
23174
- function filteredIssues(report, filters) {
23175
- return report.issues.filter((issue) => {
23176
- if (filters.category && issue.category !== filters.category) return false;
23177
- if (filters.severity && issue.severity !== filters.severity) return false;
23178
- if (filters.url && !matchesPattern(filters.url, issue.url)) return false;
23179
- if ((filters.recommendation ?? "fix") !== recommendationForRule(issue.ruleId)) return false;
23254
+ var CrawlUrlQueue = class {
23255
+ #items = [];
23256
+ #positions = /* @__PURE__ */ new Map();
23257
+ get size() {
23258
+ return this.#items.length;
23259
+ }
23260
+ has(url) {
23261
+ return this.#positions.has(url);
23262
+ }
23263
+ push(item) {
23264
+ if (this.has(item.url)) throw new Error(`URL is already queued: ${item.url}`);
23265
+ const index = this.#items.length;
23266
+ this.#items.push(item);
23267
+ this.#positions.set(item.url, index);
23268
+ this.#bubbleUp(index);
23269
+ }
23270
+ decreaseDepth(url, depth) {
23271
+ const index = this.#positions.get(url);
23272
+ if (index === void 0) return false;
23273
+ const item = this.#items[index];
23274
+ if (!item || depth >= item.depth) return false;
23275
+ item.depth = depth;
23276
+ this.#bubbleUp(index);
23180
23277
  return true;
23181
- });
23182
- }
23183
- function groupIssues(issues) {
23184
- const groups = /* @__PURE__ */ new Map();
23185
- for (const issue of issues) {
23186
- const existing = groups.get(issue.ruleId) ?? {
23187
- ruleId: issue.ruleId,
23188
- title: issue.title,
23189
- category: issue.category,
23190
- severity: issue.severity,
23191
- count: 0,
23192
- sampleUrls: []
23193
- };
23194
- existing.count += 1;
23195
- if (existing.sampleUrls.length < 10) existing.sampleUrls.push(issue.url);
23196
- groups.set(issue.ruleId, existing);
23197
23278
  }
23198
- return [...groups.values()];
23199
- }
23200
- function searchValueForGroup(report, urls) {
23201
- const wanted = new Set(urls);
23202
- const pages = report.pages.filter((page) => wanted.has(page.url) || wanted.has(page.finalUrl));
23203
- const metrics = pages.map((page) => page.searchMetrics).filter((item) => Boolean(item));
23204
- const clicks = metrics.reduce((sum, item) => sum + item.clicks, 0);
23205
- const impressions = metrics.reduce((sum, item) => sum + item.impressions, 0);
23206
- const sessions = pages.reduce((sum, page) => sum + (page.analytics?.sessions ?? 0), 0);
23207
- const observedMetrics = new Set(report.dataSources?.analytics.observedMetrics ?? [
23208
- "sessions",
23209
- "totalUsers",
23210
- "conversions"
23211
- ]);
23212
- const totalUsers = observedMetrics.has("totalUsers") ? pages.reduce((sum, page) => sum + (page.analytics?.totalUsers ?? 0), 0) : void 0;
23213
- const conversions = observedMetrics.has("conversions") ? pages.reduce((sum, page) => sum + (page.analytics?.conversions ?? 0), 0) : void 0;
23214
- const avgPosition = metrics.length ? metrics.reduce((sum, item) => sum + item.position, 0) / metrics.length : void 0;
23215
- return {
23216
- searchVisibleUrls: metrics.length,
23217
- clicks,
23218
- impressions,
23219
- sessions,
23220
- totalUsers,
23221
- conversions,
23222
- avgPosition
23223
- };
23224
- }
23225
- function whyThisRanks(input, recommendation, searchStatus, analyticsProvider) {
23226
- const visibility = input.searchVisibleUrls ? `${input.searchVisibleUrls} affected URLs have GSC visibility (${input.clicks} clicks, ${input.impressions} impressions).` : searchStatus && !["joined", "none"].includes(searchStatus) ? `No joined GSC visibility is available for these affected URLs; Search Console evidence is ${searchStatus}.` : "No affected URL has joined GSC visibility yet.";
23227
- const analyticsDetails = analyticsProvider ? analyticsProviderDetails(analyticsProvider) : analyticsProviderDetails("google");
23228
- const analyticsLabel = analyticsDetails.label;
23229
- const sessionLabel = analyticsDetails.landingMetricLabel;
23230
- return `${visibility}${input.sessions ? ` ${analyticsLabel} adds ${input.sessions} ${sessionLabel}${input.conversions === void 0 ? "" : ` and ${input.conversions} conversions`} from affected pages.` : ""} ${recommendation === "review" ? "This is a review observation. Confirm it before scheduling implementation work." : `This ranks as a fix. Severity is ${input.severity}; ${input.affectedUrls} URL${input.affectedUrls === 1 ? " is" : "s are"} affected; estimated effort is ${input.effort}.`}`;
23231
- }
23232
- function verificationCommand(report, severity) {
23233
- return `seo crawl ${report.config.url} --severity ${severity} --max-pages ${Math.min(report.config.maxPages, 100)}`;
23234
- }
23235
- function topFixes(report, filters = {}) {
23236
- return groupIssues(filteredIssues(report, filters)).map((group) => {
23237
- const search = searchValueForGroup(report, group.sampleUrls);
23238
- const effort = EFFORT_BY_CATEGORY[group.category] ?? "medium";
23239
- const scoreFactors = {
23240
- severity: SEVERITY_SCORE[group.severity],
23241
- affectedUrls: group.count,
23242
- searchVisibleUrls: search.searchVisibleUrls,
23243
- clicks: search.clicks,
23244
- impressions: search.impressions,
23245
- sessions: search.sessions,
23246
- totalUsers: search.totalUsers,
23247
- conversions: search.conversions,
23248
- avgPosition: search.avgPosition,
23249
- effort,
23250
- effortScore: EFFORT_SCORE[effort]
23251
- };
23252
- const valueWeight = VALUE_WEIGHT[group.severity];
23253
- const score = scoreFactors.severity + scoreFactors.affectedUrls * AFFECTED_URL_WEIGHT[group.severity] + scoreFactors.searchVisibleUrls * 100 * valueWeight + scoreFactors.clicks * 20 * valueWeight + Math.min(scoreFactors.impressions, 1e4) / 25 * valueWeight + scoreFactors.sessions * 2 * valueWeight + (scoreFactors.conversions ?? 0) * 100 * valueWeight + scoreFactors.effortScore;
23254
- const rule = explainRule(group.ruleId);
23255
- const recommendation = recommendationForRule(group.ruleId);
23256
- return {
23257
- ...group,
23258
- recommendation,
23259
- score: Math.round(score),
23260
- scoreFactors,
23261
- whyThisRanks: whyThisRanks(scoreFactors, recommendation, report.dataSources?.searchConsole.status, report.dataSources?.analytics.provider),
23262
- howToFix: rule?.howToFix ?? "",
23263
- howToVerify: rule?.howToVerify ?? "",
23264
- verification: {
23265
- command: verificationCommand(report, group.severity),
23266
- expected: rule?.howToVerify ?? ""
23267
- }
23268
- };
23269
- }).sort((a, b) => b.score - a.score || SEVERITY_SCORE[b.severity] - SEVERITY_SCORE[a.severity] || b.count - a.count || a.ruleId.localeCompare(b.ruleId)).slice(0, filters.limit ?? 10);
23270
- }
23271
- function reviewObservations(report, filters = {}) {
23272
- return topFixes(report, {
23273
- ...filters,
23274
- recommendation: "review"
23275
- });
23276
- }
23279
+ take() {
23280
+ const first = this.#items[0];
23281
+ if (!first) return void 0;
23282
+ const last = this.#items.pop();
23283
+ this.#positions.delete(first.url);
23284
+ if (last && last !== first) {
23285
+ this.#items[0] = last;
23286
+ this.#positions.set(last.url, 0);
23287
+ this.#bubbleDown(0);
23288
+ }
23289
+ return first;
23290
+ }
23291
+ #swap(left, right) {
23292
+ const leftItem = this.#items[left];
23293
+ const rightItem = this.#items[right];
23294
+ if (!leftItem || !rightItem) return;
23295
+ this.#items[left] = rightItem;
23296
+ this.#items[right] = leftItem;
23297
+ this.#positions.set(rightItem.url, left);
23298
+ this.#positions.set(leftItem.url, right);
23299
+ }
23300
+ #bubbleUp(start) {
23301
+ let index = start;
23302
+ while (index > 0) {
23303
+ const parent = Math.floor((index - 1) / 2);
23304
+ const item = this.#items[index];
23305
+ const parentItem = this.#items[parent];
23306
+ if (!item || !parentItem || !before(item, parentItem)) break;
23307
+ this.#swap(index, parent);
23308
+ index = parent;
23309
+ }
23310
+ }
23311
+ #bubbleDown(start) {
23312
+ let index = start;
23313
+ while (true) {
23314
+ const left = index * 2 + 1;
23315
+ const right = left + 1;
23316
+ let next = index;
23317
+ const item = this.#items[next];
23318
+ const leftItem = this.#items[left];
23319
+ const rightItem = this.#items[right];
23320
+ if (leftItem && item && before(leftItem, item)) next = left;
23321
+ const nextItem = this.#items[next];
23322
+ if (rightItem && nextItem && before(rightItem, nextItem)) next = right;
23323
+ if (next === index) return;
23324
+ this.#swap(index, next);
23325
+ index = next;
23326
+ }
23327
+ }
23328
+ };
23277
23329
  // packages/core/dist/analyze/crawler/report.js
23278
23330
  const MAX_CRAWL_PAGES = 1e4;
23279
23331
  const MAX_CRAWL_CONCURRENCY = 16;
@@ -23443,6 +23495,10 @@ function sanitizeMessages(values = []) {
23443
23495
  function compareText$15(left, right) {
23444
23496
  return left < right ? -1 : left > right ? 1 : 0;
23445
23497
  }
23498
+ function compareCaveats(left, right) {
23499
+ const rank = (value) => value.startsWith("Coverage incomplete:") ? 0 : 1;
23500
+ return rank(left) - rank(right) || compareText$15(left, right);
23501
+ }
23446
23502
  function sortPages(pages) {
23447
23503
  return [...pages].sort((left, right) => compareText$15(left.url, right.url));
23448
23504
  }
@@ -23677,9 +23733,229 @@ function createCrawlReport(input) {
23677
23733
  ...input.externalLinkVerification ? { externalLinkVerification: sanitizeTenantValue(input.externalLinkVerification) } : {},
23678
23734
  ...input.siteChecks ? { siteChecks: sanitizeTenantValue(input.siteChecks) } : {},
23679
23735
  warnings: sanitizeMessages(input.warnings).sort(compareText$15),
23680
- caveats: sanitizeMessages(input.caveats).sort(compareText$15)
23736
+ caveats: sanitizeMessages(input.caveats).sort(compareCaveats)
23681
23737
  };
23682
23738
  }
23739
+ // packages/core/dist/analyze/crawler/crawl-caveats.js
23740
+ /** Reports rank a caveat with this prefix first and lead with it. */
23741
+ const CRAWL_COVERAGE_CAVEAT_PREFIX = "Coverage incomplete:";
23742
+ function rateLimitRerunCommand(config, intervalMs) {
23743
+ const scope = config.strategy === "health" ? " --health" : config.mode === "site" ? "" : ` --mode ${config.mode}`;
23744
+ const sitemap = config.sitemapUrl ? ` --sitemap-url ${config.sitemapUrl}` : "";
23745
+ const slowMs = Math.min(8e3, Math.max(2e3, intervalMs * 2));
23746
+ return `seo crawl ${config.url}${scope}${sitemap} --max-pages ${config.maxPages} --concurrency 1 --fetch-interval-cap 1 --fetch-interval-ms ${slowMs}`;
23747
+ }
23748
+ /** Set when the sitemaps list more URLs than the page limit let the crawl read. */
23749
+ function sitemapLimitCaveatInput(config, discovery, defaulted) {
23750
+ if (!discovery || discovery.urlsReturned < config.maxPages || !discovery.roots.some((root) => root.possiblyTruncated)) return;
23751
+ return {
23752
+ maxPages: config.maxPages,
23753
+ defaulted,
23754
+ ceiling: MAX_CRAWL_PAGES,
23755
+ health: config.strategy === "health",
23756
+ url: config.url
23757
+ };
23758
+ }
23759
+ function crawlCaveats(input) {
23760
+ const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
23761
+ if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
23762
+ if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
23763
+ if (input.rateLimit) {
23764
+ const { readPages, skippedUrls, rerunCommand } = input.rateLimit;
23765
+ caveats.push(`${CRAWL_COVERAGE_CAVEAT_PREFIX} read ${readPages} of ${readPages + skippedUrls} pages because the origin kept rate limiting the crawl (HTTP 429) after it backed off. Rerun slower: ${rerunCommand}`);
23766
+ }
23767
+ if (input.sitemapLimit) {
23768
+ const { maxPages, defaulted, ceiling, health, url } = input.sitemapLimit;
23769
+ caveats.push(`Checked only the first ${maxPages} sitemap URLs because the page limit is ${maxPages}${defaulted ? " by default" : ""}, and the sitemaps list more. Raise --max-pages, up to ${ceiling.toLocaleString("en-US")}, to check more: seo crawl ${url}${health ? " --health" : ""} --max-pages ${ceiling}`);
23770
+ }
23771
+ if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
23772
+ const renderingFallbackUrls = input.renderingFallbackUrls ?? 0;
23773
+ if (renderingFallbackUrls > 0) caveats.push(`JavaScript rendering could not run for ${renderingFallbackUrls} page${renderingFallbackUrls === 1 ? "" : "s"}, so ${renderingFallbackUrls === 1 ? "it was" : "they were"} audited from raw HTML. Client-rendered content, metadata and links may be missing. Install Chromium or set SEO_BROWSER_EXECUTABLE_PATH, then rerun.`);
23774
+ const queryVariantSkippedUrls = input.queryVariantSkippedUrls ?? 0;
23775
+ if (queryVariantSkippedUrls > 0) caveats.push(`Left ${queryVariantSkippedUrls} query-string variant${queryVariantSkippedUrls === 1 ? "" : "s"} unqueued after 5 variants of the same path, so tracking parameters do not fill the crawl.`);
23776
+ for (const issue of input.reachIssues ?? []) if (issue.detail) caveats.push(issue.detail);
23777
+ return caveats;
23778
+ }
23779
+ /**
23780
+ * A site crawl that found no internal links on the start page and no sitemap
23781
+ * URLs checked one page, so its other findings cannot stand for the site.
23782
+ */
23783
+ function crawlReachIssues(input) {
23784
+ const [page] = input.pages;
23785
+ if (input.crawledPages !== 1 || input.pages.length !== 1 || !page || page.outgoingInternalCount > 0 || input.sitemapUrlsReturned > 0) return [];
23786
+ const rule = explainRule("crawl_single_page_reach");
23787
+ if (!rule) throw new Error("Missing rule guidance for crawl_single_page_reach");
23788
+ return [{
23789
+ ruleId: "crawl_single_page_reach",
23790
+ title: rule.title,
23791
+ category: rule.category,
23792
+ severity: rule.defaultSeverity,
23793
+ url: page.url,
23794
+ detail: page.usedJs ? "The rendered start page had 0 internal links and no sitemap returned URLs, so the crawl checked only this page. List the pages to check with --mode list." : "The start page had 0 internal links in its HTML and no sitemap returned URLs, so the crawl checked only this page. It probably builds its links with JavaScript. Rerun with --rendering on, or list the pages with --mode list.",
23795
+ evidence: {
23796
+ startUrl: input.startUrl,
23797
+ crawledPages: 1,
23798
+ outgoingInternalCount: 0,
23799
+ sitemapUrlsReturned: 0,
23800
+ rendered: Boolean(page.usedJs)
23801
+ }
23802
+ }];
23803
+ }
23804
+ // packages/core/dist/analyze/crawler/top-fixes.js
23805
+ const SEVERITY_SCORE = {
23806
+ high: 5e3,
23807
+ medium: 1200,
23808
+ low: 100
23809
+ };
23810
+ const EFFORT_BY_CATEGORY = {
23811
+ canonical: "medium",
23812
+ content: "high",
23813
+ agents: "low",
23814
+ headings: "low",
23815
+ images: "medium",
23816
+ indexability: "medium",
23817
+ international: "low",
23818
+ metadata: "low",
23819
+ mobile: "low",
23820
+ response: "medium",
23821
+ security: "medium",
23822
+ social: "low",
23823
+ "structured-data": "medium"
23824
+ };
23825
+ const EFFORT_SCORE = {
23826
+ low: 60,
23827
+ medium: 30,
23828
+ high: 0
23829
+ };
23830
+ const AFFECTED_URL_WEIGHT = {
23831
+ high: 25,
23832
+ medium: 15,
23833
+ low: 2
23834
+ };
23835
+ const VALUE_WEIGHT = {
23836
+ high: 1,
23837
+ medium: .75,
23838
+ low: .15
23839
+ };
23840
+ const CRAWL_REACH_RULES = /* @__PURE__ */ new Set(["crawl_single_page_reach"]);
23841
+ function reachRank(ruleId) {
23842
+ return CRAWL_REACH_RULES.has(ruleId) ? 0 : 1;
23843
+ }
23844
+ function matchesPattern(pattern, value) {
23845
+ if (!pattern.includes("*")) return value.includes(pattern);
23846
+ const parts = pattern.split("*");
23847
+ let position = 0;
23848
+ for (const part of parts) {
23849
+ if (!part) continue;
23850
+ const found = value.slice(position).indexOf(part);
23851
+ if (found === -1) return false;
23852
+ position += found + part.length;
23853
+ }
23854
+ return true;
23855
+ }
23856
+ function filteredIssues(report, filters) {
23857
+ return report.issues.filter((issue) => {
23858
+ if (filters.category && issue.category !== filters.category) return false;
23859
+ if (filters.severity && issue.severity !== filters.severity) return false;
23860
+ if (filters.url && !matchesPattern(filters.url, issue.url)) return false;
23861
+ if ((filters.recommendation ?? "fix") !== recommendationForRule(issue.ruleId)) return false;
23862
+ return true;
23863
+ });
23864
+ }
23865
+ function groupIssues(issues) {
23866
+ const groups = /* @__PURE__ */ new Map();
23867
+ for (const issue of issues) {
23868
+ const existing = groups.get(issue.ruleId) ?? {
23869
+ ruleId: issue.ruleId,
23870
+ title: issue.title,
23871
+ category: issue.category,
23872
+ severity: issue.severity,
23873
+ count: 0,
23874
+ sampleUrls: []
23875
+ };
23876
+ existing.count += 1;
23877
+ if (existing.sampleUrls.length < 10) existing.sampleUrls.push(issue.url);
23878
+ groups.set(issue.ruleId, existing);
23879
+ }
23880
+ return [...groups.values()];
23881
+ }
23882
+ function searchValueForGroup(report, urls) {
23883
+ const wanted = new Set(urls);
23884
+ const pages = report.pages.filter((page) => wanted.has(page.url) || wanted.has(page.finalUrl));
23885
+ const metrics = pages.map((page) => page.searchMetrics).filter((item) => Boolean(item));
23886
+ const clicks = metrics.reduce((sum, item) => sum + item.clicks, 0);
23887
+ const impressions = metrics.reduce((sum, item) => sum + item.impressions, 0);
23888
+ const sessions = pages.reduce((sum, page) => sum + (page.analytics?.sessions ?? 0), 0);
23889
+ const observedMetrics = new Set(report.dataSources?.analytics.observedMetrics ?? [
23890
+ "sessions",
23891
+ "totalUsers",
23892
+ "conversions"
23893
+ ]);
23894
+ const totalUsers = observedMetrics.has("totalUsers") ? pages.reduce((sum, page) => sum + (page.analytics?.totalUsers ?? 0), 0) : void 0;
23895
+ const conversions = observedMetrics.has("conversions") ? pages.reduce((sum, page) => sum + (page.analytics?.conversions ?? 0), 0) : void 0;
23896
+ const avgPosition = metrics.length ? metrics.reduce((sum, item) => sum + item.position, 0) / metrics.length : void 0;
23897
+ return {
23898
+ searchVisibleUrls: metrics.length,
23899
+ clicks,
23900
+ impressions,
23901
+ sessions,
23902
+ totalUsers,
23903
+ conversions,
23904
+ avgPosition
23905
+ };
23906
+ }
23907
+ function whyThisRanks(input, recommendation, searchStatus, analyticsProvider) {
23908
+ const visibility = input.searchVisibleUrls ? `${input.searchVisibleUrls} affected URLs have GSC visibility (${input.clicks} clicks, ${input.impressions} impressions).` : searchStatus && !["joined", "none"].includes(searchStatus) ? `No joined GSC visibility is available for these affected URLs; Search Console evidence is ${searchStatus}.` : "No affected URL has joined GSC visibility yet.";
23909
+ const analyticsDetails = analyticsProvider ? analyticsProviderDetails(analyticsProvider) : analyticsProviderDetails("google");
23910
+ const analyticsLabel = analyticsDetails.label;
23911
+ const sessionLabel = analyticsDetails.landingMetricLabel;
23912
+ return `${visibility}${input.sessions ? ` ${analyticsLabel} adds ${input.sessions} ${sessionLabel}${input.conversions === void 0 ? "" : ` and ${input.conversions} conversions`} from affected pages.` : ""} ${recommendation === "review" ? "This is a review observation. Confirm it before scheduling implementation work." : `This ranks as a fix. Severity is ${input.severity}; ${input.affectedUrls} URL${input.affectedUrls === 1 ? " is" : "s are"} affected; estimated effort is ${input.effort}.`}`;
23913
+ }
23914
+ function verificationCommand(report, severity) {
23915
+ return `seo crawl ${report.config.url} --severity ${severity} --max-pages ${Math.min(report.config.maxPages, 100)}`;
23916
+ }
23917
+ function topFixes(report, filters = {}) {
23918
+ return groupIssues(filteredIssues(report, filters)).map((group) => {
23919
+ const search = searchValueForGroup(report, group.sampleUrls);
23920
+ const effort = EFFORT_BY_CATEGORY[group.category] ?? "medium";
23921
+ const scoreFactors = {
23922
+ severity: SEVERITY_SCORE[group.severity],
23923
+ affectedUrls: group.count,
23924
+ searchVisibleUrls: search.searchVisibleUrls,
23925
+ clicks: search.clicks,
23926
+ impressions: search.impressions,
23927
+ sessions: search.sessions,
23928
+ totalUsers: search.totalUsers,
23929
+ conversions: search.conversions,
23930
+ avgPosition: search.avgPosition,
23931
+ effort,
23932
+ effortScore: EFFORT_SCORE[effort]
23933
+ };
23934
+ const valueWeight = VALUE_WEIGHT[group.severity];
23935
+ const score = scoreFactors.severity + scoreFactors.affectedUrls * AFFECTED_URL_WEIGHT[group.severity] + scoreFactors.searchVisibleUrls * 100 * valueWeight + scoreFactors.clicks * 20 * valueWeight + Math.min(scoreFactors.impressions, 1e4) / 25 * valueWeight + scoreFactors.sessions * 2 * valueWeight + (scoreFactors.conversions ?? 0) * 100 * valueWeight + scoreFactors.effortScore;
23936
+ const rule = explainRule(group.ruleId);
23937
+ const recommendation = recommendationForRule(group.ruleId);
23938
+ return {
23939
+ ...group,
23940
+ recommendation,
23941
+ score: Math.round(score),
23942
+ scoreFactors,
23943
+ whyThisRanks: whyThisRanks(scoreFactors, recommendation, report.dataSources?.searchConsole.status, report.dataSources?.analytics.provider),
23944
+ howToFix: rule?.howToFix ?? "",
23945
+ howToVerify: rule?.howToVerify ?? "",
23946
+ verification: {
23947
+ command: verificationCommand(report, group.severity),
23948
+ expected: rule?.howToVerify ?? ""
23949
+ }
23950
+ };
23951
+ }).sort((a, b) => reachRank(a.ruleId) - reachRank(b.ruleId) || b.score - a.score || SEVERITY_SCORE[b.severity] - SEVERITY_SCORE[a.severity] || b.count - a.count || a.ruleId.localeCompare(b.ruleId)).slice(0, filters.limit ?? 10);
23952
+ }
23953
+ function reviewObservations(report, filters = {}) {
23954
+ return topFixes(report, {
23955
+ ...filters,
23956
+ recommendation: "review"
23957
+ });
23958
+ }
23683
23959
  // packages/core/dist/analyze/crawler/site-checks-schema.js
23684
23960
  const crawlSiteChecksSchema = z.object({ soft404: z.object({
23685
23961
  status: z.enum([
@@ -24106,6 +24382,7 @@ const crawlPageSnapshotSchema = z.object({
24106
24382
  source: z.enum([
24107
24383
  "environment",
24108
24384
  "playwright-cache",
24385
+ "agent-browser-cache",
24109
24386
  "system"
24110
24387
  ]),
24111
24388
  product: z.string(),
@@ -24794,14 +25071,6 @@ function agentReadyIssues(input) {
24794
25071
  }
24795
25072
  return issues;
24796
25073
  }
24797
- // packages/core/dist/analyze/crawler/crawl-caveats.js
24798
- function crawlCaveats(input) {
24799
- const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
24800
- if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
24801
- if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
24802
- if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
24803
- return caveats;
24804
- }
24805
25074
  // packages/core/dist/analyze/crawler/crawl-control.js
24806
25075
  const CRAWL_CANCELLED = Symbol("crawl-cancelled");
24807
25076
  function abortController(input) {
@@ -24843,83 +25112,94 @@ function crawlRssLimitBytes(totalMemoryBytes) {
24843
25112
  function crawlMemoryPressure(input) {
24844
25113
  return input.memoryUsage.heapUsed + input.memoryUsage.external >= crawlMemoryLimitBytes(input.totalMemoryBytes) || input.memoryUsage.rss >= crawlRssLimitBytes(input.totalMemoryBytes);
24845
25114
  }
24846
- // packages/core/dist/analyze/crawler/crawl-url-queue.js
24847
- function before(left, right) {
24848
- return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
25115
+ // packages/core/dist/analyze/crawler/crawl-rate-limit.js
25116
+ const RETRIES_PER_URL = 3;
25117
+ const BASE_BACKOFF_MS = 2e3;
25118
+ const MAX_BACKOFF_MS = 3e4;
25119
+ const COOLDOWN_BUDGET_MS = 18e4;
25120
+ const LIMITED_INTERVAL_MS = 1e3;
25121
+ const LIMITED_MAX_INTERVAL_MS = 4e3;
25122
+ function rateLimitEvidence(result, status, nowMs) {
25123
+ if (status !== 429 && status !== 503) return void 0;
25124
+ const header = retryAfterMs(result.page?.responseHeaders?.["retry-after"]);
25125
+ const cooldownUntil = Date.parse(result.page?.fetchDiagnostics?.backpressure?.cooldownUntil ?? "");
25126
+ const fromCooldown = Number.isFinite(cooldownUntil) ? Math.max(0, cooldownUntil - nowMs) : void 0;
25127
+ const wait = header ?? fromCooldown;
25128
+ if (status === 503 && wait === void 0) return void 0;
25129
+ return wait === void 0 ? {} : { retryAfterMs: Math.min(wait, MAX_BACKOFF_MS) };
24849
25130
  }
24850
- var CrawlUrlQueue = class {
24851
- #items = [];
24852
- #positions = /* @__PURE__ */ new Map();
24853
- get size() {
24854
- return this.#items.length;
24855
- }
24856
- has(url) {
24857
- return this.#positions.has(url);
24858
- }
24859
- push(item) {
24860
- if (this.has(item.url)) throw new Error(`URL is already queued: ${item.url}`);
24861
- const index = this.#items.length;
24862
- this.#items.push(item);
24863
- this.#positions.set(item.url, index);
24864
- this.#bubbleUp(index);
24865
- }
24866
- decreaseDepth(url, depth) {
24867
- const index = this.#positions.get(url);
24868
- if (index === void 0) return false;
24869
- const item = this.#items[index];
24870
- if (!item || depth >= item.depth) return false;
24871
- item.depth = depth;
24872
- this.#bubbleUp(index);
24873
- return true;
24874
- }
24875
- take() {
24876
- const first = this.#items[0];
24877
- if (!first) return void 0;
24878
- const last = this.#items.pop();
24879
- this.#positions.delete(first.url);
24880
- if (last && last !== first) {
24881
- this.#items[0] = last;
24882
- this.#positions.set(last.url, 0);
24883
- this.#bubbleDown(0);
24884
- }
24885
- return first;
24886
- }
24887
- #swap(left, right) {
24888
- const leftItem = this.#items[left];
24889
- const rightItem = this.#items[right];
24890
- if (!leftItem || !rightItem) return;
24891
- this.#items[left] = rightItem;
24892
- this.#items[right] = leftItem;
24893
- this.#positions.set(rightItem.url, left);
24894
- this.#positions.set(leftItem.url, right);
25131
+ /**
25132
+ * Rate-limit state for one crawl. After the first 429 the crawl drops to one
25133
+ * request at a time with a per-host interval and never speeds back up.
25134
+ */
25135
+ var CrawlRateLimit = class {
25136
+ rate;
25137
+ responses = 0;
25138
+ skippedUrls = 0;
25139
+ pendingMs = 0;
25140
+ spentMs = 0;
25141
+ #level = 0;
25142
+ #budgetSpent = false;
25143
+ #retries = /* @__PURE__ */ new Map();
25144
+ constructor(rate) {
25145
+ this.rate = rate;
25146
+ }
25147
+ get limited() {
25148
+ return this.responses > 0;
25149
+ }
25150
+ get intervalMs() {
25151
+ return this.rate.intervalMs ?? LIMITED_INTERVAL_MS;
25152
+ }
25153
+ /** The cool-down before probes that run after the crawl. */
25154
+ get settleMs() {
25155
+ return Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** this.#level);
24895
25156
  }
24896
- #bubbleUp(start) {
24897
- let index = start;
24898
- while (index > 0) {
24899
- const parent = Math.floor((index - 1) / 2);
24900
- const item = this.#items[index];
24901
- const parentItem = this.#items[parent];
24902
- if (!item || !parentItem || !before(item, parentItem)) break;
24903
- this.#swap(index, parent);
24904
- index = parent;
24905
- }
25157
+ /**
25158
+ * `retry` requeues the URL after `pendingMs`. `skip` gives it up, and
25159
+ * `stop` gives up the rest of the queue too once the budget is spent.
25160
+ */
25161
+ observe(url, result, status, nowMs) {
25162
+ const evidence = rateLimitEvidence(result, status, nowMs);
25163
+ if (!evidence) return void 0;
25164
+ this.responses += 1;
25165
+ if (this.pendingMs === 0) this.#level += 1;
25166
+ const step = 2 ** Math.max(0, this.#level - 1);
25167
+ const waitMs = evidence.retryAfterMs ?? Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * step);
25168
+ this.rate = {
25169
+ ...this.rate,
25170
+ concurrency: 1,
25171
+ intervalCap: 1,
25172
+ intervalMs: Math.min(LIMITED_MAX_INTERVAL_MS, Math.max(this.rate.intervalMs ?? 0, LIMITED_INTERVAL_MS * step))
25173
+ };
25174
+ const nextMs = Math.max(this.pendingMs, waitMs);
25175
+ if (this.spentMs + nextMs > COOLDOWN_BUDGET_MS) this.#budgetSpent = true;
25176
+ const retries = this.#retries.get(url) ?? 0;
25177
+ if (!this.#budgetSpent && retries < RETRIES_PER_URL) {
25178
+ this.#retries.set(url, retries + 1);
25179
+ this.pendingMs = nextMs;
25180
+ return "retry";
25181
+ }
25182
+ this.skippedUrls += 1;
25183
+ return this.#budgetSpent ? "stop" : "skip";
25184
+ }
25185
+ async settle(sleep, signal) {
25186
+ if (this.pendingMs <= 0) return;
25187
+ await sleep(this.pendingMs, signal);
25188
+ this.spentMs += this.pendingMs;
25189
+ this.pendingMs = 0;
25190
+ }
25191
+ caveatInput(config, readPages) {
25192
+ if (this.skippedUrls === 0) return void 0;
25193
+ return {
25194
+ readPages,
25195
+ skippedUrls: this.skippedUrls,
25196
+ rerunCommand: rateLimitRerunCommand(config, this.intervalMs)
25197
+ };
24906
25198
  }
24907
- #bubbleDown(start) {
24908
- let index = start;
24909
- while (true) {
24910
- const left = index * 2 + 1;
24911
- const right = left + 1;
24912
- let next = index;
24913
- const item = this.#items[next];
24914
- const leftItem = this.#items[left];
24915
- const rightItem = this.#items[right];
24916
- if (leftItem && item && before(leftItem, item)) next = left;
24917
- const nextItem = this.#items[next];
24918
- if (rightItem && nextItem && before(rightItem, nextItem)) next = right;
24919
- if (next === index) return;
24920
- this.#swap(index, next);
24921
- index = next;
24922
- }
25199
+ warning() {
25200
+ if (!this.limited) return void 0;
25201
+ const plural = (count, noun) => `${count} ${noun}${count === 1 ? "" : "s"}`;
25202
+ return `The origin rate limited ${plural(this.responses, "request")}. The crawl slowed to one request at a time, at least ${Math.round(this.intervalMs / 1e3)}s apart, and waited ${Math.round(this.spentMs / 1e3)}s in cool-downs${this.skippedUrls > 0 ? `, then left ${plural(this.skippedUrls, "URL")} unread` : " before reading every queued URL"}.`;
24923
25203
  }
24924
25204
  };
24925
25205
  // packages/core/dist/analyze/crawler/external-link-checks.js
@@ -25154,11 +25434,15 @@ function observationFromPage(requestedUrl, page) {
25154
25434
  };
25155
25435
  }
25156
25436
  // packages/core/dist/analyze/crawler/site-checks.js
25157
- const PROBE_COUNT = 2;
25437
+ const PROBE_COUNT = 3;
25158
25438
  function probeUrls(startUrl) {
25159
25439
  const origin = new URL(startUrl).origin;
25160
25440
  const seed = createHash("sha256").update(origin).digest("hex").slice(0, 12);
25161
- return Array.from({ length: PROBE_COUNT }, (_, index) => new URL(`/.well-known/seo-audit/not-found-${seed}-${index + 1}`, origin).toString());
25441
+ return [
25442
+ `/${seed}-not-found`,
25443
+ `/${seed}-not-found/${seed}-missing`,
25444
+ `/.well-known/seo-audit/not-found-${seed}-1`
25445
+ ].map((path) => new URL(path, origin).toString());
25162
25446
  }
25163
25447
  function probeObservation(url, result) {
25164
25448
  const request = result.request;
@@ -25178,29 +25462,40 @@ function probeObservation(url, result) {
25178
25462
  error: request.outcome === "failure" ? request.error : `Probe skipped: ${request.reason}`
25179
25463
  };
25180
25464
  }
25465
+ function isConclusive(probe) {
25466
+ return probe.status !== void 0 && !probe.accessBlocked && probe.status !== 429 && probe.status !== 503;
25467
+ }
25181
25468
  async function collectCrawlSiteChecks(input) {
25469
+ const probe = async (url) => probeObservation(url, await input.fetchStatusPage(url, {
25470
+ js: "off",
25471
+ refresh: true,
25472
+ timeoutMs: input.timeoutMs,
25473
+ signal: input.signal,
25474
+ writeCache: false,
25475
+ respectRobots: input.respectRobots,
25476
+ ...input.rate ? { rate: input.rate } : {}
25477
+ }));
25182
25478
  const probes = [];
25183
25479
  for (const url of probeUrls(input.startUrl)) {
25184
25480
  if (input.signal?.aborted) break;
25185
- const result = await input.fetchStatusPage(url, {
25186
- js: "off",
25187
- refresh: true,
25188
- timeoutMs: input.timeoutMs,
25189
- signal: input.signal,
25190
- writeCache: false,
25191
- respectRobots: input.respectRobots
25192
- });
25193
- probes.push(probeObservation(url, result));
25481
+ probes.push(await probe(url));
25482
+ }
25483
+ if (input.sleep && !input.signal?.aborted && probes.some((item) => !isConclusive(item))) {
25484
+ await input.sleep(input.retryDelayMs ?? 2e3, input.signal);
25485
+ for (const [index, item] of probes.entries()) {
25486
+ if (input.signal?.aborted) break;
25487
+ if (!isConclusive(item)) probes[index] = await probe(item.url);
25488
+ }
25194
25489
  }
25195
- const conclusive = probes.filter((probe) => probe.status !== void 0 && !probe.accessBlocked);
25490
+ const conclusive = probes.filter(isConclusive);
25196
25491
  const successful = conclusive.filter((probe) => (probe.status ?? 0) >= 200 && (probe.status ?? 0) < 400);
25197
25492
  const correctMissing = conclusive.filter((probe) => probe.status === 404 || probe.status === 410);
25198
- const status = conclusive.length !== PROBE_COUNT ? "unknown" : successful.length === PROBE_COUNT ? "warning" : correctMissing.length === PROBE_COUNT ? "pass" : "unknown";
25493
+ const status = successful.length > 0 ? "warning" : correctMissing.length === PROBE_COUNT ? "pass" : "unknown";
25199
25494
  const checks = { soft404: {
25200
25495
  status,
25201
25496
  probes,
25202
25497
  probeLimit: PROBE_COUNT,
25203
- complete: probes.length === PROBE_COUNT
25498
+ complete: conclusive.length === PROBE_COUNT
25204
25499
  } };
25205
25500
  if (status !== "warning") return {
25206
25501
  checks,
@@ -25216,7 +25511,7 @@ async function collectCrawlSiteChecks(input) {
25216
25511
  category: rule.category,
25217
25512
  severity: rule.defaultSeverity,
25218
25513
  url: new URL("/", input.startUrl).toString(),
25219
- detail: `${successful.length} known-nonexistent paths returned a successful response or redirect.`,
25514
+ detail: `${successful.length} of ${probes.length} made-up paths returned a successful response or redirect: ${successful.map((probe) => `${new URL(probe.url).pathname} (${probe.status})`).join(", ")}.`,
25220
25515
  evidence: checks.soft404
25221
25516
  }]
25222
25517
  };
@@ -27648,6 +27943,18 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
27648
27943
  }[connection.provider](connection);
27649
27944
  }
27650
27945
  // packages/core/dist/analyze/crawler/site-crawl-dependencies.js
27946
+ function sleep$2(ms, signal) {
27947
+ return new Promise((resolve) => {
27948
+ if (signal?.aborted) return resolve();
27949
+ const done = () => {
27950
+ clearTimeout(timer);
27951
+ signal?.removeEventListener("abort", done);
27952
+ resolve();
27953
+ };
27954
+ const timer = setTimeout(done, ms);
27955
+ signal?.addEventListener("abort", done, { once: true });
27956
+ });
27957
+ }
27651
27958
  function resolveCrawlSiteDependencies(dependencies = {}) {
27652
27959
  const hasInjectedPerPageSearchProvider = Boolean(dependencies.queryPageMetrics || dependencies.queryPageTopQuery);
27653
27960
  const hasInjectedLegacyBulkSearchProvider = Boolean(dependencies.queryPagesMetrics || dependencies.queryPagesTopQueries);
@@ -27668,6 +27975,7 @@ function resolveCrawlSiteDependencies(dependencies = {}) {
27668
27975
  fetchLandingPageValues: dependencies.fetchLandingPageValues ?? fetchLandingPageValues,
27669
27976
  landingValueForUrl: dependencies.landingValueForUrl ?? landingValueForUrl,
27670
27977
  now: dependencies.now ?? (() => /* @__PURE__ */ new Date()),
27978
+ sleep: dependencies.sleep ?? sleep$2,
27671
27979
  memoryUsage: dependencies.memoryUsage ?? (() => process.memoryUsage()),
27672
27980
  totalMemory: dependencies.totalMemory ?? totalmem
27673
27981
  };
@@ -27883,6 +28191,10 @@ async function crawlSite(input, dependencies) {
27883
28191
  const queueSafetyDeferredUrls = /* @__PURE__ */ new Set();
27884
28192
  const maxTrackedQueueSafetyUrls = Math.min(5e4, Math.max(1e3, config.maxPages * 5));
27885
28193
  let queueSafetyInventoryCapped = false;
28194
+ const queryVariantsByPath = /* @__PURE__ */ new Map();
28195
+ const queryVariantSkipped = /* @__PURE__ */ new Set();
28196
+ let renderingFallbackUrls = 0;
28197
+ const rateLimit = new CrawlRateLimit(config.fetchRate);
27886
28198
  let failedUrls = 0;
27887
28199
  let observedInternalLinks = 0;
27888
28200
  let memoryLimited = false;
@@ -27967,6 +28279,15 @@ async function crawlSite(input, dependencies) {
27967
28279
  });
27968
28280
  return;
27969
28281
  }
28282
+ if (isCdnUtilityUrl(normalized)) {
28283
+ recordSkip("cdn-utility-url");
28284
+ emitStatus("url_skipped", {
28285
+ url: normalized,
28286
+ depth,
28287
+ reason: "cdn_utility_url"
28288
+ });
28289
+ return;
28290
+ }
27970
28291
  if (isLikelyAsset(normalized)) {
27971
28292
  recordSkip("asset-url");
27972
28293
  emitStatus("url_skipped", {
@@ -27985,6 +28306,20 @@ async function crawlSite(input, dependencies) {
27985
28306
  });
27986
28307
  return;
27987
28308
  }
28309
+ const parsed = new URL(normalized);
28310
+ const pathKey = parsed.search ? `${parsed.origin}${parsed.pathname}` : "";
28311
+ if (pathKey && (queryVariantsByPath.get(pathKey) ?? 0) >= 5) {
28312
+ if (!queryVariantSkipped.has(normalized) && queryVariantSkipped.size < maxTrackedQueueSafetyUrls) {
28313
+ queryVariantSkipped.add(normalized);
28314
+ recordSkip("query-variant-limit");
28315
+ emitStatus("url_skipped", {
28316
+ url: normalized,
28317
+ depth,
28318
+ reason: "query_variant_limit"
28319
+ });
28320
+ }
28321
+ return;
28322
+ }
27988
28323
  if (queue.size + visited.size + inFlight.size >= config.maxPages * 5) {
27989
28324
  if (!queueSafetyDeferredUrls.has(normalized)) {
27990
28325
  if (queueSafetyDeferredUrls.size < maxTrackedQueueSafetyUrls) {
@@ -28000,6 +28335,7 @@ async function crawlSite(input, dependencies) {
28000
28335
  }
28001
28336
  return;
28002
28337
  }
28338
+ if (pathKey) queryVariantsByPath.set(pathKey, (queryVariantsByPath.get(pathKey) ?? 0) + 1);
28003
28339
  discovered.add(normalized);
28004
28340
  queuedUrls += 1;
28005
28341
  queue.push({
@@ -28056,7 +28392,7 @@ async function crawlSite(input, dependencies) {
28056
28392
  js: config.js,
28057
28393
  refresh: config.refresh,
28058
28394
  timeoutMs: config.timeoutMs,
28059
- rate: config.fetchRate,
28395
+ rate: rateLimit.rate,
28060
28396
  signal,
28061
28397
  writeCache: false,
28062
28398
  respectRobots: config.respectRobots,
@@ -28087,7 +28423,11 @@ async function crawlSite(input, dependencies) {
28087
28423
  });
28088
28424
  };
28089
28425
  while (!isCancelled() && (queue.size || inFlight.size) && (!memoryLimited || inFlight.size > 0) && pages.length < config.maxPages) {
28090
- while (!isCancelled() && !memoryLimited && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
28426
+ if (rateLimit.pendingMs > 0 && inFlight.size === 0) {
28427
+ await rateLimit.settle(deps.sleep, signal);
28428
+ if (isCancelled()) break;
28429
+ }
28430
+ while (!isCancelled() && !memoryLimited && rateLimit.pendingMs === 0 && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
28091
28431
  const item = nextQueueItem();
28092
28432
  if (!item) break;
28093
28433
  startTask(item);
@@ -28107,11 +28447,46 @@ async function crawlSite(input, dependencies) {
28107
28447
  error: result.warning ?? "Crawler returned no page snapshot.",
28108
28448
  extraction: "not-applicable"
28109
28449
  });
28450
+ const limited = rateLimit.observe(task.url, result, request.outcome === "response" ? request.status : void 0, deps.now().getTime());
28451
+ if (limited) {
28452
+ activeConcurrency = 1;
28453
+ healthyProbeStreak = 0;
28454
+ }
28455
+ if (limited === "retry") {
28456
+ visited.delete(task.url);
28457
+ if (!queue.has(task.url)) queue.push({
28458
+ url: task.url,
28459
+ depth: task.depth
28460
+ });
28461
+ emitStatus("url_queued", {
28462
+ url: task.url,
28463
+ depth: task.depth,
28464
+ reason: "rate_limited_retry",
28465
+ message: `The origin rate limited ${task.url}. Waiting ${Math.ceil(rateLimit.pendingMs / 1e3)}s, then retrying one request at a time.`
28466
+ });
28467
+ continue;
28468
+ }
28469
+ if (limited) {
28470
+ requests.push(request);
28471
+ recordSkip("origin-backpressure");
28472
+ emitStatus("page_skipped", {
28473
+ url: task.url,
28474
+ depth: task.depth,
28475
+ reason: "origin-backpressure"
28476
+ });
28477
+ if (limited === "stop") {
28478
+ let left = 0;
28479
+ while (queue.take()) left += 1;
28480
+ recordSkips("origin-backpressure", left);
28481
+ rateLimit.skippedUrls += left;
28482
+ }
28483
+ continue;
28484
+ }
28110
28485
  requests.push(request);
28111
28486
  if (healthStrategy) {
28112
28487
  if (request.outcome === "response" && request.status < 400 && !request.accessBlock) {
28113
28488
  healthyProbeStreak += 1;
28114
- activeConcurrency = Math.min(config.concurrency, 1 + Math.floor(healthyProbeStreak / 20));
28489
+ activeConcurrency = Math.min(rateLimit.limited ? 1 : config.concurrency, 1 + Math.floor(healthyProbeStreak / 20));
28115
28490
  } else {
28116
28491
  healthyProbeStreak = 0;
28117
28492
  activeConcurrency = 1;
@@ -28152,6 +28527,9 @@ async function crawlSite(input, dependencies) {
28152
28527
  continue;
28153
28528
  }
28154
28529
  const page = result.page;
28530
+ const rendering = page.fetchDiagnostics?.rendering;
28531
+ if (rendering?.status === "unavailable" && config.js === "on") throw new SeoError("INVALID_INPUT", `--rendering on needs a local Chrome or Chromium browser, and none could start. ${rendering.error ?? ""} Install Chromium, or set SEO_BROWSER_EXECUTABLE_PATH to a Chromium executable, then rerun. Use --rendering auto to accept raw HTML.`.replace(/\s+/g, " ").trim());
28532
+ if (rendering?.status === "unavailable" || rendering?.status === "failed") renderingFallbackUrls += 1;
28155
28533
  const finalUrl = normalizeUrl$1(page.finalUrl, config.url);
28156
28534
  if (!finalUrl) {
28157
28535
  failedUrls += 1;
@@ -28181,7 +28559,7 @@ async function crawlSite(input, dependencies) {
28181
28559
  const storeDocument = (urls) => {
28182
28560
  const sourceRequest = normalizeUrl$1(task.url, config.url) ?? task.url;
28183
28561
  const directlyRequested = sourceRequest === finalUrl;
28184
- const normalizedLinks = urls.map((url) => normalizeUrl$1(url, finalUrl)).filter((url) => Boolean(url));
28562
+ const normalizedLinks = urls.map((url) => normalizeUrl$1(url, finalUrl)).filter((url) => Boolean(url) && !isCdnUtilityUrl(url ?? ""));
28185
28563
  const syncLinkEvidence = (target, links) => {
28186
28564
  const sortedLinks = [...links].sort((left, right) => left < right ? -1 : left > right ? 1 : 0);
28187
28565
  target.outgoingInternalCount = sortedLinks.length;
@@ -28260,7 +28638,7 @@ async function crawlSite(input, dependencies) {
28260
28638
  if (followLinks && task.depth < config.maxDepth) {
28261
28639
  if (memoryLimited) for (const value of result.urls) {
28262
28640
  const normalized = normalizeUrl$1(value, finalUrl);
28263
- if (normalized && sameOrigin(normalized, origin) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
28641
+ if (normalized && sameOrigin(normalized, origin) && !isCdnUtilityUrl(normalized) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
28264
28642
  discovered.add(normalized);
28265
28643
  memoryDeferredUrls.add(normalized);
28266
28644
  }
@@ -28285,6 +28663,8 @@ async function crawlSite(input, dependencies) {
28285
28663
  cancelled = isCancelled();
28286
28664
  const robotsDisallowed = skipReasonCounts["robots-disallowed"] ?? 0;
28287
28665
  if (robotsDisallowed > 0) warnings.push(`${robotsDisallowed} URL${robotsDisallowed === 1 ? "" : "s"} disallowed ${SEO_CRAWLER_USER_AGENT} in robots.txt. If the audit should reach them, allow the SEO-Skill token on only those public paths.`);
28666
+ const rateLimitWarning = rateLimit.warning();
28667
+ if (rateLimitWarning) warnings.push(rateLimitWarning);
28288
28668
  if (cancelled) {
28289
28669
  warnings.push("Crawl cancelled; returning a partial report.");
28290
28670
  emitStatus("cancelled", {
@@ -28296,7 +28676,7 @@ async function crawlSite(input, dependencies) {
28296
28676
  const sitemapEvidencePartial = sitemapDiscovery?.dataStatus === "partial";
28297
28677
  const accessEvidencePartial = requests.some((request) => request.outcome === "response" && request.accessBlock);
28298
28678
  const requiredSitemapUnavailable = config.mode === "sitemap" && sitemapDiscovery?.dataStatus === "unavailable";
28299
- const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
28679
+ const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || rateLimit.skippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
28300
28680
  const failed = !cancelled && pages.length === 0 && (requiredSitemapUnavailable || requests.length > 0 && requests.every((request) => request.outcome === "failure"));
28301
28681
  const requestEvidenceStatus = cancelled && inFlight.size > 0 ? "partial" : "available";
28302
28682
  const dataSources = await crawlDataSources({
@@ -28343,7 +28723,10 @@ async function crawlSite(input, dependencies) {
28343
28723
  timeoutMs: config.timeoutMs,
28344
28724
  fetchStatusPage: deps.fetchStatusPage,
28345
28725
  respectRobots: config.respectRobots,
28346
- signal
28726
+ signal,
28727
+ rate: rateLimit.rate,
28728
+ sleep: deps.sleep,
28729
+ retryDelayMs: rateLimit.settleMs
28347
28730
  }) : void 0;
28348
28731
  for (const page of pages) {
28349
28732
  if (!page.geo || !llmsTxt) continue;
@@ -28374,6 +28757,12 @@ async function crawlSite(input, dependencies) {
28374
28757
  ...startPage.responseHeaders?.["content-signal"] ? { contentSignal: startPage.responseHeaders["content-signal"] } : {}
28375
28758
  } } : {}
28376
28759
  } : aiSignals?.signals;
28760
+ const reachIssues = followLinks && config.maxDepth > 0 && config.maxPages > 1 && !cancelled ? crawlReachIssues({
28761
+ startUrl: config.url,
28762
+ pages: pages.filter((page) => page.status >= 200 && page.status < 300),
28763
+ crawledPages: pages.length,
28764
+ sitemapUrlsReturned: sitemapDiscovery?.urlsReturned ?? 0
28765
+ }) : [];
28377
28766
  const report = createCrawlReport({
28378
28767
  config,
28379
28768
  site,
@@ -28388,6 +28777,7 @@ async function crawlSite(input, dependencies) {
28388
28777
  ...externalLinkVerification ? { externalLinkVerification } : {},
28389
28778
  ...siteCheckResult ? { siteChecks: siteCheckResult.checks } : {},
28390
28779
  additionalIssues: [
28780
+ ...reachIssues,
28391
28781
  ...siteCheckResult?.issues ?? [],
28392
28782
  ...sitemapDiscovery ? sitemapHostIssues(sitemapDiscovery, config.url) : [],
28393
28783
  ...ai ? agentReadyIssues({
@@ -28405,7 +28795,12 @@ async function crawlSite(input, dependencies) {
28405
28795
  queueSafetySkippedUrls,
28406
28796
  queueSafetyInventoryCapped,
28407
28797
  originBackpressureSkippedUrls,
28408
- memoryPressureSkippedUrls
28798
+ rateLimit: rateLimit.caveatInput(config, pages.length),
28799
+ sitemapLimit: sitemapLimitCaveatInput(config, sitemapDiscovery, input.maxPages === void 0),
28800
+ memoryPressureSkippedUrls,
28801
+ queryVariantSkippedUrls: queryVariantSkipped.size,
28802
+ renderingFallbackUrls,
28803
+ reachIssues
28409
28804
  }),
28410
28805
  stats: {
28411
28806
  discoveredUrls: discovered.size,
@@ -28958,7 +29353,7 @@ function analyzeCtrUnderperformersFromRows(input) {
28958
29353
  // packages/core/dist/analyze/ctr-underperformers-report.js
28959
29354
  const MAX_GSC_ROWS$3 = 1e5;
28960
29355
  const DEFAULT_DAYS$3 = 28;
28961
- const MAX_DAYS$2 = 548;
29356
+ const MAX_DAYS$3 = 548;
28962
29357
  function plural$8(count, singular, pluralLabel = `${singular}s`) {
28963
29358
  return count === 1 ? singular : pluralLabel;
28964
29359
  }
@@ -28977,12 +29372,12 @@ async function ctrUnderperformersReportWith(input, dependencies) {
28977
29372
  maximum: 100,
28978
29373
  label: "limit"
28979
29374
  });
28980
- const explicitRange = explicitDateRange(input, MAX_DAYS$2);
29375
+ const explicitRange = explicitDateRange(input, MAX_DAYS$3);
28981
29376
  const days = explicitRange?.days ?? integerOption$1({
28982
29377
  value: input.days,
28983
29378
  fallback: DEFAULT_DAYS$3,
28984
29379
  minimum: 1,
28985
- maximum: MAX_DAYS$2,
29380
+ maximum: MAX_DAYS$3,
28986
29381
  label: "Days"
28987
29382
  });
28988
29383
  const now = dependencies.now();
@@ -32171,7 +32566,7 @@ function quickWinTemplateSummaries(items) {
32171
32566
  }
32172
32567
  // packages/core/dist/analyze/site-diagnostics/quick-wins-report.js
32173
32568
  const DEFAULT_DAYS$2 = 28;
32174
- const MAX_DAYS$1 = 548;
32569
+ const MAX_DAYS$2 = 548;
32175
32570
  const DEFAULT_VERIFY_LIMIT$1 = 5;
32176
32571
  const MAX_VERIFY_LIMIT = 100;
32177
32572
  const MAX_SOURCE_ROWS$2 = 1e5;
@@ -32189,12 +32584,12 @@ function applyVerification$1(item, coverage) {
32189
32584
  };
32190
32585
  }
32191
32586
  async function quickWinsReportWith(input, dependencies) {
32192
- const explicitRange = explicitDateRange(input, MAX_DAYS$1);
32587
+ const explicitRange = explicitDateRange(input, MAX_DAYS$2);
32193
32588
  const days = explicitRange?.days ?? integerOption$1({
32194
32589
  value: input.days,
32195
32590
  fallback: DEFAULT_DAYS$2,
32196
32591
  minimum: 1,
32197
- maximum: MAX_DAYS$1,
32592
+ maximum: MAX_DAYS$2,
32198
32593
  label: "Days"
32199
32594
  });
32200
32595
  const verifyLimit = integerOption$1({
@@ -32613,7 +33008,7 @@ function analyzeStrikingDistanceRows(input) {
32613
33008
  }
32614
33009
  // packages/core/dist/analyze/striking-distance-report.js
32615
33010
  const DEFAULT_DAYS$1 = 28;
32616
- const MAX_DAYS = 548;
33011
+ const MAX_DAYS$1 = 548;
32617
33012
  const DEFAULT_VERIFY_LIMIT = 5;
32618
33013
  const MAX_SOURCE_ROWS$1 = 1e5;
32619
33014
  function boundedInteger$2(value, fallback, minimum, maximum) {
@@ -32622,7 +33017,7 @@ function boundedInteger$2(value, fallback, minimum, maximum) {
32622
33017
  }
32623
33018
  function reportDays(value) {
32624
33019
  if (value === void 0) return DEFAULT_DAYS$1;
32625
- if (!Number.isInteger(value) || value < 1 || value > MAX_DAYS) throw new SeoError("INVALID_INPUT", `Days must be a whole number between 1 and ${MAX_DAYS}.`);
33020
+ if (!Number.isInteger(value) || value < 1 || value > MAX_DAYS$1) throw new SeoError("INVALID_INPUT", `Days must be a whole number between 1 and ${MAX_DAYS$1}.`);
32626
33021
  return value;
32627
33022
  }
32628
33023
  function dateRange(days, now) {
@@ -32662,7 +33057,7 @@ function verdict$1(analysis) {
32662
33057
  return `${countLabel(analysis.summary.eligibleRows, "eligible query/page row")} found; ${countLabel(analysis.summary.returnedRows, "row")} returned for review.`;
32663
33058
  }
32664
33059
  async function strikingDistanceWith(input, dependencies) {
32665
- const explicitRange = explicitDateRange(input, MAX_DAYS);
33060
+ const explicitRange = explicitDateRange(input, MAX_DAYS$1);
32666
33061
  const days = explicitRange?.days ?? reportDays(input.days);
32667
33062
  const generatedAt = dependencies.now().toISOString();
32668
33063
  const range = explicitRange?.range ?? dateRange(days, new Date(generatedAt));
@@ -41868,6 +42263,7 @@ const wait = (ms) => new Promise((done) => setTimeout(done, ms));
41868
42263
  function requireInput(input) {
41869
42264
  const needsKeywords = input.action === "keyword_metrics" || input.action === "serp";
41870
42265
  if (needsKeywords && input.keywords.length === 0) throw new CommandError("INVALID_INPUT", `${input.action} needs keywords.`);
42266
+ if (input.action === "link_gap" && !input.target) throw new CommandError("INVALID_INPUT", "link_gap needs a target domain and competitors.");
41871
42267
  if (!needsKeywords && !input.target) throw new CommandError("INVALID_INPUT", `${input.action} needs a target domain or URL.`);
41872
42268
  }
41873
42269
  async function eachKeyword(keywords, missed, rowsFor) {
@@ -41885,14 +42281,17 @@ async function eachKeyword(keywords, missed, rowsFor) {
41885
42281
  }
41886
42282
  async function research(provider, headers, input) {
41887
42283
  requireInput(input);
42284
+ if (input.action === "link_gap") return linkGap(provider, headers, input);
41888
42285
  const keys = need(headers, provider);
41889
42286
  if (provider === "dataforseo") return dataForSeo(keys, input);
41890
42287
  const missed = [];
42288
+ const rows = await ahrefs(keys[0] ?? "", input, missed);
41891
42289
  return {
41892
- rows: await ahrefs(keys[0] ?? "", input, missed),
42290
+ rows,
41893
42291
  costUsd: null,
41894
42292
  partial: false,
41895
- missed
42293
+ missed,
42294
+ noData: withoutData(input, rows, missed, 100)
41896
42295
  };
41897
42296
  }
41898
42297
  function countryName(code) {
@@ -41944,7 +42343,8 @@ async function dataForSeoTask([login, password], path, task) {
41944
42343
  if (result?.status_code === NO_RESULTS) return {
41945
42344
  cost: result.cost ?? 0,
41946
42345
  result: null,
41947
- partial: false
42346
+ partial: false,
42347
+ noResults: true
41948
42348
  };
41949
42349
  if (!result || result.status_code >= 4e4 && result.status_code !== PARTIAL) throw Object.assign(new CommandError("PROVIDER_ERROR", `dataforseo: ${result?.status_message ?? "no task result"}`), { cost: result?.cost ?? 0 });
41950
42350
  return {
@@ -41985,18 +42385,29 @@ async function dataForSeo(keys, input) {
41985
42385
  partial: false,
41986
42386
  missed: []
41987
42387
  };
42388
+ const rows = await dataForSeoRows(keys, input, spent);
41988
42389
  return {
41989
- rows: await dataForSeoRows(keys, input, spent),
42390
+ rows,
41990
42391
  costUsd: Math.round(spent.cost * 1e4) / 1e4,
41991
42392
  partial: spent.partial,
41992
- missed: spent.missed
42393
+ missed: spent.missed,
42394
+ noData: withoutData(input, rows, spent.missed, 700)
41993
42395
  };
41994
42396
  }
42397
+ function withoutData(input, rows, missed, keywordLimit) {
42398
+ if (input.action === "keyword_metrics" || input.action === "serp") {
42399
+ const max = input.action === "serp" ? 5 : keywordLimit;
42400
+ const answered = new Set(rows.map((row) => String(row.keyword ?? "").toLowerCase()));
42401
+ const failed = new Set(missed);
42402
+ return input.keywords.slice(0, max).filter((keyword) => !failed.has(keyword) && !answered.has(keyword.toLowerCase()));
42403
+ }
42404
+ return [];
42405
+ }
41995
42406
  async function dataForSeoRows(keys, input, spent) {
41996
42407
  const language = dataForSeoLanguage(input.country);
41997
42408
  if (!language && input.action !== "serp" && input.action !== "backlinks") throw new CommandError("INVALID_INPUT", `DataForSEO has no ${input.action} data for country "${input.country}".`);
41998
42409
  const market = dataForSeoMarket(input.country, language);
41999
- const post = async (path, task) => {
42410
+ const post = async (path, task, attempts = 1) => {
42000
42411
  const done = await dataForSeoTask(keys, path, {
42001
42412
  ...market,
42002
42413
  ...task
@@ -42006,6 +42417,7 @@ async function dataForSeoRows(keys, input, spent) {
42006
42417
  });
42007
42418
  spent.cost += done.cost;
42008
42419
  spent.partial ||= done.partial;
42420
+ if (done.noResults && attempts > 1) return post(path, task, attempts - 1);
42009
42421
  return done.result?.items ?? [];
42010
42422
  };
42011
42423
  const get = (value, path) => path.split(".").reduce((current, key) => current && typeof current === "object" ? current[key] : void 0, value);
@@ -42035,7 +42447,7 @@ async function dataForSeoRows(keys, input, spent) {
42035
42447
  return (await post("v3/serp/google/organic/live/advanced", {
42036
42448
  keyword,
42037
42449
  depth: Math.min(input.limit, 100)
42038
- })).filter((item) => item.type === "organic").map((item) => {
42450
+ }, 2)).filter((item) => item.type === "organic").map((item) => {
42039
42451
  const row = pick(item, {
42040
42452
  position: "rank_absolute",
42041
42453
  url: "url",
@@ -42043,9 +42455,11 @@ async function dataForSeoRows(keys, input, spent) {
42043
42455
  title: "title"
42044
42456
  });
42045
42457
  if (/^https?:\/\/(www\.)?google\.[a-z.]+\/goto\b/.test(String(row.url))) {
42046
- const site = /^https?:\/\/([^/\s›]+)/.exec(String(item.breadcrumb ?? ""));
42458
+ const crumb = String(item.breadcrumb ?? "");
42459
+ const site = /^https?:\/\/([^/\s›]+)/.exec(crumb);
42047
42460
  row.url = null;
42048
42461
  row.domain = site?.[1] ?? null;
42462
+ row.url_hint = site ? crumb.replace(/^https?:\/\//, "") : null;
42049
42463
  }
42050
42464
  return {
42051
42465
  keyword,
@@ -42075,6 +42489,93 @@ async function dataForSeoRows(keys, input, spent) {
42075
42489
  organic_etv: "full_domain_metrics.organic.etv"
42076
42490
  }));
42077
42491
  }
42492
+ const GAP_DOMAINS = 1e3;
42493
+ const GAP_COMPETITORS = 5;
42494
+ const SHORTENERS = [
42495
+ "bit.ly",
42496
+ "tinyurl.com",
42497
+ "t.co",
42498
+ "ow.ly",
42499
+ "buff.ly",
42500
+ "is.gd",
42501
+ "cutt.ly",
42502
+ "rebrand.ly",
42503
+ "shorturl.at",
42504
+ "rb.gy",
42505
+ "lnkd.in"
42506
+ ];
42507
+ const FREE_APP_HOSTS = [
42508
+ "web.app",
42509
+ "firebaseapp.com",
42510
+ "pages.dev"
42511
+ ];
42512
+ const LINK_LIST_NAME = /(subdomain|whois|login|signin|coupon|promo|voucher|discount|sitelike|similarsite|sitesimilar|siteprice|sitevalue|siteworth|websiteworth|hypestat|statshow|websiteoutlook|siteinfo|domaininfo|domainstat|domaintools|ipaddress|dnslookup|linkdirectory|webdirectory|alternativeto-)/;
42513
+ function isLinkList(domain) {
42514
+ const within = (name) => domain === name || domain.endsWith(`.${name}`);
42515
+ if (SHORTENERS.some(within)) return true;
42516
+ if (FREE_APP_HOSTS.some((name) => domain.endsWith(`.${name}`))) return true;
42517
+ return LINK_LIST_NAME.test(domain.replace(/[-.]/g, ""));
42518
+ }
42519
+ async function linkGap(provider, headers, input) {
42520
+ const target = bareTarget(input.target).replace(/^www\./, "");
42521
+ const competitors = [...new Set((input.competitors ?? []).map((site) => bareTarget(site).replace(/^www\./, "")))].filter((site) => site && site !== target);
42522
+ if (competitors.length < 2 || competitors.length > GAP_COMPETITORS) throw new CommandError("INVALID_INPUT", `link_gap needs a target and 2 to ${GAP_COMPETITORS} competitors.`);
42523
+ const score = provider === "ahrefs" ? "domain_rating" : "rank";
42524
+ const sites = [target, ...competitors];
42525
+ const lists = [];
42526
+ let cost = 0;
42527
+ let partial = false;
42528
+ for (const site of sites) {
42529
+ const got = await research(provider, headers, {
42530
+ ...input,
42531
+ action: "backlinks",
42532
+ target: site,
42533
+ limit: GAP_DOMAINS
42534
+ });
42535
+ cost += got.costUsd ?? 0;
42536
+ partial ||= got.partial;
42537
+ lists.push(got.rows);
42538
+ }
42539
+ const domainOf = (row) => String(row.domain ?? "").toLowerCase().replace(/^www\./, "");
42540
+ const authority = (row) => Number(row[score] ?? 0);
42541
+ const floors = lists.filter((rows) => rows.length >= GAP_DOMAINS).map((rows) => Math.min(...rows.map(authority)));
42542
+ const floor = floors.length ? Math.max(...floors) : null;
42543
+ const own = new Set((lists[0] ?? []).map(domainOf));
42544
+ const found = /* @__PURE__ */ new Map();
42545
+ competitors.forEach((site, index) => {
42546
+ for (const row of lists[index + 1] ?? []) {
42547
+ const domain = domainOf(row);
42548
+ if (!domain || own.has(domain) || domain === target) continue;
42549
+ if (domain.endsWith(`.${target}`)) continue;
42550
+ const entry = found.get(domain) ?? {
42551
+ value: 0,
42552
+ sites: []
42553
+ };
42554
+ entry.value = Math.max(entry.value, authority(row));
42555
+ if (!entry.sites.includes(site)) entry.sites.push(site);
42556
+ found.set(domain, entry);
42557
+ }
42558
+ });
42559
+ const gap = [...found.entries()].filter(([, entry]) => entry.sites.length >= 2 && (floor === null || entry.value >= floor));
42560
+ const linkLists = gap.map(([domain]) => domain).filter(isLinkList).sort((a, b) => a.localeCompare(b));
42561
+ return {
42562
+ rows: gap.filter(([domain]) => !isLinkList(domain)).sort(([a, left], [b, right]) => right.value - left.value || right.sites.length - left.sites.length || a.localeCompare(b)).map(([domain, entry]) => ({
42563
+ domain,
42564
+ [score]: entry.value,
42565
+ competitors_linking: entry.sites.length,
42566
+ links_to: entry.sites.join(" ")
42567
+ })),
42568
+ costUsd: provider === "dataforseo" ? Math.round(cost * 1e4) / 1e4 : null,
42569
+ partial,
42570
+ missed: [],
42571
+ noData: [],
42572
+ ...floor !== null ? { floor: {
42573
+ field: score,
42574
+ value: floor
42575
+ } } : {},
42576
+ ...linkLists.length ? { leftOut: linkLists } : {}
42577
+ };
42578
+ }
42078
42579
  function isPage(target) {
42079
42580
  return /\/./.test(bareTarget(target));
42080
42581
  }
@@ -42165,7 +42666,7 @@ const DIMENSIONS = [
42165
42666
  "device",
42166
42667
  "searchAppearance"
42167
42668
  ];
42168
- const ROWS = 25e3;
42669
+ const ROWS$1 = 25e3;
42169
42670
  async function runRows(input, deps) {
42170
42671
  const dimensions = input.dimensions?.length ? input.dimensions : ["query"];
42171
42672
  const filters = input.filters ?? [];
@@ -42174,7 +42675,7 @@ async function runRows(input, deps) {
42174
42675
  const query = {
42175
42676
  dimensions,
42176
42677
  type: "web",
42177
- maxRows: ROWS,
42678
+ maxRows: ROWS$1,
42178
42679
  ...filters.length ? { dimensionFilterGroups: [{
42179
42680
  groupType: "and",
42180
42681
  filters
@@ -42231,9 +42732,9 @@ async function runRows(input, deps) {
42231
42732
  const sum = (key) => rows.reduce((total, row) => total + Number(row[key] ?? 0), 0);
42232
42733
  notes.push(`Totals: clicks ${sum("clicks")}, impressions ${sum("impressions")}; previous period clicks ${sum("previous_clicks")}.`);
42233
42734
  }
42234
- notes.push(`Compared with ${previous.startDate} to ${previous.endDate}${before.rowsFetched >= ROWS ? " (earlier period hit the row limit)" : ""}. 0 clicks with no position means the row had no impressions in the current period.`);
42735
+ notes.push(`Compared with ${previous.startDate} to ${previous.endDate}${before.rowsFetched >= ROWS$1 ? " (earlier period hit the row limit)" : ""}. 0 clicks with no position means the row had no impressions in the current period.`);
42235
42736
  if (pageAt >= 0) notes.push("Page URLs with #fragments are merged into their page.");
42236
- if (current.rowsFetched >= ROWS) notes.push(`Search Console has more than ${ROWS} rows; add a filter to narrow it.`);
42737
+ if (current.rowsFetched >= ROWS$1) notes.push(`Search Console has more than ${ROWS$1} rows; add a filter to narrow it.`);
42237
42738
  notes.push("Search Console hides some rare queries for privacy, so query rows do not add up to the site totals.");
42238
42739
  const detail = input.detail ?? "summary";
42239
42740
  return {
@@ -42342,6 +42843,22 @@ const hostOf$2 = (url) => {
42342
42843
  return null;
42343
42844
  }
42344
42845
  };
42846
+ const NOT_SOURCES = [
42847
+ "dataforseo.com",
42848
+ "images.openai.com",
42849
+ "gstatic.com",
42850
+ "googleusercontent.com",
42851
+ "ggpht.com",
42852
+ "example.com",
42853
+ "example.org",
42854
+ "example.net"
42855
+ ];
42856
+ const IMAGE = /\.(png|jpe?g|gif|webp|avif|svg)(\?|#|$)/i;
42857
+ const isSource = (url) => {
42858
+ const host = hostOf$2(url);
42859
+ if (!host || IMAGE.test(url)) return false;
42860
+ return !NOT_SOURCES.some((name) => host === name || host.endsWith(`.${name}`));
42861
+ };
42345
42862
  const empty = {
42346
42863
  cited_url: null,
42347
42864
  sources: [],
@@ -42370,7 +42887,7 @@ function judge(engine, task, site) {
42370
42887
  const cited = all.find((url) => onSite(url, site.domain)) ?? null;
42371
42888
  const base = {
42372
42889
  cited_url: cited,
42373
- sources: [...new Set(all.filter((url) => !onSite(url, site.domain)).map(hostOf$2).filter((host) => Boolean(host)))].slice(0, MAX_SOURCES),
42890
+ sources: [...new Set(all.filter((url) => !onSite(url, site.domain) && isSource(url)).map(hostOf$2).filter((host) => Boolean(host)))].slice(0, MAX_SOURCES),
42374
42891
  model: shown.model,
42375
42892
  answered_at
42376
42893
  };
@@ -42423,6 +42940,7 @@ async function runAiAnswers(input, headers, deps, more) {
42423
42940
  question,
42424
42941
  impressions,
42425
42942
  task: null,
42943
+ ref: null,
42426
42944
  cost_usd: 0,
42427
42945
  ...empty
42428
42946
  });
@@ -42434,7 +42952,8 @@ async function runAiAnswers(input, headers, deps, more) {
42434
42952
  ...blank("", null),
42435
42953
  engine: engine.name,
42436
42954
  result: "pending",
42437
- task: value
42955
+ task: value,
42956
+ ref: value
42438
42957
  };
42439
42958
  rows.push(row);
42440
42959
  waiting.push({
@@ -42487,6 +43006,7 @@ async function runAiAnswers(input, headers, deps, more) {
42487
43006
  }
42488
43007
  row.result = "pending";
42489
43008
  row.task = `${engine.name}:${task.id}`;
43009
+ row.ref = row.task;
42490
43010
  waiting.push({
42491
43011
  row,
42492
43012
  engine,
@@ -42527,6 +43047,7 @@ async function runAiAnswers(input, headers, deps, more) {
42527
43047
  const cost = usd(rows.reduce((sum, row) => sum + row.cost_usd, 0));
42528
43048
  const count = (result) => rows.filter((row) => row.result === result).length;
42529
43049
  const left = rows.filter((row) => row.task).map((row) => row.task);
43050
+ const refs = rows.flatMap((row) => row.ref ? [row.ref] : []);
42530
43051
  const cites = /* @__PURE__ */ new Map();
42531
43052
  for (const row of rows) for (const host of row.sources) cites.set(host, (cites.get(host) ?? 0) + 1);
42532
43053
  const topCited = [...cites].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10);
@@ -42560,7 +43081,7 @@ async function runAiAnswers(input, headers, deps, more) {
42560
43081
  limit: 50,
42561
43082
  detail: input.detail,
42562
43083
  notes,
42563
- more
43084
+ more: (offset) => more(offset, refs)
42564
43085
  }),
42565
43086
  data: {
42566
43087
  provider: "dataforseo",
@@ -42574,6 +43095,7 @@ async function runAiAnswers(input, headers, deps, more) {
42574
43095
  questions: [...new Set(rows.map((row) => row.question))],
42575
43096
  costUsd: cost,
42576
43097
  pendingTasks: left,
43098
+ tasks: refs,
42577
43099
  citedSites: topCited.map(([host, answers]) => ({
42578
43100
  host,
42579
43101
  answers
@@ -42688,7 +43210,7 @@ function matchGa4WebStreams(site, candidates) {
42688
43210
  }
42689
43211
  // packages/core/dist/commands/analytics-site.js
42690
43212
  const STREAM_CHECKS = 30;
42691
- const AT_ONCE = 5;
43213
+ const AT_ONCE$1 = 5;
42692
43214
  const TIED = 3;
42693
43215
  const hostOf$1 = (site) => {
42694
43216
  const value = site.replace(/^sc-domain:/, "");
@@ -42753,8 +43275,8 @@ async function lookProperty(access, site) {
42753
43275
  }
42754
43276
  const order = ranked(site, candidates).slice(0, STREAM_CHECKS);
42755
43277
  let failures = 0;
42756
- for (let start = 0; start < order.length; start += AT_ONCE) {
42757
- const batch = order.slice(start, start + AT_ONCE);
43278
+ for (let start = 0; start < order.length; start += AT_ONCE$1) {
43279
+ const batch = order.slice(start, start + AT_ONCE$1);
42758
43280
  const streams = await Promise.all(batch.map((candidate) => listAnalyticsDataStreams(candidate.google, candidate.property, { signIn: access.signIn }).catch(() => {
42759
43281
  failures += 1;
42760
43282
  return [];
@@ -42955,7 +43477,7 @@ async function totals$1(found, access, range, before, pages) {
42955
43477
  notes
42956
43478
  };
42957
43479
  }
42958
- const change$1 = (value) => value === null ? "n/a" : `${value > 0 ? "+" : ""}${value}%`;
43480
+ const change$2 = (value) => value === null ? "n/a" : `${value > 0 ? "+" : ""}${value}%`;
42959
43481
  function engineText(rows) {
42960
43482
  const shown = rows.filter((row) => row.sessions || row.previousSessions);
42961
43483
  if (shown.length < 2) return "";
@@ -42965,7 +43487,7 @@ function aiText(ai) {
42965
43487
  const list = (rows, key) => rows.slice(0, AI_ROWS).map((row) => `${row[key]} ${row.sessions} (was ${row.previousSessions})`).join(", ");
42966
43488
  if (!ai.sessions && !ai.previousSessions) return "Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): none in either period.";
42967
43489
  return [
42968
- `Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): ${ai.sessions} sessions (was ${ai.previousSessions}, ${change$1(ai.sessionsChangePct)}). By source: ${list(ai.bySource, "source")}.`,
43490
+ `Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): ${ai.sessions} sessions (was ${ai.previousSessions}, ${change$2(ai.sessionsChangePct)}). By source: ${list(ai.bySource, "source")}.`,
42969
43491
  ai.topLandingPages.length ? `Top landing pages from AI assistants: ${list(ai.topLandingPages, "page")}.` : "",
42970
43492
  "Only visits that pass a referrer or utm_source count; AI apps that send none show as Direct."
42971
43493
  ].filter(Boolean).join(" ");
@@ -43082,7 +43604,7 @@ async function analyticsBeside(access, site, range, before, pages) {
43082
43604
  const keyEvents = organic.keyEvents || organic.previousKeyEvents ? `, Organic Search key events ${organic.keyEvents} (was ${organic.previousKeyEvents})` : "";
43083
43605
  return {
43084
43606
  text: [
43085
- `Google Analytics, property ${data.propertyName} (${data.property}), matched by its web stream ${data.webStream}${pages ? `, landing pages whose path contains "${pages}"` : ""}: Organic Search sessions from Google ${google.sessions} (was ${google.previousSessions}, ${change$1(google.sessionsChangePct)}), from every search engine ${organic.sessions} (was ${organic.previousSessions}, ${change$1(organic.sessionsChangePct)})${keyEvents}, sessions from all channels ${all.sessions} (was ${all.previousSessions}, ${change$1(all.sessionsChangePct)}).`,
43607
+ `Google Analytics, property ${data.propertyName} (${data.property}), matched by its web stream ${data.webStream}${pages ? `, landing pages whose path contains "${pages}"` : ""}: Organic Search sessions from Google ${google.sessions} (was ${google.previousSessions}, ${change$2(google.sessionsChangePct)}), from every search engine ${organic.sessions} (was ${organic.previousSessions}, ${change$2(organic.sessionsChangePct)})${keyEvents}, sessions from all channels ${all.sessions} (was ${all.previousSessions}, ${change$2(all.sessionsChangePct)}).`,
43086
43608
  engineText(data.organicSearchByEngine),
43087
43609
  "Sessions are Google Analytics’ own count, not Search Console clicks.",
43088
43610
  ...data.notes,
@@ -43309,7 +43831,7 @@ function auditPreview(snapshot) {
43309
43831
  }
43310
43832
  const WEEK_MS = 6048e5;
43311
43833
  const day$1 = (ms) => ms === null ? "" : new Date(ms).toISOString().slice(0, 10);
43312
- const count$2 = (value) => value.toLocaleString("en-US");
43834
+ const count$3 = (value) => value.toLocaleString("en-US");
43313
43835
  function relative$2(value, origin) {
43314
43836
  if (!value) return "";
43315
43837
  return origin && value.startsWith(origin) ? value.slice(origin.length) || "/" : value;
@@ -43330,7 +43852,7 @@ function rowsTable(rows, checks) {
43330
43852
  changed(row) ? row.prev_coverage ?? "" : "",
43331
43853
  changed(row) ? day$1(row.changed_at) : "",
43332
43854
  day$1(row.checked_at),
43333
- `${count$2(row.clicks)} (${count$2(row.previous_clicks)})${row.lost ? " lost" : ""}`,
43855
+ `${count$3(row.clicks)} (${count$3(row.previous_clicks)})${row.lost ? " lost" : ""}`,
43334
43856
  row.in_sitemap ? "yes" : "no",
43335
43857
  row.google_canonical && row.google_canonical !== row.url ? relative$2(row.google_canonical, checks.origin) : "",
43336
43858
  row.error ?? ""
@@ -43338,12 +43860,12 @@ function rowsTable(rows, checks) {
43338
43860
  }
43339
43861
  function head(site, checks) {
43340
43862
  const { queued } = checks;
43341
- const quota = checks.properties.map((item) => `${item.property} (${count$2(item.urls)} URLs: ${count$2(item.top)} top pages checked daily, the rest every ${item.cycle_days === 1 ? "day" : `${item.cycle_days} days`}; ${count$2(item.used_today)} of ${count$2(checks.per_day)} checks used today)`).join(", ");
43863
+ const quota = checks.properties.map((item) => `${item.property} (${count$3(item.urls)} URLs: ${count$3(item.top)} top pages checked daily, the rest every ${item.cycle_days === 1 ? "day" : `${item.cycle_days} days`}; ${count$3(item.used_today)} of ${count$3(checks.per_day)} checks used today)`).join(", ");
43342
43864
  return [
43343
43865
  `Google indexing of ${site}, from the URL Inspection results this server keeps for it (as of ${new Date(checks.as_of).toISOString().slice(0, 16).replace("T", " ")} UTC).`,
43344
- checks.dropped_this_week || checks.returned_this_week ? `In the last 7 days ${count$2(checks.dropped_this_week)} ${checks.dropped_this_week === 1 ? "URL" : "URLs"} dropped out of Google's index and ${count$2(checks.returned_this_week)} came back; they are listed first. A top page's drop counts only after a second check an hour later.` : null,
43345
- `Covered: ${count$2(checks.covered)} of ${count$2(checks.known)} known URLs have a current result. Known URLs are the sitemap's, the crawl's and the Search Console pages with clicks or impressions. Each property spends its full ${count$2(checks.per_day)} checks a day: its top pages (lost clicks, then clicks, then impressions, up to ${count$2(checks.top_per_day)}) daily, the rest as often as the quota allows, and a page the weekly crawl saw change at once.`,
43346
- queued.total ? `Queued: ${count$2(queued.total)} (${count$2(queued.lost)} pages that lost their clicks, ${count$2(queued.search)} with clicks or impressions, ${count$2(queued.sitemap)} sitemap URLs never checked, ${count$2(queued.stale)} older or changed), checked in that order. Time left: ${etaText(checks.eta_hours)}, at up to ${count$2(checks.per_day)} checks a day per property, spread over the day.` : "Queued: nothing; every known URL has a current result.",
43866
+ checks.dropped_this_week || checks.returned_this_week ? `In the last 7 days ${count$3(checks.dropped_this_week)} ${checks.dropped_this_week === 1 ? "URL" : "URLs"} dropped out of Google's index and ${count$3(checks.returned_this_week)} came back; they are listed first. A top page's drop counts only after a second check an hour later.` : null,
43867
+ `Covered: ${count$3(checks.covered)} of ${count$3(checks.known)} known URLs have a current result. Known URLs are the sitemap's, the crawl's and the Search Console pages with clicks or impressions. Each property spends its full ${count$3(checks.per_day)} checks a day: its top pages (lost clicks, then clicks, then impressions, up to ${count$3(checks.top_per_day)}) daily, the rest as often as the quota allows, and a page the weekly crawl saw change at once.`,
43868
+ queued.total ? `Queued: ${count$3(queued.total)} (${count$3(queued.lost)} pages that lost their clicks, ${count$3(queued.search)} with clicks or impressions, ${count$3(queued.sitemap)} sitemap URLs never checked, ${count$3(queued.stale)} older or changed), checked in that order. Time left: ${etaText(checks.eta_hours)}, at up to ${count$3(checks.per_day)} checks a day per property, spread over the day.` : "Queued: nothing; every known URL has a current result.",
43347
43869
  quota ? `Checked through ${quota}.` : null,
43348
43870
  checks.stopped ? `Checks stopped early: ${checks.stopped}` : null,
43349
43871
  "A URL without a result has none here, so this is not an all-clear for it."
@@ -43352,9 +43874,9 @@ function head(site, checks) {
43352
43874
  function suggestionsText(checks) {
43353
43875
  if (!checks.suggestions.length) return null;
43354
43876
  return [
43355
- `The sitemap has more URLs than these properties can check every 7 days. Each Search Console property gets its own ${count$2(checks.per_day)} checks a day, so adding these folders as URL-prefix properties (on the same Google account) would check more:`,
43877
+ `The sitemap has more URLs than these properties can check every 7 days. Each Search Console property gets its own ${count$3(checks.per_day)} checks a day, so adding these folders as URL-prefix properties (on the same Google account) would check more:`,
43356
43878
  "property to add | sitemap URLs in it | more URLs checked a week",
43357
- ...checks.suggestions.map((item) => `${item.property} | ${count$2(item.urls)} | ${count$2(item.gain)}`),
43879
+ ...checks.suggestions.map((item) => `${item.property} | ${count$3(item.urls)} | ${count$3(item.gain)}`),
43358
43880
  "The next day the checks find a new property in Search Console and use its quota for the URLs in that folder."
43359
43881
  ].join("\n");
43360
43882
  }
@@ -43369,7 +43891,7 @@ function runAuditChecks(input, checks, call) {
43369
43891
  data: checks
43370
43892
  };
43371
43893
  if (!checks.started) return {
43372
- text: `${site} is in the weekly site audit, and its Google checks are starting now: pages that lost their clicks first, then pages by clicks and impressions, then sitemap URLs. ${checks.known ? `${count$2(checks.known)} URLs are known so far. ` : ""}Run ${call("index", { site })} again in a few minutes for the coverage and the time left.`,
43894
+ text: `${site} is in the weekly site audit, and its Google checks are starting now: pages that lost their clicks first, then pages by clicks and impressions, then sitemap URLs. ${checks.known ? `${count$3(checks.known)} URLs are known so far. ` : ""}Run ${call("index", { site })} again in a few minutes for the coverage and the time left.`,
43373
43895
  data: checks
43374
43896
  };
43375
43897
  const more = (offset) => input.paged ? `More: ${call("index", {
@@ -43393,7 +43915,7 @@ function runAuditChecks(input, checks, call) {
43393
43915
  const reasons = checks.reasons.length ? [
43394
43916
  "Google's reason at each URL's latest check:",
43395
43917
  "reason | verdict | urls | changed to it this week",
43396
- ...checks.reasons.map((item) => `${item.coverage ?? ""} | ${item.verdict ?? ""} | ${count$2(item.urls)} | ${count$2(item.changed_this_week)}`)
43918
+ ...checks.reasons.map((item) => `${item.coverage ?? ""} | ${item.verdict ?? ""} | ${count$3(item.urls)} | ${count$3(item.changed_this_week)}`)
43397
43919
  ].join("\n") : "No URL has a result yet.";
43398
43920
  const shown = checks.rows.slice(0, 20);
43399
43921
  return {
@@ -43450,7 +43972,7 @@ async function bingFor(deps, property) {
43450
43972
  missing: site ? `${site.url} is in the connected Bing account but is not verified, so Bing gives no data for it. Verify it at ${BING_HOME}.` : `The connected Bing account does not have ${property.replace(/^sc-domain:/, "")}. Add and verify it at ${BING_HOME} (it can import sites from Search Console). New sites take a few days to show data.`
43451
43973
  };
43452
43974
  }
43453
- function days(range) {
43975
+ function days$1(range) {
43454
43976
  return Math.round((Date.parse(range.endDate) - Date.parse(range.startDate)) / 864e5) + 1;
43455
43977
  }
43456
43978
  async function bingTotals(source, range, before) {
@@ -43463,7 +43985,7 @@ async function bingTotals(source, range, before) {
43463
43985
  clicks: rows.reduce((total, row) => total + row.clicks, 0),
43464
43986
  impressions: rows.reduce((total, row) => total + row.impressions, 0),
43465
43987
  days: new Set(rows.map((row) => row.date)).size,
43466
- expectedDays: days(window)
43988
+ expectedDays: days$1(window)
43467
43989
  };
43468
43990
  };
43469
43991
  return {
@@ -43473,14 +43995,14 @@ async function bingTotals(source, range, before) {
43473
43995
  invalidRows: traffic.invalidRows
43474
43996
  };
43475
43997
  }
43476
- const change = (now, before) => {
43998
+ const change$1 = (now, before) => {
43477
43999
  const value = pct(now, before);
43478
44000
  return value === null ? "n/a" : `${value > 0 ? "+" : ""}${value}%`;
43479
44001
  };
43480
44002
  function enginesText(google, bing) {
43481
44003
  if ("missing" in bing) return `Bing: ${bing.missing}`;
43482
44004
  const { current: now, previous: then } = bing;
43483
- const lines = [`Search engines, same dates. Google (Search Console): clicks ${google.clicks} (was ${google.previousClicks}, ${change(google.clicks, google.previousClicks)}), impressions ${google.impressions}. Bing (${bing.site}): clicks ${now.clicks} (was ${then.clicks}, ${change(now.clicks, then.clicks)}), impressions ${now.impressions} (was ${then.impressions}).`];
44005
+ const lines = [`Search engines, same dates. Google (Search Console): clicks ${google.clicks} (was ${google.previousClicks}, ${change$1(google.clicks, google.previousClicks)}), impressions ${google.impressions}. Bing (${bing.site}): clicks ${now.clicks} (was ${then.clicks}, ${change$1(now.clicks, then.clicks)}), impressions ${now.impressions} (was ${then.impressions}).`];
43484
44006
  const both = google.clicks + now.clicks;
43485
44007
  if (both > 0) lines.push(`Share of these clicks: Google ${Math.round(google.clicks / both * 100)}%, Bing ${Math.round(now.clicks / both * 100)}%.`);
43486
44008
  if (now.days < now.expectedDays || then.days < then.expectedDays) lines.push(`Bing has data for ${now.days} of ${now.expectedDays} days (earlier period ${then.days} of ${then.expectedDays}), so its totals are partial.`);
@@ -43634,7 +44156,7 @@ async function runBingRows(input, deps) {
43634
44156
  }
43635
44157
  };
43636
44158
  }
43637
- function moved(rows, range, before, name, limit) {
44159
+ function moved$1(rows, range, before, name, limit) {
43638
44160
  const now = sumWeeks(rows, range);
43639
44161
  const then = sumWeeks(rows, before);
43640
44162
  const old = new Map(then.rows.map((row) => [row.key, row]));
@@ -43672,8 +44194,8 @@ async function bingOverview(source, range, before) {
43672
44194
  ]);
43673
44195
  if ("missing" in totals) throw new CommandError("NOT_CONNECTED", totals.missing);
43674
44196
  const { current: now, previous: then } = totals;
43675
- const q = moved(queries.rows, range, before, "query", 15);
43676
- const p = moved(pages.rows, range, before, "page", 15);
44197
+ const q = moved$1(queries.rows, range, before, "query", 15);
44198
+ const p = moved$1(pages.rows, range, before, "page", 15);
43677
44199
  return {
43678
44200
  bingSite: source.site,
43679
44201
  range,
@@ -43724,7 +44246,7 @@ function bingOverviewText(report, heading, origin, rowsPerList) {
43724
44246
  };
43725
44247
  return [
43726
44248
  `${heading}, ${range.startDate} to ${range.endDate} against ${before.startDate} to ${before.endDate}.`,
43727
- t.days || t.previousDays ? `Bing totals: clicks ${t.clicks} (was ${t.previousClicks}, ${change(t.clicks, t.previousClicks)}), impressions ${t.impressions} (was ${t.previousImpressions}, ${change(t.impressions, t.previousImpressions)}).` : "Bing totals: unknown. Bing has no daily data for these dates.",
44249
+ t.days || t.previousDays ? `Bing totals: clicks ${t.clicks} (was ${t.previousClicks}, ${change$1(t.clicks, t.previousClicks)}), impressions ${t.impressions} (was ${t.previousImpressions}, ${change$1(t.impressions, t.previousImpressions)}).` : "Bing totals: unknown. Bing has no daily data for these dates.",
43728
44250
  group("pages", "page"),
43729
44251
  group("queries", "query"),
43730
44252
  `Limits:\n${report.caveats.map((line) => `- ${line}`).join("\n")}`
@@ -43801,13 +44323,13 @@ function clickyRange(input) {
43801
44323
  endDate: input.endDate
43802
44324
  };
43803
44325
  }
43804
- const count$1 = (item) => {
44326
+ const count$2 = (item) => {
43805
44327
  const value = Number(item.value);
43806
44328
  return Number.isSafeInteger(value) && value >= 0 ? value : null;
43807
44329
  };
43808
44330
  const totals = (items) => items.map((item) => ({
43809
44331
  name: item.title ?? "",
43810
- visits: count$1(item)
44332
+ visits: count$2(item)
43811
44333
  })).filter((item) => Boolean(item.name) && item.visits !== null);
43812
44334
  const line = (items) => {
43813
44335
  const top = items.slice(0, 10).map((item) => `${item.name} ${item.visits}`);
@@ -43838,7 +44360,7 @@ async function runClickyRows(input, keys, options) {
43838
44360
  const entrances = stats["pages-entrance"] ?? [];
43839
44361
  const visitsByPage = /* @__PURE__ */ new Map();
43840
44362
  for (const item of entrances) {
43841
- const visits = count$1(item);
44363
+ const visits = count$2(item);
43842
44364
  if (visits === null || !item.url) continue;
43843
44365
  let page;
43844
44366
  try {
@@ -43887,7 +44409,8 @@ async function runClickyRows(input, keys, options) {
43887
44409
  // packages/core/dist/commands/domain-rating.js
43888
44410
  const DR_ATTRIBUTION = "domain_rating: Domain Rating by Ahrefs (https://ahrefs.com/).";
43889
44411
  const ENDPOINT = "https://api.ahrefs.com/v3/public/domain-rating-free";
43890
- const MAX_DOMAINS = 25;
44412
+ const MAX_DOMAINS = 50;
44413
+ const AT_ONCE = 25;
43891
44414
  const TTL_SECONDS = 86400;
43892
44415
  const DOMAIN_FIELDS = [
43893
44416
  "domain",
@@ -43918,23 +44441,26 @@ async function lookup(domain, key) {
43918
44441
  }
43919
44442
  return body.domain_rating?.domain_rating ?? null;
43920
44443
  }
43921
- async function domainRatings(domains, key) {
44444
+ async function domainRatings(domains, key, max = MAX_DOMAINS) {
43922
44445
  const found = /* @__PURE__ */ new Map();
43923
44446
  if (!key) return found;
43924
- const unique = [...new Set(domains.map(bareDomain).filter(Boolean))].slice(0, MAX_DOMAINS);
43925
- const values = await Promise.all(unique.map((domain) => lookup(domain, key).catch(() => null)));
43926
- unique.forEach((domain, index) => {
43927
- found.set(domain, values[index] ?? null);
43928
- });
44447
+ const unique = [...new Set(domains.map(bareDomain).filter(Boolean))].slice(0, max);
44448
+ for (let start = 0; start < unique.length; start += AT_ONCE) {
44449
+ const batch = unique.slice(start, start + AT_ONCE);
44450
+ const values = await Promise.all(batch.map((domain) => lookup(domain, key).catch(() => null)));
44451
+ batch.forEach((domain, index) => {
44452
+ found.set(domain, values[index] ?? null);
44453
+ });
44454
+ }
43929
44455
  return found;
43930
44456
  }
43931
- async function withDomainRating(rows, key) {
44457
+ async function withDomainRating(rows, key, start = 0) {
43932
44458
  const field = DOMAIN_FIELDS.find((name) => rows.some((row) => typeof row[name] === "string"));
43933
44459
  if (!key || !field || rows.some((row) => "domain_rating" in row)) return {
43934
44460
  rows,
43935
44461
  added: false
43936
44462
  };
43937
- const ratings = await domainRatings(rows.map((row) => String(row[field] ?? "")), key);
44463
+ const ratings = await domainRatings(rows.slice(start).map((row) => String(row[field] ?? "")), key);
43938
44464
  return {
43939
44465
  rows: rows.map((row) => {
43940
44466
  const next = {};
@@ -45167,6 +45693,265 @@ function temporaryRedirectNote(pages) {
45167
45693
  if (!temporary.length) return null;
45168
45694
  return `Note: 302 and 307 are temporary redirects; a permanent move usually uses 301 or 308. Temporary redirects here: ${temporary.map((page) => `${page.url} (${page.status} to ${page.redirects_to})`).join("; ")}.`;
45169
45695
  }
45696
+ // packages/core/dist/commands/updates.js
45697
+ const UPDATES_SECTION = "google-updates";
45698
+ const UPDATES_SECTION_TEXT = "One Google ranking update (core, spam and similar, from Google’s Search Status Dashboard) laid over the site’s Search Console data: clicks and impressions from the day it started against the same number of days just before (whole weeks, up to 28 days), whether that change is outside the site’s own movement in the 8 comparisons before it (a heuristic), and the folders, pages and queries that gained or lost most. The latest update by default; with end_date, the latest update that started on or before it. Timing only, not cause.";
45699
+ const MAX_DAYS = 28;
45700
+ const BASELINE = 8;
45701
+ const ROWS = 25e3;
45702
+ const LISTED = 10;
45703
+ const SHOWN = 5;
45704
+ const MIN_CLICKS = 50;
45705
+ const days = (from, to) => Math.round((Date.parse(to) - Date.parse(from)) / 864e5) + 1;
45706
+ const asImpactUpdate = (update) => ({
45707
+ id: update.id,
45708
+ name: update.name,
45709
+ start: update.start,
45710
+ end: update.end ?? null,
45711
+ url: update.sourceUrl
45712
+ });
45713
+ function sum(rows, from, to) {
45714
+ let clicks = 0;
45715
+ let impressions = 0;
45716
+ for (const row of rows) {
45717
+ const day = row.keys[0] ?? "";
45718
+ if (day < from || day > to) continue;
45719
+ clicks += row.clicks;
45720
+ impressions += row.impressions;
45721
+ }
45722
+ return {
45723
+ clicks,
45724
+ impressions
45725
+ };
45726
+ }
45727
+ const change = (now, before) => before > 0 ? (now - before) / before * 100 : null;
45728
+ function joined(before, after) {
45729
+ const byKey = /* @__PURE__ */ new Map();
45730
+ const at = (key) => {
45731
+ let row = byKey.get(key);
45732
+ if (!row) {
45733
+ row = {
45734
+ key,
45735
+ clicks: 0,
45736
+ previous: 0,
45737
+ impressions: 0,
45738
+ previous_impressions: 0
45739
+ };
45740
+ byKey.set(key, row);
45741
+ }
45742
+ return row;
45743
+ };
45744
+ for (const row of before) {
45745
+ const item = at(row.keys[0] ?? "");
45746
+ item.previous += row.clicks;
45747
+ item.previous_impressions += row.impressions;
45748
+ }
45749
+ for (const row of after) {
45750
+ const item = at(row.keys[0] ?? "");
45751
+ item.clicks += row.clicks;
45752
+ item.impressions += row.impressions;
45753
+ }
45754
+ return [...byKey.values()];
45755
+ }
45756
+ const moved = (row) => row.clicks - row.previous;
45757
+ const byKey = (a, b) => a.key < b.key ? -1 : a.key > b.key ? 1 : 0;
45758
+ function extremes(rows) {
45759
+ return {
45760
+ lost: rows.filter((row) => moved(row) < 0).sort((a, b) => moved(a) - moved(b) || byKey(a, b)).slice(0, LISTED),
45761
+ gained: rows.filter((row) => moved(row) > 0).sort((a, b) => moved(b) - moved(a) || byKey(a, b)).slice(0, LISTED)
45762
+ };
45763
+ }
45764
+ function folderOf(url) {
45765
+ try {
45766
+ const parts = new URL(url).pathname.split("/").filter(Boolean);
45767
+ return parts.length > 1 ? `/${parts[0]}/` : "/";
45768
+ } catch {
45769
+ return "/";
45770
+ }
45771
+ }
45772
+ function folders(pages) {
45773
+ const byFolder = /* @__PURE__ */ new Map();
45774
+ for (const page of pages) {
45775
+ const key = folderOf(page.key);
45776
+ const row = byFolder.get(key) ?? {
45777
+ key,
45778
+ clicks: 0,
45779
+ previous: 0,
45780
+ impressions: 0,
45781
+ previous_impressions: 0
45782
+ };
45783
+ row.clicks += page.clicks;
45784
+ row.previous += page.previous;
45785
+ row.impressions += page.impressions;
45786
+ row.previous_impressions += page.previous_impressions;
45787
+ byFolder.set(key, row);
45788
+ }
45789
+ return [...byFolder.values()].filter((row) => moved(row) !== 0).sort((a, b) => Math.abs(moved(b)) - Math.abs(moved(a)) || byKey(a, b)).slice(0, LISTED);
45790
+ }
45791
+ function windowDays(update, latest) {
45792
+ const start = update.start.slice(0, 10);
45793
+ const end = update.end?.slice(0, 10) ?? null;
45794
+ const wanted = Math.min(MAX_DAYS, Math.max(7, Math.ceil(days(start, end ?? latest) / 7) * 7));
45795
+ const covered = Math.floor(Math.max(0, days(start, latest)) / 7) * 7;
45796
+ return {
45797
+ start,
45798
+ end,
45799
+ wanted,
45800
+ n: Math.min(wanted, covered)
45801
+ };
45802
+ }
45803
+ async function updateImpact(site, update, read, now) {
45804
+ const { start, end, wanted, n } = windowDays(update, latestFinalGscDate(now));
45805
+ const empty = {
45806
+ update,
45807
+ site,
45808
+ before: null,
45809
+ after: null,
45810
+ normal_range: null,
45811
+ verdict: "not-enough-data",
45812
+ capped: {
45813
+ pages: false,
45814
+ queries: false
45815
+ },
45816
+ folders: [],
45817
+ pages: {
45818
+ lost: [],
45819
+ gained: []
45820
+ },
45821
+ queries: {
45822
+ lost: [],
45823
+ gained: []
45824
+ }
45825
+ };
45826
+ if (n < 7) return {
45827
+ ...empty,
45828
+ window: "waiting"
45829
+ };
45830
+ const after = {
45831
+ startDate: start,
45832
+ endDate: shiftIsoDate(start, n - 1)
45833
+ };
45834
+ const before = {
45835
+ startDate: shiftIsoDate(start, -n),
45836
+ endDate: shiftIsoDate(start, -1)
45837
+ };
45838
+ const first = Math.ceil(n / 7);
45839
+ const pairs = Array.from({ length: BASELINE }, (_, i) => {
45840
+ const back = 7 * (first + i);
45841
+ return {
45842
+ after: [shiftIsoDate(start, -back), shiftIsoDate(start, n - 1 - back)],
45843
+ before: [shiftIsoDate(start, -back - n), shiftIsoDate(start, -back - 1)]
45844
+ };
45845
+ });
45846
+ const shared = sharedReader(read);
45847
+ const call = (range, dimension) => shared(site, {
45848
+ ...range,
45849
+ dimensions: [dimension],
45850
+ type: "web",
45851
+ dataState: "final",
45852
+ ...dimension === "date" ? {} : { maxRows: ROWS }
45853
+ });
45854
+ const [daily, pagesBefore, pagesAfter, queriesBefore, queriesAfter] = await Promise.all([
45855
+ call({
45856
+ startDate: pairs.at(-1)?.before[0],
45857
+ endDate: after.endDate
45858
+ }, "date"),
45859
+ call(before, "page"),
45860
+ call(after, "page"),
45861
+ call(before, "query"),
45862
+ call(after, "query")
45863
+ ]);
45864
+ const totals = (range) => ({
45865
+ ...range,
45866
+ ...sum(daily.rows, range.startDate, range.endDate)
45867
+ });
45868
+ const was = totals(before);
45869
+ const is = totals(after);
45870
+ const seen = pairs.map((pair) => change(sum(daily.rows, pair.after[0], pair.after[1]).clicks, sum(daily.rows, pair.before[0], pair.before[1]).clicks)).filter((value) => value !== null);
45871
+ const normal = seen.length ? {
45872
+ comparisons: seen.length,
45873
+ clicks_pct: Math.round(Math.max(...seen.map((v) => Math.abs(v))) * 10) / 10
45874
+ } : null;
45875
+ const moves = change(is.clicks, was.clicks);
45876
+ const verdict = !normal || normal.comparisons < BASELINE / 2 || was.clicks < MIN_CLICKS || moves === null ? "not-enough-data" : Math.abs(moves) > normal.clicks_pct ? "outside-normal" : "within-normal";
45877
+ const pageRows = joined(pagesBefore.rows, pagesAfter.rows);
45878
+ return {
45879
+ ...empty,
45880
+ window: end && n === wanted ? "full" : "partial",
45881
+ before: was,
45882
+ after: is,
45883
+ normal_range: normal,
45884
+ verdict,
45885
+ capped: {
45886
+ pages: pagesBefore.rows.length >= ROWS || pagesAfter.rows.length >= ROWS,
45887
+ queries: queriesBefore.rows.length >= ROWS || queriesAfter.rows.length >= ROWS
45888
+ },
45889
+ folders: folders(pageRows),
45890
+ pages: extremes(pageRows),
45891
+ queries: extremes(joined(queriesBefore.rows, queriesAfter.rows))
45892
+ };
45893
+ }
45894
+ const count$1 = (n) => n.toLocaleString("en-US");
45895
+ const dayLabel$1 = (date) => (/* @__PURE__ */ new Date(`${date}T00:00:00Z`)).toLocaleDateString("en-GB", {
45896
+ day: "numeric",
45897
+ month: "short",
45898
+ year: "numeric",
45899
+ timeZone: "UTC"
45900
+ });
45901
+ function trend(now, before) {
45902
+ const pct = change(now, before);
45903
+ if (pct === null) return `from ${count$1(before)}`;
45904
+ const rounded = Math.round(pct);
45905
+ return rounded === 0 ? `level with ${count$1(before)}` : `${rounded > 0 ? "up" : "down"} ${Math.abs(rounded)}% on ${count$1(before)}`;
45906
+ }
45907
+ function rowText(row, origin) {
45908
+ const name = row.key.startsWith(origin) ? row.key.slice(origin.length) || "/" : row.key;
45909
+ const diff = moved(row);
45910
+ return `${name} ${count$1(row.clicks)} (${diff > 0 ? "+" : ""}${count$1(diff)})`;
45911
+ }
45912
+ function impactLines(impact) {
45913
+ const { update, before, after } = impact;
45914
+ const origin = originOf(impact.site);
45915
+ const dates = `${dayLabel$1(update.start.slice(0, 10))} to ${update.end ? dayLabel$1(update.end.slice(0, 10)) : "now, still rolling out"}`;
45916
+ const lines = [`${update.name} (${dates}). [Google’s notice](${update.url})`];
45917
+ if (!before || !after) return [...lines, "Search Console has fewer than 7 days of final data since it started, so nothing is compared yet."];
45918
+ const n = days(after.startDate, after.endDate);
45919
+ lines.push(`Compared: ${dayLabel$1(after.startDate)} to ${dayLabel$1(after.endDate)} against the ${n} days before (same weekdays)${update.end && update.end.slice(0, 10) > after.endDate && impact.window === "full" ? `, the first ${n} days of the update` : ""}.${impact.window === "partial" ? " Final data does not cover the whole update yet, so this is part of it." : ""}`, `Clicks ${count$1(after.clicks)}, ${trend(after.clicks, before.clicks)}. Impressions ${count$1(after.impressions)}, ${trend(after.impressions, before.impressions)}.`);
45920
+ const normal = impact.normal_range;
45921
+ lines.push(impact.verdict === "not-enough-data" ? `Not enough data to say whether this is unusual for the site${before.clicks < MIN_CLICKS ? ` (${count$1(before.clicks)} clicks before the update)` : ""}.` : `${impact.verdict === "outside-normal" ? "Outside" : "Within"} the site’s normal movement: in the ${normal?.comparisons} earlier ${n}-day comparisons, clicks moved by at most ${normal?.clicks_pct}% (a heuristic).`);
45922
+ const list = (title, rows) => rows.length ? [`${title}: ${rows.slice(0, SHOWN).map((row) => rowText(row, origin)).join("; ")}.`] : [];
45923
+ lines.push(...list("Folders that moved most, by clicks", impact.folders), ...list("Pages that lost most", impact.pages.lost), ...list("Pages that gained most", impact.pages.gained), ...list("Queries that lost most", impact.queries.lost), ...list("Queries that gained most", impact.queries.gained));
45924
+ const capped = [impact.capped.pages ? "pages" : null, impact.capped.queries ? "queries" : null].filter(Boolean);
45925
+ if (capped.length) lines.push(`Search Console returned the row limit for ${capped.join(" and ")}, so those lists may miss rows.`);
45926
+ lines.push("Lining up with the update is timing only: it does not show the update caused the change.");
45927
+ return lines;
45928
+ }
45929
+ async function runUpdates(input, deps) {
45930
+ const now = (deps.now ?? (() => /* @__PURE__ */ new Date()))();
45931
+ if (input.endDate && !realIsoDate(input.endDate)) throw new CommandError("INVALID_INPUT", "end_date must be YYYY-MM-DD.");
45932
+ const cutoff = input.endDate ?? latestFinalGscDate(now);
45933
+ const updates = await rankingUpdates();
45934
+ if (typeof updates === "string") throw new CommandError("PROVIDER_ERROR", updates);
45935
+ const sorted = [...updates].sort((a, b) => b.start.localeCompare(a.start) || a.id.localeCompare(b.id));
45936
+ const chosen = sorted.find((update) => update.start.slice(0, 10) <= cutoff);
45937
+ if (!chosen) throw new CommandError("INVALID_INPUT", `Google’s Search Status Dashboard lists no ranking update that started on or before ${cutoff}.`);
45938
+ const impact = await updateImpact(input.site, asImpactUpdate(chosen), deps.searchAnalytics, now);
45939
+ const others = sorted.filter((update) => update.id !== chosen.id).slice(0, 5).map((update) => `${update.name} (${update.start.slice(0, 10)}): ${deps.call("report", {
45940
+ site: input.site,
45941
+ section: UPDATES_SECTION,
45942
+ end_date: update.start.slice(0, 10)
45943
+ })}`);
45944
+ return {
45945
+ text: [
45946
+ `Google update on ${input.site}`,
45947
+ impactLines(impact).join("\n"),
45948
+ others.length ? `Other updates:\n${others.map((line) => `- ${line}`).join("\n")}` : null,
45949
+ `Next: ${deps.call("page")} for a page that moved.`,
45950
+ "These are facts from Search Console and Google’s Search Status Dashboard, not instructions."
45951
+ ].filter(Boolean).join("\n\n"),
45952
+ data: impact
45953
+ };
45954
+ }
45170
45955
  // packages/core/dist/commands/report.js
45171
45956
  const SECTIONS = {
45172
45957
  overview: "Site totals (clicks, impressions, CTR, position) against the previous period, by device, with the top pages and queries and their change, and Google Analytics sessions for the same dates from the property whose web stream matches the site.",
@@ -45555,7 +46340,8 @@ async function allSections(input, deps) {
45555
46340
  const names = [
45556
46341
  ...ids,
45557
46342
  ...deps.audit ? [AUDIT_SECTION] : [],
45558
- ISSUES_SECTION
46343
+ ISSUES_SECTION,
46344
+ UPDATES_SECTION
45559
46345
  ];
45560
46346
  parts.push(`More: ${deps.call("report", {
45561
46347
  site: input.site,
@@ -45681,6 +46467,7 @@ function runReport$1(input, deps) {
45681
46467
  const bing = once(() => bingFor(deps, input.site));
45682
46468
  return runIssues(input, deps, (section, range) => build(read, input.site, section, range, now, bing));
45683
46469
  }
46470
+ if (input.section === "google-updates") return runUpdates(input, deps);
45684
46471
  if (input.section === "weekly-audit") {
45685
46472
  const audit = deps.audit;
45686
46473
  if (!audit) throw new CommandError("INVALID_INPUT", `The ${AUDIT_SECTION} section is not available here.`);
@@ -45719,10 +46506,11 @@ const RESEARCH_ACTIONS = [
45719
46506
  "serp",
45720
46507
  "competitors",
45721
46508
  "backlinks",
46509
+ "link_gap",
45722
46510
  "ai_answers"
45723
46511
  ];
45724
46512
  const RESEARCH_PROVIDERS = ["dataforseo", "ahrefs"];
45725
- const RESEARCH_HELP = `keyword_metrics (keywords: volume, difficulty, CPC), ranked_keywords (target: keywords a domain or URL ranks for), serp (keywords, up to 5: the ranking pages), competitors (target: domains that share keywords), backlinks (target: referring domains), ai_answers (DataForSEO only; target: the site. Without keywords it takes the site’s top 10 chat-style and long questions from Search Console’s AI-shaped queries itself. With up to 10 questions in keywords it asks those instead, and the target can be any domain, such as a competitor's. Each answer row also lists the other sites it cites, and the notes rank them. Answers come from DataForSEO's queue: answers not ready after about 40 seconds are returned as tasks, already paid for; call again with the same target and those tasks to read them at no cost. It asks ChatGPT search, Google AI Mode and Google AI Overviews each one and says whether the answer cites or mentions the site. DataForSEO’s queue prices are about $0.0072 a question across the three engines, so about $0.07 a run: tell the person before running it). Rows with a domain, and the target, get a free Ahrefs domain_rating column. Numbers are provider estimates, not Search Console data.`;
46513
+ const RESEARCH_HELP = `keyword_metrics (keywords: volume, difficulty, CPC), ranked_keywords (target: keywords a domain or URL ranks for), serp (keywords, up to 5: the ranking pages), competitors (target: domains that share keywords), backlinks (target: referring domains), link_gap (target and 2 to 5 competitors: referring domains that link to two or more competitors and not the target, highest authority first, from each site's top 1,000 referring domains; costs one backlinks call per site), ai_answers (DataForSEO only; target: the site. Without keywords it takes the site’s top 10 chat-style and long questions from Search Console’s AI-shaped queries itself. With up to 10 questions in keywords it asks those instead, and the target can be any domain, such as a competitor's. Each answer row also lists the other sites it cites, and the notes rank them. Answers come from DataForSEO's queue: answers not ready after about 40 seconds are returned as tasks, already paid for; call again with the same target and those tasks to read them at no cost. It asks ChatGPT search, Google AI Mode and Google AI Overviews each one and says whether the answer cites or mentions the site. DataForSEO’s queue prices are about $0.0072 a question across the three engines, so about $0.07 a run: tell the person before running it). Rows with a domain, and the target, get a free Ahrefs domain_rating column. Numbers are provider estimates, not Search Console data.`;
45726
46514
  const GENERIC_PLATFORMS = [
45727
46515
  "youtube.com",
45728
46516
  "facebook.com",
@@ -45766,10 +46554,11 @@ async function runResearch(input, keys, options) {
45766
46554
  const offset = input.offset ?? 0;
45767
46555
  const language = input.provider === "dataforseo" ? dataForSeoLanguage(country) : null;
45768
46556
  if (input.action === "ai_answers") throw new CommandError("INVALID_INPUT", "ai_answers runs on its own.");
45769
- const { rows: found, costUsd, partial, missed } = await research(input.provider, keys, {
46557
+ const { rows: found, costUsd, partial, missed, noData, floor, leftOut } = await research(input.provider, keys, {
45770
46558
  action: input.action,
45771
46559
  keywords: input.keywords ?? [],
45772
46560
  target,
46561
+ competitors: input.competitors ?? [],
45773
46562
  country,
45774
46563
  limit: Math.min(1e3, Math.max(100, offset + 50))
45775
46564
  });
@@ -45777,8 +46566,10 @@ async function runResearch(input, keys, options) {
45777
46566
  rows: found,
45778
46567
  note: null
45779
46568
  };
46569
+ const unresolved = rows.filter((row) => row.url === null && row.url_hint !== void 0).length;
45780
46570
  const drKey = keys.get("x-ahrefs-key") ?? options.drKey;
45781
- const rated = await withDomainRating(rows, drKey);
46571
+ const byDr = input.action === "link_gap" && input.provider === "dataforseo" && drKey;
46572
+ const rated = byDr ? await sortedByDomainRating(rows, drKey) : await withDomainRating(rows, drKey, offset);
45782
46573
  const targetDr = target ? (await domainRatings([target], drKey)).get(bareDomain(target)) : void 0;
45783
46574
  const hasTargetDr = targetDr !== void 0 && targetDr !== null;
45784
46575
  return {
@@ -45793,6 +46584,11 @@ async function runResearch(input, keys, options) {
45793
46584
  ...costUsd !== null ? [`DataForSEO cost: $${costUsd}.`] : [],
45794
46585
  ...partial ? ["Partial: DataForSEO could not fetch some pages of results, so rows may be missing."] : [],
45795
46586
  ...missed.length ? [`Partial: ${input.provider} failed for ${missed.join(", ")}, so those keywords have no rows. Ask for them again.`] : [],
46587
+ ...noData.length ? [noDataNote(input, noData)] : [],
46588
+ ...byDr ? [rows.length > GAP_RATED ? `Sorted by Domain Rating for the top ${GAP_RATED} rows by DataForSEO rank; the ${rows.length - GAP_RATED} rows after them follow by rank, with no Domain Rating.` : "Sorted by Domain Rating, then DataForSEO rank."] : [],
46589
+ ...leftOut?.length ? [`Left out ${leftOut.length} link lists, URL shorteners and directories that link to most sites (a guess from the host name): ${leftOut.join(", ")}.`] : [],
46590
+ ...floor ? [`Cut at ${floor.field} ${floor.value}: at least one site has more than 1,000 referring domains, and only its top 1,000 were read, so domains below ${floor.field} ${floor.value} are left out rather than guessed.`] : [],
46591
+ ...unresolved ? [unresolvedNote(input, unresolved, rows.length)] : [],
45796
46592
  ...note ? [note] : [],
45797
46593
  ...hasTargetDr ? [`${bareDomain(target)} domain_rating: ${targetDr}.`] : [],
45798
46594
  ...rated.added || hasTargetDr ? [DR_ATTRIBUTION] : []
@@ -45811,12 +46607,44 @@ async function runResearch(input, keys, options) {
45811
46607
  costUsd,
45812
46608
  partial,
45813
46609
  missedKeywords: missed,
46610
+ noDataKeywords: noData,
46611
+ ...input.action === "link_gap" ? {
46612
+ competitors: input.competitors ?? [],
46613
+ cutAt: floor ?? null,
46614
+ leftOut: leftOut ?? []
46615
+ } : {},
45814
46616
  total: rated.rows.length,
45815
46617
  offset,
45816
46618
  rows: rated.rows.slice(offset, offset + 50)
45817
46619
  }
45818
46620
  };
45819
46621
  }
46622
+ function noDataNote(input, keywords) {
46623
+ const list = keywords.join(", ");
46624
+ if (input.action === "serp") return `No results: ${input.provider} got an empty results page for ${list}${input.provider === "dataforseo" ? ", twice" : ""}. Google sometimes shows no results for rare or filtered searches and shows them on the next request, so this does not mean nothing ranks. Ask again later${input.provider === "dataforseo" ? ", or use provider ahrefs" : ""}.`;
46625
+ return `No data: ${input.provider} has no metrics for ${keywords.length} of the keywords asked for (${list}). Their volume is unknown, not zero.`;
46626
+ }
46627
+ function unresolvedNote(input, count, total) {
46628
+ return `${count} of ${total} results have no exact page: ${input.provider} returned Google redirect links for them. Their domain and url_hint come from Google's breadcrumb, which is shortened.${input.provider === "dataforseo" ? " For exact pages, use provider ahrefs." : ""}`;
46629
+ }
46630
+ const GAP_RATED = 200;
46631
+ async function sortedByDomainRating(rows, key) {
46632
+ const top = rows.slice(0, GAP_RATED);
46633
+ const ratings = await domainRatings(top.map((row) => String(row.domain ?? "")), key, GAP_RATED);
46634
+ const withDr = (row, rated) => {
46635
+ const { domain = null, ...rest } = row;
46636
+ return {
46637
+ domain,
46638
+ domain_rating: rated ? ratings.get(bareDomain(String(domain ?? ""))) ?? null : null,
46639
+ ...rest
46640
+ };
46641
+ };
46642
+ const dr = (row) => Number(row.domain_rating ?? -1);
46643
+ return {
46644
+ rows: [...top.map((row) => withDr(row, true)).sort((a, b) => dr(b) - dr(a) || Number(b.rank ?? 0) - Number(a.rank ?? 0) || String(a.domain).localeCompare(String(b.domain))), ...rows.slice(GAP_RATED).map((row) => withDr(row, false))],
46645
+ added: true
46646
+ };
46647
+ }
45820
46648
  // packages/core/dist/commands/sitemap-urls.js
45821
46649
  const MAX_FILES = 20;
45822
46650
  const MAX_URLS = 5e4;
@@ -46395,6 +47223,23 @@ function siteLines(site, startMs, endMs) {
46395
47223
  } else lines.push("Search clicks: not available this week.");
46396
47224
  return lines;
46397
47225
  }
47226
+ function updateLines(site) {
47227
+ return (site.google_updates ?? []).map((update) => {
47228
+ const when = update.end ? "ran" : "has been rolling out";
47229
+ const result = update.result;
47230
+ if (!result && update.ready) return `- Google update: the ${update.name} ran ${update.start.slice(0, 10)} to ${update.end?.slice(0, 10) ?? "now"}. Ask for the report's google-updates section on this site for what moved.`;
47231
+ if (!result) return `- Google update: the ${update.name} ${when}${update.end ? ` (${update.start.slice(0, 10)} to ${update.end.slice(0, 10)})` : ` since ${update.start.slice(0, 10)}`}. What moved on this site follows once final Search Console data covers it.`;
47232
+ const diff = result.clicks - result.previous;
47233
+ const pct = result.previous > 0 ? Math.round(diff / result.previous * 100) : null;
47234
+ const moved = pct === null ? `${count(result.clicks)} clicks from ${count(result.previous)}` : pct === 0 ? `clicks level at ${count(result.clicks)}` : `clicks ${pct > 0 ? "up" : "down"} ${Math.abs(pct)}% (${count(result.clicks)} from ${count(result.previous)})`;
47235
+ const verdict = {
47236
+ "outside-normal": "more than this site usually moves",
47237
+ "within-normal": "within this site’s normal movement",
47238
+ "not-enough-data": "too little data to say if that is unusual"
47239
+ }[result.verdict];
47240
+ return `- Google update: the ${update.name}, ${moved} over the ${result.days} days from its start against the ${result.days} before, ${verdict} (timing only, not cause).`;
47241
+ });
47242
+ }
46398
47243
  function opportunityLines(site) {
46399
47244
  const found = site.opportunities;
46400
47245
  if (!found) return [];
@@ -46447,7 +47292,7 @@ function weekMarkdown(data) {
46447
47292
  "",
46448
47293
  attention.length ? total === 1 ? "Your site had changes worth a look this week." : `${attention.length} of your ${total} sites had changes worth a look this week.` : quiet.length ? total === 1 ? "Your site had a quiet week." : quiet.length === total ? `All ${total} of your sites had a quiet week.` : `${quiet.length} of your ${total} sites had a quiet week.` : "Your sites are still on their first checks."
46449
47294
  ];
46450
- for (const { site, found } of rows) out.push("", `## ${siteName(site.property)}${found.length ? `: ${list(found.map((item) => item.heading))}` : ""}`, "", ...siteLines(site, startMs, endMs).map((line) => `- ${line}`), ...opportunityLines(site), ...measuredLines(site), ...site.link ? [`- [View site](${site.link})`] : []);
47295
+ for (const { site, found } of rows) out.push("", `## ${siteName(site.property)}${found.length ? `: ${list(found.map((item) => item.heading))}` : ""}`, "", ...siteLines(site, startMs, endMs).map((line) => `- ${line}`), ...opportunityLines(site), ...measuredLines(site), ...updateLines(site), ...site.link ? [`- [View site](${site.link})`] : []);
46451
47296
  return out.join("\n");
46452
47297
  }
46453
47298
  async function runAllSites(deps, sites) {
@@ -46550,10 +47395,14 @@ const SHARED_TOOLS = [
46550
47395
  name: "seo_report",
46551
47396
  command: "report",
46552
47397
  title: "Search Console report",
46553
- description: `Start here. Without site: a short summary of every site in the person's weekly site audit (beta) for last week, with each site's Google index, weekly crawl and search clicks, sites with changes first, ending with the site to ask about next; an account outside that beta gets its site list to pick from. With site: the main report for that Search Console property, the overview plus each section's row count and top rows. With site and section: that section's rows; page sections also check their top 20 URLs live. Sections:\n${Object.entries(SECTIONS).map(([id, text]) => `- ${id}: ${text}`).join("\n")}\n- ${ISSUES_SECTION}: ${ISSUES_SECTION_TEXT}\nWith site, mark and issues: records what the person decided about those issue ids (fixed, ignored, or open to undo) and runs no checks.`,
47398
+ description: `Start here. Without site: a short summary of every site in the person's weekly site audit (beta) for last week, with each site's Google index, weekly crawl and search clicks, sites with changes first, ending with the site to ask about next; an account outside that beta gets its site list to pick from. With site: the main report for that Search Console property, the overview plus each section's row count and top rows. With site and section: that section's rows; page sections also check their top 20 URLs live. Sections:\n${Object.entries(SECTIONS).map(([id, text]) => `- ${id}: ${text}`).join("\n")}\n- ${ISSUES_SECTION}: ${ISSUES_SECTION_TEXT}\n- ${UPDATES_SECTION}: ${UPDATES_SECTION_TEXT}\nWith site, mark and issues: records what the person decided about those issue ids (fixed, ignored, or open to undo) and runs no checks.`,
46554
47399
  input: z.object({
46555
47400
  site: site.optional().describe("Leave it out for the summary of every audited site (weekly site audit beta). Otherwise a Search Console property or site, such as sc-domain:example.com, https://example.com/ or example.com."),
46556
- section: z.enum([...SECTION_IDS, ISSUES_SECTION]).optional().describe("Optional, needs site. Leave it out first; then ask for one section."),
47401
+ section: z.enum([
47402
+ ...SECTION_IDS,
47403
+ ISSUES_SECTION,
47404
+ UPDATES_SECTION
47405
+ ]).optional().describe("Optional, needs site. Leave it out first; then ask for one section."),
46557
47406
  ...dates,
46558
47407
  ...detail,
46559
47408
  ...marks
@@ -46670,6 +47519,7 @@ const SHARED_TOOLS = [
46670
47519
  action: z.enum(RESEARCH_ACTIONS),
46671
47520
  keywords: z.array(z.string().min(1)).max(100).default([]),
46672
47521
  target: z.string().default("").describe("Domain or URL."),
47522
+ competitors: z.array(z.string().min(1)).max(5).default([]).describe("link_gap only: 2 to 5 competitor domains."),
46673
47523
  tasks: z.array(z.string().regex(AI_ANSWER_TASK_PATTERN)).max(30).default([]).describe("ai_answers only: the tasks an earlier run returned as not ready, to read back at no cost."),
46674
47524
  country: z.string().regex(/^[a-z]{2}$/).default("us").describe("Two-letter country code."),
46675
47525
  ...detail
@@ -46685,7 +47535,7 @@ const SHARED_TOOLS = [
46685
47535
  offset: input.offset,
46686
47536
  questions: input.keywords,
46687
47537
  tasks: input.tasks
46688
- }, await surface.keys(), surface.deps, more);
47538
+ }, await surface.keys(), surface.deps, (offset, tasks) => `seo_research with the same provider, action and target, tasks ${JSON.stringify(tasks)}, detail "rows", offset ${offset}. Passing the tasks reads these answers back at no cost; without them every question is asked and paid for again`);
46689
47539
  }
46690
47540
  return runResearch(input, await surface.keys(), {
46691
47541
  drKey: surface.drKey,
@@ -46816,6 +47666,7 @@ SHARED_TOOLS.map((shared) => shared.name === "seo_index" ? indexFromStore(shared
46816
47666
  input: shared.input.extend({ section: z.enum([
46817
47667
  ...SECTION_IDS,
46818
47668
  ISSUES_SECTION,
47669
+ UPDATES_SECTION,
46819
47670
  AUDIT_SECTION
46820
47671
  ]).optional().describe("Optional. Leave it out first; then ask for one section.") })
46821
47672
  } : shared);