seo 0.15.2 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
2
- import { n as version, t as load } from "./esm-Dt4JJZ0P.js";
2
+ import { n as version, t as load } from "./esm-Cvmg0qhZ.js";
3
3
  import { SEO_PROVIDER_API_VERSION } from "./provider-sdk.js";
4
4
  import crypto, { createHash, randomUUID, sign } from "node:crypto";
5
5
  import { z } from "zod";
@@ -343,6 +343,24 @@ const RULE_DEFINITIONS = [
343
343
  suggestedCommands: ["seo crawl <url> --max-pages 1 --json"]
344
344
  }
345
345
  },
346
+ {
347
+ id: "crawl_single_page_reach",
348
+ title: "Crawl could reach only the start page",
349
+ category: "links",
350
+ defaultSeverity: "high",
351
+ whyItMatters: "The start page had no crawlable internal links and no sitemap returned URLs, so the crawl checked one page. Every other finding in this report covers that page only. The page probably builds its links with JavaScript.",
352
+ howToFix: "Rerun with `--rendering on` so the crawler reads the links a browser sees, or pass the pages to check with `--mode list`. For crawlers that do not run JavaScript, put plain `<a href>` links in the initial HTML and publish a sitemap.",
353
+ impactIfIgnored: "Crawlers and audits that do not run JavaScript can find only this page, so the rest of the site stays unchecked and harder to discover.",
354
+ howToVerify: "Rerun the crawl and confirm it retains more than one page and the start page reports internal links.",
355
+ agentHints: {
356
+ evidenceFields: [
357
+ "page.outgoingInternalCount",
358
+ "report.sitemapDiscovery.urlsReturned",
359
+ "page.fetchDiagnostics.rendering"
360
+ ],
361
+ suggestedCommands: ["seo crawl <url> --rendering on --json", "seo crawl --mode list --urls <url>,<url> --json"]
362
+ }
363
+ },
346
364
  {
347
365
  id: "soft_404",
348
366
  title: "Unknown URLs return a successful response",
@@ -351,7 +369,7 @@ const RULE_DEFINITIONS = [
351
369
  whyItMatters: "Known-nonexistent URLs returned a successful response or redirected to another successful page. Search engines and users cannot reliably distinguish missing content from a real page.",
352
370
  howToFix: "Return 404 or 410 for missing URLs. If a replacement exists, redirect only the specific retired URL to its closest relevant replacement rather than sending every unknown path to a generic page.",
353
371
  impactIfIgnored: "Missing URLs can remain in crawl queues, waste crawl attention, and create confusing search or monitoring evidence.",
354
- howToVerify: "Request several known-nonexistent paths and confirm each returns 404 or 410 without redirecting to a generic successful page.",
372
+ howToVerify: "Request several made-up paths, including one at the site root and one nested path, and confirm each returns 404 or 410 without redirecting to a generic successful page.",
355
373
  agentHints: {
356
374
  evidenceFields: ["report.siteChecks.soft404"],
357
375
  suggestedCommands: ["seo crawl <url> --json"]
@@ -1803,7 +1821,7 @@ function day$3(value) {
1803
1821
  const date = legacy ? new Date(Number(legacy[1])) : new Date(value);
1804
1822
  return Number.isNaN(date.getTime()) ? null : date.toISOString().slice(0, 10);
1805
1823
  }
1806
- const count$3 = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
1824
+ const count$4 = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
1807
1825
  function list$2(data, label) {
1808
1826
  if (!Array.isArray(data)) throw new SeoError("OPTIONAL_PROVIDER_UNAVAILABLE", `Bing Webmaster returned an invalid ${label}.`);
1809
1827
  return data.filter((item) => Boolean(item) && typeof item === "object");
@@ -1819,8 +1837,8 @@ async function bingTraffic(call, site) {
1819
1837
  let invalidRows = 0;
1820
1838
  for (const row of list$2(await call("GetRankAndTrafficStats", { siteUrl: site }), "traffic list")) {
1821
1839
  const date = day$3(row.Date);
1822
- const clicks = count$3(row.Clicks);
1823
- const impressions = count$3(row.Impressions);
1840
+ const clicks = count$4(row.Clicks);
1841
+ const impressions = count$4(row.Impressions);
1824
1842
  if (!date || clicks === null || impressions === null) {
1825
1843
  invalidRows += 1;
1826
1844
  continue;
@@ -1843,9 +1861,9 @@ async function keyed(call, method, params) {
1843
1861
  for (const row of list$2(await call(method, params), "row list")) {
1844
1862
  const date = day$3(row.Date);
1845
1863
  const key = typeof row.Query === "string" ? row.Query.trim() : "";
1846
- const clicks = count$3(row.Clicks);
1847
- const impressions = count$3(row.Impressions);
1848
- const position = count$3(row.AvgImpressionPosition);
1864
+ const clicks = count$4(row.Clicks);
1865
+ const impressions = count$4(row.Impressions);
1866
+ const position = count$4(row.AvgImpressionPosition);
1849
1867
  if (!date || !key || clicks === null || impressions === null) {
1850
1868
  invalidRows += 1;
1851
1869
  continue;
@@ -2377,6 +2395,14 @@ function sharedReader(read, redirects = /* @__PURE__ */ new Map()) {
2377
2395
  };
2378
2396
  }
2379
2397
  const MAX_STATUS_BYTES = 10485760;
2398
+ async function rankingUpdates() {
2399
+ const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? readJson(response, "Google Search Status", { maxBytes: MAX_STATUS_BYTES }) : null).catch(() => null);
2400
+ if (!Array.isArray(incidents)) return "Google Search Status could not be read.";
2401
+ return searchUpdatesFromIncidents(withRankingHistory(incidents), {
2402
+ product: "Ranking",
2403
+ limit: Number.POSITIVE_INFINITY
2404
+ });
2405
+ }
2380
2406
  async function googleUpdates(window) {
2381
2407
  if (!window) return [];
2382
2408
  const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? readJson(response, "Google Search Status", { maxBytes: MAX_STATUS_BYTES }) : null).catch(() => null);
@@ -7702,17 +7728,15 @@ function recordHostFetch(input) {
7702
7728
  const cfg = input.rate.backpressure;
7703
7729
  health.durations.push(input.durationMs);
7704
7730
  if (health.durations.length > 20) health.durations.shift();
7705
- const blocked = [
7706
- 401,
7707
- 403,
7708
- 429
7709
- ].includes(input.status);
7710
- const errored = input.status >= 500;
7731
+ const rateLimited = input.status === 429 || input.status === 503 && input.retryAfterMs !== void 0;
7732
+ const blocked = [401, 403].includes(input.status);
7733
+ const errored = input.status >= 500 && !rateLimited;
7711
7734
  const slow = input.durationMs >= cfg.slowMs;
7712
7735
  health.consecutiveBlocked = blocked ? health.consecutiveBlocked + 1 : 0;
7713
7736
  health.consecutiveErrors = errored ? health.consecutiveErrors + 1 : 0;
7714
7737
  health.consecutiveSlow = slow ? health.consecutiveSlow + 1 : 0;
7715
- if (!blocked && !errored && !slow) health.reason = void 0;
7738
+ if (!blocked && !errored && !slow && !rateLimited) health.reason = void 0;
7739
+ if (rateLimited) health.reason = `origin rate limited the crawler (HTTP ${input.status})`;
7716
7740
  const retryDelay = input.retryAfterMs === void 0 ? 0 : Math.min(input.retryAfterMs, cfg.retryAfterCapMs);
7717
7741
  if (retryDelay > 0) {
7718
7742
  health.reason = `origin sent Retry-After (${Math.round(retryDelay / 1e3)}s)`;
@@ -8113,6 +8137,56 @@ function systemBrowserCandidates(input) {
8113
8137
  product: path.includes("google-chrome") ? "Google Chrome" : "Chromium"
8114
8138
  }));
8115
8139
  }
8140
+ function listDirectorySafely(path) {
8141
+ try {
8142
+ return readdirSync(path);
8143
+ } catch {
8144
+ return [];
8145
+ }
8146
+ }
8147
+ function versionParts(name) {
8148
+ return (name.match(/\d+/g) ?? []).map(Number);
8149
+ }
8150
+ function newestFirst(left, right) {
8151
+ const a = versionParts(left);
8152
+ const b = versionParts(right);
8153
+ for (let index = 0; index < Math.max(a.length, b.length); index += 1) {
8154
+ const difference = (b[index] ?? -1) - (a[index] ?? -1);
8155
+ if (difference) return difference;
8156
+ }
8157
+ return left < right ? -1 : left > right ? 1 : 0;
8158
+ }
8159
+ const MAX_CACHED_BROWSER_VERSIONS = 20;
8160
+ function versionDirectories(root, prefix, listDirectory) {
8161
+ return listDirectory(root).filter((name) => name.startsWith(prefix)).sort(newestFirst).slice(0, MAX_CACHED_BROWSER_VERSIONS).map((name) => join(root, name));
8162
+ }
8163
+ const CHROME_FOR_TESTING_APP = join("Google Chrome for Testing.app", "Contents", "MacOS", "Google Chrome for Testing");
8164
+ /**
8165
+ * Browsers that other local tools already downloaded: agent-browser keeps
8166
+ * Chrome for Testing builds and Playwright keeps one Chromium per revision.
8167
+ */
8168
+ function cachedBrowserCandidates(input) {
8169
+ const agentBrowserExecutables = input.platform === "darwin" ? [CHROME_FOR_TESTING_APP] : input.platform === "win32" ? ["chrome.exe"] : ["chrome"];
8170
+ const agentBrowser = versionDirectories(join(input.homeDirectory, ".agent-browser", "browsers"), "chrome-", input.listDirectory).flatMap((directory) => agentBrowserExecutables.map((executable) => ({
8171
+ path: join(directory, executable),
8172
+ source: "agent-browser-cache",
8173
+ product: "agent-browser Chrome for Testing"
8174
+ })));
8175
+ const configuredRoot = input.environment.PLAYWRIGHT_BROWSERS_PATH?.trim();
8176
+ const playwrightRoot = configuredRoot && configuredRoot !== "0" ? configuredRoot : input.platform === "darwin" ? join(input.homeDirectory, "Library", "Caches", "ms-playwright") : input.platform === "win32" ? join(input.environment.LOCALAPPDATA ?? join(input.homeDirectory, "AppData", "Local"), "ms-playwright") : join(input.environment.XDG_CACHE_HOME ?? join(input.homeDirectory, ".cache"), "ms-playwright");
8177
+ const playwrightExecutables = input.platform === "darwin" ? [
8178
+ join("chrome-mac-arm64", CHROME_FOR_TESTING_APP),
8179
+ join("chrome-mac-x64", CHROME_FOR_TESTING_APP),
8180
+ join("chrome-mac", CHROME_FOR_TESTING_APP),
8181
+ join("chrome-mac", "Chromium.app", "Contents", "MacOS", "Chromium")
8182
+ ] : input.platform === "win32" ? [join("chrome-win64", "chrome.exe"), join("chrome-win", "chrome.exe")] : [join("chrome-linux64", "chrome"), join("chrome-linux", "chrome")];
8183
+ const playwright = versionDirectories(playwrightRoot, "chromium-", input.listDirectory).flatMap((directory) => playwrightExecutables.map((executable) => ({
8184
+ path: join(directory, executable),
8185
+ source: "playwright-cache",
8186
+ product: "Playwright Chromium"
8187
+ })));
8188
+ return [...agentBrowser, ...playwright];
8189
+ }
8116
8190
  /**
8117
8191
  * Resolve a browser without downloading one during normal package installation.
8118
8192
  * The explicit path wins so CI and managed workstations can select their own
@@ -8144,11 +8218,23 @@ function resolveBrowserExecutable(playwrightExecutablePath, dependencies = {}) {
8144
8218
  product: "Playwright Chromium"
8145
8219
  }
8146
8220
  };
8147
- const browser = systemBrowserCandidates({
8148
- platform: dependencies.platform ?? process.platform,
8149
- homeDirectory: dependencies.homeDirectory ?? homedir(),
8221
+ const platform = dependencies.platform ?? process.platform;
8222
+ const homeDirectory = dependencies.homeDirectory ?? homedir();
8223
+ const systemBrowser = systemBrowserCandidates({
8224
+ platform,
8225
+ homeDirectory,
8150
8226
  environment
8151
8227
  }).find((candidate) => fileExists(candidate.path));
8228
+ if (systemBrowser) return {
8229
+ status: "available",
8230
+ browser: systemBrowser
8231
+ };
8232
+ const browser = cachedBrowserCandidates({
8233
+ platform,
8234
+ homeDirectory,
8235
+ environment,
8236
+ listDirectory: dependencies.listDirectory ?? listDirectorySafely
8237
+ }).find((candidate) => fileExists(candidate.path));
8152
8238
  return browser ? {
8153
8239
  status: "available",
8154
8240
  browser
@@ -11200,6 +11286,17 @@ function isAuditableHtmlPage(page) {
11200
11286
  const contentAuditAllowed = page.contentAuditAllowed ?? (!page.blocked && page.robotsTxt?.allowed !== false);
11201
11287
  return isSuccessfulResponse(page) && isHtmlPage(page) && !isRedirectedPage(page) && contentAuditAllowed;
11202
11288
  }
11289
+ /**
11290
+ * Cloudflare serves its own endpoints under `/cdn-cgi/`, such as the email
11291
+ * obfuscation link `/cdn-cgi/l/email-protection`. They are not site pages.
11292
+ */
11293
+ function isCdnUtilityUrl(value) {
11294
+ try {
11295
+ return new URL(value).pathname.startsWith("/cdn-cgi/");
11296
+ } catch {
11297
+ return false;
11298
+ }
11299
+ }
11203
11300
  // packages/core/src/analyze/crawler/audit.ts
11204
11301
  function sameUrl(a, b) {
11205
11302
  if (!a || !b) return false;
@@ -11312,38 +11409,61 @@ function isValuablePage(page) {
11312
11409
  function normalizedMetadata(value) {
11313
11410
  return value?.trim().replace(/\s+/g, " ").toLowerCase() || void 0;
11314
11411
  }
11315
- function metadataDuplicates(pages, read) {
11316
- const counts = /* @__PURE__ */ new Map();
11412
+ /**
11413
+ * A page whose canonical names another crawled page in its group is already
11414
+ * handled. A group counts only when two or more of its pages are
11415
+ * self-canonical or declare no canonical, and only those pages are reported.
11416
+ */
11417
+ function duplicateGroups(pages, read) {
11418
+ const grouped = /* @__PURE__ */ new Map();
11317
11419
  for (const page of pages) {
11318
11420
  if (!isAuditableHtmlPage(page)) continue;
11319
- const raw = read(page)?.trim();
11320
- const key = normalizedMetadata(raw);
11321
- if (!key || !raw) continue;
11322
- const existing = counts.get(key) ?? {
11323
- count: 0,
11324
- sampleUrls: [],
11325
- value: raw
11421
+ const entry = read(page);
11422
+ if (!entry) continue;
11423
+ const group = grouped.get(entry.key) ?? {
11424
+ value: entry.value,
11425
+ pages: []
11326
11426
  };
11327
- existing.count += 1;
11328
- if (existing.sampleUrls.length < 10) existing.sampleUrls.push(page.url);
11329
- counts.set(key, existing);
11427
+ group.pages.push(page);
11428
+ grouped.set(entry.key, group);
11429
+ }
11430
+ const duplicates = /* @__PURE__ */ new Map();
11431
+ for (const [key, group] of grouped) {
11432
+ if (group.pages.length < 2) continue;
11433
+ const groupUrls = /* @__PURE__ */ new Set();
11434
+ for (const page of group.pages) for (const value of [page.url, page.finalUrl]) {
11435
+ const url = urlKey(value);
11436
+ if (url) groupUrls.add(url);
11437
+ }
11438
+ const counted = group.pages.filter((page) => {
11439
+ const canonical = urlKey(page.canonical);
11440
+ return !canonical || canonical === urlKey(page.url) || canonical === urlKey(page.finalUrl) || !groupUrls.has(canonical);
11441
+ });
11442
+ if (counted.length < 2) continue;
11443
+ duplicates.set(key, {
11444
+ count: counted.length,
11445
+ sampleUrls: counted.slice(0, 10).map((page) => page.url),
11446
+ value: group.value,
11447
+ urls: new Set(counted.map((page) => page.url))
11448
+ });
11330
11449
  }
11331
- return new Map([...counts].filter(([, value]) => value.count > 1));
11450
+ return duplicates;
11451
+ }
11452
+ function metadataDuplicates(pages, read) {
11453
+ return duplicateGroups(pages, (page) => {
11454
+ const value = read(page)?.trim();
11455
+ const key = normalizedMetadata(value);
11456
+ return key && value ? {
11457
+ key,
11458
+ value
11459
+ } : void 0;
11460
+ });
11332
11461
  }
11333
11462
  function contentDuplicates(pages) {
11334
- const counts = /* @__PURE__ */ new Map();
11335
- for (const page of pages) {
11336
- if (!isAuditableHtmlPage(page)) continue;
11337
- if (page.wordCount < 20 || !page.mainContentHash) continue;
11338
- const existing = counts.get(page.mainContentHash) ?? {
11339
- count: 0,
11340
- sampleUrls: []
11341
- };
11342
- existing.count += 1;
11343
- if (existing.sampleUrls.length < 10) existing.sampleUrls.push(page.url);
11344
- counts.set(page.mainContentHash, existing);
11345
- }
11346
- return new Map([...counts].filter(([, value]) => value.count > 1));
11463
+ return duplicateGroups(pages, (page) => page.wordCount >= 20 && page.mainContentHash ? {
11464
+ key: page.mainContentHash,
11465
+ value: page.mainContentHash
11466
+ } : void 0);
11347
11467
  }
11348
11468
  const QUERY_STOP_WORDS = /* @__PURE__ */ new Set([
11349
11469
  "a",
@@ -11446,7 +11566,7 @@ function auditCrawlPages(pages, opts = {}) {
11446
11566
  if (page.accessBlock) continue;
11447
11567
  if (page.status >= 500) {
11448
11568
  issues.push(issue$1("server_error", page, String(page.status), { status: page.status }));
11449
- if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
11569
+ if ((page.internalInlinkCount ?? 0) > 0 && !isCdnUtilityUrl(page.url)) issues.push(issue$1("broken_internal_link", page, String(page.status), {
11450
11570
  status: page.status,
11451
11571
  internalInlinkCount: page.internalInlinkCount
11452
11572
  }));
@@ -11454,7 +11574,7 @@ function auditCrawlPages(pages, opts = {}) {
11454
11574
  }
11455
11575
  if (page.status >= 400) {
11456
11576
  issues.push(issue$1("client_error", page, String(page.status), { status: page.status }));
11457
- if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
11577
+ if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0 && !isCdnUtilityUrl(page.url)) issues.push(issue$1("broken_internal_link", page, String(page.status), {
11458
11578
  status: page.status,
11459
11579
  internalInlinkCount: page.internalInlinkCount
11460
11580
  }));
@@ -11553,7 +11673,7 @@ function auditCrawlPages(pages, opts = {}) {
11553
11673
  title,
11554
11674
  ...titleWidth
11555
11675
  }));
11556
- if (title && duplicateTitle) issues.push(issue$1("title_duplicate", page, duplicateTitle.value, {
11676
+ if (title && duplicateTitle?.urls.has(page.url)) issues.push(issue$1("title_duplicate", page, duplicateTitle.value, {
11557
11677
  title,
11558
11678
  duplicateCount: duplicateTitle.count,
11559
11679
  sampleUrls: duplicateTitle.sampleUrls
@@ -11561,7 +11681,7 @@ function auditCrawlPages(pages, opts = {}) {
11561
11681
  const metaDescription = page.metaDescription?.trim();
11562
11682
  const duplicateDescription = duplicateDescriptions.get(normalizedMetadata(metaDescription) ?? "");
11563
11683
  if (!metaDescription) issues.push(issue$1("missing_meta_description", page));
11564
- if (metaDescription && duplicateDescription) issues.push(issue$1("meta_description_duplicate", page, duplicateDescription.value, {
11684
+ if (metaDescription && duplicateDescription?.urls.has(page.url)) issues.push(issue$1("meta_description_duplicate", page, duplicateDescription.value, {
11565
11685
  metaDescription,
11566
11686
  duplicateCount: duplicateDescription.count,
11567
11687
  sampleUrls: duplicateDescription.sampleUrls
@@ -11609,7 +11729,7 @@ function auditCrawlPages(pages, opts = {}) {
11609
11729
  finalUrl: page.finalUrl
11610
11730
  }));
11611
11731
  const contentDuplicate = page.mainContentHash ? duplicateContent.get(page.mainContentHash) : void 0;
11612
- if (contentDuplicate) issues.push(issue$1("duplicate_content", page, "Duplicate main content", {
11732
+ if (contentDuplicate?.urls.has(page.url)) issues.push(issue$1("duplicate_content", page, "Duplicate main content", {
11613
11733
  mainContentHash: page.mainContentHash,
11614
11734
  duplicateCount: contentDuplicate.count,
11615
11735
  sampleUrls: contentDuplicate.sampleUrls
@@ -11677,6 +11797,7 @@ function auditCrawlPages(pages, opts = {}) {
11677
11797
  // packages/core/src/analyze/crawler/crawl-skip-reasons.ts
11678
11798
  const CRAWL_SKIP_REASONS = [
11679
11799
  "asset-url",
11800
+ "cdn-utility-url",
11680
11801
  "configured-exclusion",
11681
11802
  "duplicate-normalization",
11682
11803
  "invalid-url",
@@ -11684,12 +11805,14 @@ const CRAWL_SKIP_REASONS = [
11684
11805
  "off-origin",
11685
11806
  "origin-backpressure",
11686
11807
  "queue-safety-limit",
11808
+ "query-variant-limit",
11687
11809
  "robots-disallowed",
11688
11810
  "robots-uncertain",
11689
11811
  "legacy-unclassified"
11690
11812
  ];
11691
11813
  const SKIP_IMPACT = {
11692
11814
  "asset-url": "non-impacting",
11815
+ "cdn-utility-url": "non-impacting",
11693
11816
  "configured-exclusion": "non-impacting",
11694
11817
  "duplicate-normalization": "non-impacting",
11695
11818
  "invalid-url": "non-impacting",
@@ -11697,6 +11820,7 @@ const SKIP_IMPACT = {
11697
11820
  "off-origin": "non-impacting",
11698
11821
  "origin-backpressure": "coverage-affecting",
11699
11822
  "queue-safety-limit": "coverage-affecting",
11823
+ "query-variant-limit": "coverage-affecting",
11700
11824
  "robots-disallowed": "coverage-affecting",
11701
11825
  "robots-uncertain": "coverage-affecting",
11702
11826
  "legacy-unclassified": "coverage-affecting"
@@ -12315,64 +12439,169 @@ function refreshPrioritiesCsvFiles(report) {
12315
12439
  ...prefixCsvFiles("diagnosis", diagnoseCsvFiles(report.output.diagnosis))
12316
12440
  ]);
12317
12441
  }
12318
- // packages/core/src/analyze/crawler/top-fixes.ts
12319
- const SEVERITY_SCORE = {
12320
- high: 5e3,
12321
- medium: 1200,
12322
- low: 100
12323
- };
12324
- const EFFORT_BY_CATEGORY = {
12325
- canonical: "medium",
12326
- content: "high",
12327
- agents: "low",
12328
- headings: "low",
12329
- images: "medium",
12330
- indexability: "medium",
12331
- international: "low",
12332
- metadata: "low",
12333
- mobile: "low",
12334
- response: "medium",
12335
- security: "medium",
12336
- social: "low",
12337
- "structured-data": "medium"
12338
- };
12339
- const EFFORT_SCORE = {
12340
- low: 60,
12341
- medium: 30,
12342
- high: 0
12343
- };
12344
- const AFFECTED_URL_WEIGHT = {
12345
- high: 25,
12346
- medium: 15,
12347
- low: 2
12348
- };
12349
- const VALUE_WEIGHT = {
12350
- high: 1,
12351
- medium: .75,
12352
- low: .15
12353
- };
12354
- function matchesPattern(pattern, value) {
12355
- if (!pattern.includes("*")) return value.includes(pattern);
12356
- const parts = pattern.split("*");
12357
- let position = 0;
12358
- for (const part of parts) {
12359
- if (!part) continue;
12360
- const found = value.slice(position).indexOf(part);
12361
- if (found === -1) return false;
12362
- position += found + part.length;
12363
- }
12364
- return true;
12442
+ // packages/core/src/analyze/crawler/crawl-url-queue.ts
12443
+ function before(left, right) {
12444
+ return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
12365
12445
  }
12366
- function filteredIssues(report, filters) {
12367
- return report.issues.filter((issue) => {
12368
- if (filters.category && issue.category !== filters.category) return false;
12369
- if (filters.severity && issue.severity !== filters.severity) return false;
12370
- if (filters.url && !matchesPattern(filters.url, issue.url)) return false;
12371
- if ((filters.recommendation ?? "fix") !== recommendationForRule(issue.ruleId)) return false;
12446
+ var CrawlUrlQueue = class {
12447
+ #items = [];
12448
+ #positions = /* @__PURE__ */ new Map();
12449
+ get size() {
12450
+ return this.#items.length;
12451
+ }
12452
+ has(url) {
12453
+ return this.#positions.has(url);
12454
+ }
12455
+ push(item) {
12456
+ if (this.has(item.url)) throw new Error(`URL is already queued: ${item.url}`);
12457
+ const index = this.#items.length;
12458
+ this.#items.push(item);
12459
+ this.#positions.set(item.url, index);
12460
+ this.#bubbleUp(index);
12461
+ }
12462
+ decreaseDepth(url, depth) {
12463
+ const index = this.#positions.get(url);
12464
+ if (index === void 0) return false;
12465
+ const item = this.#items[index];
12466
+ if (!item || depth >= item.depth) return false;
12467
+ item.depth = depth;
12468
+ this.#bubbleUp(index);
12372
12469
  return true;
12373
- });
12470
+ }
12471
+ take() {
12472
+ const first = this.#items[0];
12473
+ if (!first) return void 0;
12474
+ const last = this.#items.pop();
12475
+ this.#positions.delete(first.url);
12476
+ if (last && last !== first) {
12477
+ this.#items[0] = last;
12478
+ this.#positions.set(last.url, 0);
12479
+ this.#bubbleDown(0);
12480
+ }
12481
+ return first;
12482
+ }
12483
+ #swap(left, right) {
12484
+ const leftItem = this.#items[left];
12485
+ const rightItem = this.#items[right];
12486
+ if (!leftItem || !rightItem) return;
12487
+ this.#items[left] = rightItem;
12488
+ this.#items[right] = leftItem;
12489
+ this.#positions.set(rightItem.url, left);
12490
+ this.#positions.set(leftItem.url, right);
12491
+ }
12492
+ #bubbleUp(start) {
12493
+ let index = start;
12494
+ while (index > 0) {
12495
+ const parent = Math.floor((index - 1) / 2);
12496
+ const item = this.#items[index];
12497
+ const parentItem = this.#items[parent];
12498
+ if (!item || !parentItem || !before(item, parentItem)) break;
12499
+ this.#swap(index, parent);
12500
+ index = parent;
12501
+ }
12502
+ }
12503
+ #bubbleDown(start) {
12504
+ let index = start;
12505
+ while (true) {
12506
+ const left = index * 2 + 1;
12507
+ const right = left + 1;
12508
+ let next = index;
12509
+ const item = this.#items[next];
12510
+ const leftItem = this.#items[left];
12511
+ const rightItem = this.#items[right];
12512
+ if (leftItem && item && before(leftItem, item)) next = left;
12513
+ const nextItem = this.#items[next];
12514
+ if (rightItem && nextItem && before(rightItem, nextItem)) next = right;
12515
+ if (next === index) return;
12516
+ this.#swap(index, next);
12517
+ index = next;
12518
+ }
12519
+ }
12520
+ };
12521
+ // packages/core/src/analyze/crawler/report.ts
12522
+ const MAX_CRAWL_PAGES = 1e4;
12523
+ const MAX_CRAWL_DEPTH = 64;
12524
+ const MAX_CRAWL_CONCURRENCY = 16;
12525
+ const MAX_CRAWL_TIMEOUT_MS = 12e4;
12526
+ const SENSITIVE_FIELD_NAMES = /* @__PURE__ */ new Set([
12527
+ "authorization",
12528
+ "cookie",
12529
+ "proxy-authorization",
12530
+ "set-cookie",
12531
+ "www-authenticate",
12532
+ "x-api-key",
12533
+ "api-key",
12534
+ "access-token",
12535
+ "refresh-token"
12536
+ ]);
12537
+ const SENSITIVE_PARAM_PATTERN = /^(access_?token|auth|api_?key|client_?secret|code|jwt|key|password|refresh_?token|secret|session|sig|signature|token)$/i;
12538
+ function uniqueSorted(values = []) {
12539
+ return [...new Set(values.filter(Boolean))].sort();
12374
12540
  }
12375
- function groupIssues(issues) {
12541
+ function normalizeFetchRate(input, strategy) {
12542
+ const defaultConcurrency = strategy === "health" ? 4 : 8;
12543
+ return {
12544
+ concurrency: input.fetchRate?.concurrency ?? input.concurrency ?? defaultConcurrency,
12545
+ ...input.fetchRate?.intervalCap !== void 0 ? { intervalCap: input.fetchRate.intervalCap } : {},
12546
+ ...input.fetchRate?.intervalMs !== void 0 ? { intervalMs: input.fetchRate.intervalMs } : {},
12547
+ ...input.fetchRate?.backpressure ? { backpressure: input.fetchRate.backpressure } : {}
12548
+ };
12549
+ }
12550
+ function normalizeCrawlConfig(input) {
12551
+ const strategy = input.strategy ?? "full";
12552
+ const concurrency = input.concurrency ?? (strategy === "health" ? 4 : 8);
12553
+ return {
12554
+ url: new URL(input.url).toString(),
12555
+ mode: input.mode ?? (strategy === "health" ? "sitemap" : "site"),
12556
+ strategy,
12557
+ ...input.sitemapUrl ? { sitemapUrl: new URL(input.sitemapUrl).toString() } : {},
12558
+ urls: uniqueSorted(input.urls ?? []),
12559
+ maxPages: input.maxPages ?? 500,
12560
+ maxDepth: input.maxDepth ?? 16,
12561
+ concurrency,
12562
+ timeoutMs: input.timeoutMs ?? 2e4,
12563
+ include: uniqueSorted(input.include ?? []),
12564
+ exclude: uniqueSorted(input.exclude ?? []),
12565
+ respectRobots: input.respectRobots ?? true,
12566
+ useSitemap: input.useSitemap ?? true,
12567
+ checkExternal: strategy === "health" ? false : input.checkExternal ?? true,
12568
+ js: strategy === "health" ? "off" : normalizeJavaScriptRenderingMode(input.js),
12569
+ refresh: strategy === "health" ? true : input.refresh ?? false,
12570
+ fetchRate: normalizeFetchRate(input, strategy)
12571
+ };
12572
+ }
12573
+ function assertIntegerRange(value, label, minimum, maximum) {
12574
+ if (!Number.isInteger(value) || value < minimum || value > maximum) throw new RangeError(`${label} must be an integer from ${minimum} to ${maximum}.`);
12575
+ }
12576
+ function assertCrawlConfigLimits(config) {
12577
+ if (config.strategy === "health" && config.mode !== "sitemap") throw new RangeError("The health strategy requires sitemap mode.");
12578
+ if (config.strategy === "health" && !config.useSitemap) throw new RangeError("The health strategy requires sitemap discovery.");
12579
+ assertIntegerRange(config.maxPages, "maxPages", 1, MAX_CRAWL_PAGES);
12580
+ assertIntegerRange(config.maxDepth, "maxDepth", 0, 64);
12581
+ assertIntegerRange(config.concurrency, "concurrency", 1, 16);
12582
+ assertIntegerRange(config.timeoutMs, "timeoutMs", 1, MAX_CRAWL_TIMEOUT_MS);
12583
+ if (config.fetchRate.concurrency !== void 0) assertIntegerRange(config.fetchRate.concurrency, "fetchRate.concurrency", 1, 16);
12584
+ if (config.strategy === "health" && config.mode !== "sitemap") throw new RangeError("Health strategy requires sitemap mode.");
12585
+ if (config.sitemapUrl && config.mode !== "sitemap") throw new RangeError("sitemapUrl requires sitemap mode.");
12586
+ return config;
12587
+ }
12588
+ function crawlConfigHash(config) {
12589
+ const normalized = normalizeCrawlConfig(config);
12590
+ return createHash("sha256").update(JSON.stringify(normalized)).digest("hex").slice(0, 16);
12591
+ }
12592
+ function crawlDefinitionId(input) {
12593
+ const normalized = {
12594
+ config: normalizeCrawlConfig(input.config),
12595
+ site: input.site ?? null,
12596
+ googleAnalyticsPropertyId: input.googleAnalyticsPropertyId ?? null,
12597
+ ...input.analyticsConnection ? { analyticsConnection: input.analyticsConnection } : {}
12598
+ };
12599
+ return `crawl_def_${createHash("sha256").update(JSON.stringify(normalized)).digest("hex").slice(0, 20)}`;
12600
+ }
12601
+ function crawlRunId() {
12602
+ return `crawl_${randomUUID().replaceAll("-", "")}`;
12603
+ }
12604
+ function groupCrawlIssues(issues) {
12376
12605
  const groups = /* @__PURE__ */ new Map();
12377
12606
  for (const issue of issues) {
12378
12607
  const existing = groups.get(issue.ruleId) ?? {
@@ -12387,540 +12616,51 @@ function groupIssues(issues) {
12387
12616
  if (existing.sampleUrls.length < 10) existing.sampleUrls.push(issue.url);
12388
12617
  groups.set(issue.ruleId, existing);
12389
12618
  }
12390
- return [...groups.values()];
12391
- }
12392
- function searchValueForGroup(report, urls) {
12393
- const wanted = new Set(urls);
12394
- const pages = report.pages.filter((page) => wanted.has(page.url) || wanted.has(page.finalUrl));
12395
- const metrics = pages.map((page) => page.searchMetrics).filter((item) => Boolean(item));
12396
- const clicks = metrics.reduce((sum, item) => sum + item.clicks, 0);
12397
- const impressions = metrics.reduce((sum, item) => sum + item.impressions, 0);
12398
- const sessions = pages.reduce((sum, page) => sum + (page.analytics?.sessions ?? 0), 0);
12399
- const observedMetrics = new Set(report.dataSources?.analytics.observedMetrics ?? [
12400
- "sessions",
12401
- "totalUsers",
12402
- "conversions"
12403
- ]);
12404
- const totalUsers = observedMetrics.has("totalUsers") ? pages.reduce((sum, page) => sum + (page.analytics?.totalUsers ?? 0), 0) : void 0;
12405
- const conversions = observedMetrics.has("conversions") ? pages.reduce((sum, page) => sum + (page.analytics?.conversions ?? 0), 0) : void 0;
12406
- const avgPosition = metrics.length ? metrics.reduce((sum, item) => sum + item.position, 0) / metrics.length : void 0;
12407
- return {
12408
- searchVisibleUrls: metrics.length,
12409
- clicks,
12410
- impressions,
12411
- sessions,
12412
- totalUsers,
12413
- conversions,
12414
- avgPosition
12619
+ const severityRank = {
12620
+ high: 3,
12621
+ medium: 2,
12622
+ low: 1
12415
12623
  };
12624
+ return [...groups.values()].sort((a, b) => severityRank[b.severity] - severityRank[a.severity] || b.count - a.count);
12416
12625
  }
12417
- function whyThisRanks(input, recommendation, searchStatus, analyticsProvider) {
12418
- const visibility = input.searchVisibleUrls ? `${input.searchVisibleUrls} affected URLs have GSC visibility (${input.clicks} clicks, ${input.impressions} impressions).` : searchStatus && !["joined", "none"].includes(searchStatus) ? `No joined GSC visibility is available for these affected URLs; Search Console evidence is ${searchStatus}.` : "No affected URL has joined GSC visibility yet.";
12419
- const analyticsDetails = analyticsProvider ? analyticsProviderDetails(analyticsProvider) : analyticsProviderDetails("google");
12420
- const analyticsLabel = analyticsDetails.label;
12421
- const sessionLabel = analyticsDetails.landingMetricLabel;
12422
- return `${visibility}${input.sessions ? ` ${analyticsLabel} adds ${input.sessions} ${sessionLabel}${input.conversions === void 0 ? "" : ` and ${input.conversions} conversions`} from affected pages.` : ""} ${recommendation === "review" ? "This is a review observation. Confirm it before scheduling implementation work." : `This ranks as a fix. Severity is ${input.severity}; ${input.affectedUrls} URL${input.affectedUrls === 1 ? " is" : "s are"} affected; estimated effort is ${input.effort}.`}`;
12423
- }
12424
- function verificationCommand(report, severity) {
12425
- return `seo crawl ${report.config.url} --severity ${severity} --max-pages ${Math.min(report.config.maxPages, 100)}`;
12426
- }
12427
- function topFixes(report, filters = {}) {
12428
- return groupIssues(filteredIssues(report, filters)).map((group) => {
12429
- const search = searchValueForGroup(report, group.sampleUrls);
12430
- const effort = EFFORT_BY_CATEGORY[group.category] ?? "medium";
12431
- const scoreFactors = {
12432
- severity: SEVERITY_SCORE[group.severity],
12433
- affectedUrls: group.count,
12434
- searchVisibleUrls: search.searchVisibleUrls,
12435
- clicks: search.clicks,
12436
- impressions: search.impressions,
12437
- sessions: search.sessions,
12438
- totalUsers: search.totalUsers,
12439
- conversions: search.conversions,
12440
- avgPosition: search.avgPosition,
12441
- effort,
12442
- effortScore: EFFORT_SCORE[effort]
12443
- };
12444
- const valueWeight = VALUE_WEIGHT[group.severity];
12445
- const score = scoreFactors.severity + scoreFactors.affectedUrls * AFFECTED_URL_WEIGHT[group.severity] + scoreFactors.searchVisibleUrls * 100 * valueWeight + scoreFactors.clicks * 20 * valueWeight + Math.min(scoreFactors.impressions, 1e4) / 25 * valueWeight + scoreFactors.sessions * 2 * valueWeight + (scoreFactors.conversions ?? 0) * 100 * valueWeight + scoreFactors.effortScore;
12446
- const rule = explainRule(group.ruleId);
12447
- const recommendation = recommendationForRule(group.ruleId);
12448
- return {
12449
- ...group,
12450
- recommendation,
12451
- score: Math.round(score),
12452
- scoreFactors,
12453
- whyThisRanks: whyThisRanks(scoreFactors, recommendation, report.dataSources?.searchConsole.status, report.dataSources?.analytics.provider),
12454
- howToFix: rule?.howToFix ?? "",
12455
- howToVerify: rule?.howToVerify ?? "",
12456
- verification: {
12457
- command: verificationCommand(report, group.severity),
12458
- expected: rule?.howToVerify ?? ""
12459
- }
12460
- };
12461
- }).sort((a, b) => b.score - a.score || SEVERITY_SCORE[b.severity] - SEVERITY_SCORE[a.severity] || b.count - a.count || a.ruleId.localeCompare(b.ruleId)).slice(0, filters.limit ?? 10);
12462
- }
12463
- function reviewObservations(report, filters = {}) {
12464
- return topFixes(report, {
12465
- ...filters,
12466
- recommendation: "review"
12467
- });
12626
+ function isSensitiveFieldName(key) {
12627
+ const lower = key.toLowerCase();
12628
+ return SENSITIVE_FIELD_NAMES.has(lower) || SENSITIVE_PARAM_PATTERN.test(lower.replace(/-/g, "_"));
12468
12629
  }
12469
- // packages/core/src/analyze/crawler/export.ts
12470
- const issueHeaders = [
12471
- "rule_id",
12472
- "title",
12473
- "category",
12474
- "severity",
12475
- "url",
12476
- "detail",
12477
- "clicks",
12478
- "impressions",
12479
- "position"
12480
- ];
12481
- const pageHeaders = [
12482
- "url",
12483
- "final_url",
12484
- "status",
12485
- "indexable",
12486
- "title",
12487
- "meta_description",
12488
- "canonical",
12489
- "word_count",
12490
- "content_extractor",
12491
- "content_extractor_type",
12492
- "content_extraction_fallback",
12493
- "word_count_source",
12494
- "internal_links",
12495
- "internal_inlinks",
12496
- "internal_authority_score",
12497
- "external_links",
12498
- "images_total",
12499
- "images_missing_alt",
12500
- "oversized_image_candidates",
12501
- "schema_types",
12502
- "clicks",
12503
- "impressions",
12504
- "sessions"
12505
- ];
12506
- function renderCrawlCsv(report) {
12507
- return renderCsv(report.issues.map((issue) => ({
12508
- rule_id: issue.ruleId,
12509
- title: issue.title,
12510
- category: issue.category,
12511
- severity: issue.severity,
12512
- url: issue.url,
12513
- detail: issue.detail,
12514
- clicks: issue.searchMetrics?.clicks,
12515
- impressions: issue.searchMetrics?.impressions,
12516
- position: issue.searchMetrics?.position
12517
- })), issueHeaders);
12630
+ function sanitizeUrlString(value) {
12631
+ try {
12632
+ const url = new URL(value);
12633
+ for (const key of [...url.searchParams.keys()]) if (SENSITIVE_PARAM_PATTERN.test(key)) url.searchParams.set(key, "[redacted]");
12634
+ return url.toString();
12635
+ } catch {
12636
+ return value;
12637
+ }
12518
12638
  }
12519
- function renderCrawlPagesCsv(report) {
12520
- return renderCsv(report.pages.map((page) => ({
12521
- url: page.url,
12522
- final_url: page.finalUrl,
12523
- status: page.status,
12524
- indexable: page.indexable,
12525
- title: page.title,
12526
- meta_description: page.metaDescription,
12527
- canonical: page.canonical,
12528
- word_count: page.wordCount,
12529
- content_extractor: page.contentExtraction?.used,
12530
- content_extractor_type: page.contentExtraction?.extractorType,
12531
- content_extraction_fallback: page.contentExtraction?.fallback,
12532
- word_count_source: page.contentExtraction?.wordCountSource,
12533
- internal_links: page.outgoingInternalCount,
12534
- internal_inlinks: page.internalInlinkCount,
12535
- internal_authority_score: page.internalLinkAuthorityScore,
12536
- external_links: page.outgoingExternalCount,
12537
- images_total: page.imagesTotal,
12538
- images_missing_alt: page.imagesMissingAlt,
12539
- oversized_image_candidates: page.oversizedImageCandidates?.length ?? 0,
12540
- schema_types: page.schemaTypes?.join("|"),
12541
- clicks: page.searchMetrics?.clicks,
12542
- impressions: page.searchMetrics?.impressions,
12543
- sessions: page.analytics?.sessions
12544
- })), pageHeaders);
12639
+ function sanitizeTenantString(value) {
12640
+ return value.replace(/https?:\/\/[^\s"'<>`]+/g, (match) => sanitizeUrlString(match)).replace(/\b(access_?token|api_?key|auth|client_?secret|jwt|password|refresh_?token|secret|session|signature|token)(\s*[=:]\s*)[^\s,;&]+/gi, (_match, key, separator) => `${key}${separator}[redacted]`).replace(/\b[A-Za-z]:\\[^\s"'<>`]+/g, "[local-path]").replace(/\/(?:Users|home|tmp|var\/folders)\/[^\s"'<>`]+/g, "[local-path]");
12545
12641
  }
12546
- function renderCrawlPretty(report, fixes = topFixes(report)) {
12547
- const reviews = reviewObservations(report);
12548
- const requestSummary = report.requestEvidenceStatus === "available" ? `${report.requests.length} requests` : report.requestEvidenceStatus === "partial" ? `${report.requests.length} observed requests; some started requests were still in flight when the crawl stopped` : "request evidence unavailable for this legacy report";
12549
- const lines = [
12550
- `Crawl report for ${report.config.url}`,
12551
- "",
12552
- `Status: ${report.status}`,
12553
- `Strategy: ${report.config.strategy === "health" ? "sitemap health pass" : "full audit"}`,
12554
- `Crawler: ${report.access.crawler.userAgent}`,
12555
- `Documents: ${report.summary.totalPages} retained; ${requestSummary}, ${report.summary.discoveredUrls} URLs discovered, ${report.summary.failedUrls} failed, ${report.summary.skippedUrls} skipped`,
12556
- `Issues: ${report.issues.length} (${report.summary.highIssues} high, ${report.summary.mediumIssues} medium, ${report.summary.lowIssues} low)`
12557
- ];
12558
- if (report.externalLinkVerification) {
12559
- const external = report.externalLinkVerification;
12560
- lines.push(`External links: ${external.fetchedUrls} fetched, ${external.failedUrls} request failure${external.failedUrls === 1 ? "" : "s"}, ${external.deferredUrls} deferred (${external.dataStatus})`);
12561
- }
12562
- if (report.access.blockedRequests > 0) {
12563
- lines.push("", `Access blocks: ${report.access.blockedRequests}`);
12564
- for (const sample of report.access.samples.slice(0, 3)) {
12565
- const requestId = sample.evidence.requestId ? `, request ID ${sample.evidence.requestId}` : "";
12566
- lines.push(`- ${sample.url}: ${sample.evidence.guidance.summary} (${sample.evidence.provider}${requestId})`, ` Action: ${sample.evidence.guidance.recommendedAction}`, ` Security: ${sample.evidence.guidance.securityNote}`);
12567
- if (sample.evidence.guidance.documentationUrl) lines.push(` Provider docs: ${sample.evidence.guidance.documentationUrl}`);
12642
+ function sanitizeTenantValue(value, key) {
12643
+ if (key && isSensitiveFieldName(key)) return value === "[redacted]" ? value : "[redacted]";
12644
+ if (typeof value === "string") return sanitizeTenantString(value);
12645
+ if (Array.isArray(value)) {
12646
+ let sanitized;
12647
+ for (let index = 0; index < value.length; index += 1) {
12648
+ const item = sanitizeTenantValue(value[index]);
12649
+ if (sanitized) sanitized.push(item);
12650
+ else if (item !== value[index]) sanitized = [...value.slice(0, index), item];
12568
12651
  }
12652
+ return sanitized ?? value;
12569
12653
  }
12570
- if (fixes.length) {
12571
- lines.push("", "Prioritised fixes");
12572
- for (const fix of fixes.slice(0, 5)) {
12573
- lines.push(`- ${fix.title} (${fix.severity}, ${fix.count} URL${fix.count === 1 ? "" : "s"}): ${fix.howToFix}`);
12574
- if (fix.sampleUrls[0]) lines.push(` First URL: ${fix.sampleUrls[0]}`);
12575
- }
12576
- } else lines.push("", "No prioritised fixes found.");
12577
- if (reviews.length) {
12578
- lines.push("", "Review observations (check before scheduling work)");
12579
- for (const observation of reviews.slice(0, 5)) {
12580
- lines.push(`- ${observation.title} (${observation.severity}, ${observation.count} URL${observation.count === 1 ? "" : "s"}): ${observation.howToFix}`);
12581
- if (observation.sampleUrls[0]) lines.push(` First URL: ${observation.sampleUrls[0]}`);
12582
- }
12654
+ if (!value || typeof value !== "object") return value;
12655
+ const record = value;
12656
+ let sanitized;
12657
+ for (const [entryKey, entry] of Object.entries(record)) {
12658
+ const safeEntry = sanitizeTenantValue(entry, entryKey);
12659
+ if (safeEntry === entry) continue;
12660
+ sanitized ??= { ...record };
12661
+ sanitized[entryKey] = safeEntry;
12583
12662
  }
12584
- lines.push("", "Next commands", report.config.strategy === "health" ? `- seo crawl ${report.config.url} --max-pages ${report.config.maxPages} --json` : `- seo crawl ${report.config.url} --json`);
12585
- for (const command of [...new Set(fixes.slice(0, 3).map((fix) => fix.verification.command))]) lines.push(`- ${command}`);
12586
- if (report.warnings.length) lines.push("", "Warnings", ...report.warnings.map((item) => `- ${item}`));
12587
- if (report.caveats.length) lines.push("", "Caveats", ...report.caveats.map((item) => `- ${item}`));
12588
- return `${lines.join("\n")}\n`;
12589
- }
12590
- function renderCrawlJunit(report) {
12591
- const issuesByUrl = /* @__PURE__ */ new Map();
12592
- for (const issue of report.issues) {
12593
- const existing = issuesByUrl.get(issue.url) ?? [];
12594
- existing.push(issue);
12595
- issuesByUrl.set(issue.url, existing);
12596
- }
12597
- const sitemapCases = (report.sitemapDiscovery?.roots ?? []).flatMap((root) => {
12598
- return (root.documents.length ? root.documents : [{
12599
- url: root.url,
12600
- dataStatus: root.dataStatus,
12601
- compression: "none",
12602
- warning: root.warnings.join(" ") || void 0
12603
- }]).map((document) => {
12604
- const failures = [
12605
- ...document.dataStatus !== "complete" ? [document.warning ?? `Sitemap evidence is ${document.dataStatus}.`] : [],
12606
- ...document.redirected ? [`Sitemap redirected to ${document.finalUrl ?? "an unknown URL"}.`] : [],
12607
- ...document.status !== void 0 && (document.status < 200 || document.status >= 300) ? [`Sitemap returned HTTP ${document.status}.`] : []
12608
- ];
12609
- const output = [
12610
- `dataStatus=${document.dataStatus}`,
12611
- ...document.status !== void 0 ? [`status=${document.status}`] : [],
12612
- ...document.finalUrl ? [`finalUrl=${document.finalUrl}`] : [],
12613
- ...document.root ? [`root=${document.root}`] : [],
12614
- ...document.warning ? [`warning=${document.warning}`] : []
12615
- ].join("\n");
12616
- return {
12617
- xml: [
12618
- ` <testcase classname="seo.sitemap-document" name="${escapeXml(document.url)}" time="0.000">`,
12619
- ...failures.length ? [` <failure message="${escapeXml(failures[0] ?? "Sitemap check failed")}">${escapeXml(failures.join("\n"))}</failure>`] : [],
12620
- ` <system-out>${escapeXml(output)}</system-out>`,
12621
- " </testcase>"
12622
- ].join("\n"),
12623
- failed: failures.length > 0
12624
- };
12625
- });
12626
- });
12627
- const requestCases = report.requests.map((request) => {
12628
- const failures = [
12629
- ...(issuesByUrl.get(request.requestedUrl) ?? []).map((issue) => `${issue.severity}: ${issue.title}${issue.detail ? `: ${issue.detail}` : ""}`),
12630
- ...request.outcome === "failure" ? [`${request.failureKind}: ${request.error}`] : [],
12631
- ...request.outcome === "skipped" ? [request.reason === "origin-backpressure" ? request.error : `${request.reason}: ${request.robotsTxt.matchedLine ?? request.robotsTxt.availability}`] : []
12632
- ];
12633
- const output = request.outcome === "response" ? [
12634
- `status=${request.status}`,
12635
- `finalUrl=${request.finalUrl}`,
12636
- ...request.robotsTxt ? [
12637
- `robotsUrl=${request.robotsTxt.url}`,
12638
- `robotsAllowed=${request.robotsTxt.allowed ?? "unknown"}`,
12639
- `robotsAvailability=${request.robotsTxt.availability}`,
12640
- ...request.robotsTxt.status !== void 0 ? [`robotsStatus=${request.robotsTxt.status}`] : [],
12641
- ...request.robotsTxt.matchedLine ? [`robotsMatchedLine=${request.robotsTxt.matchedLine}`] : [],
12642
- ...request.robotsTxt.error ? [`robotsError=${request.robotsTxt.error}`] : []
12643
- ] : [],
12644
- ...request.accessBlock ? [
12645
- `accessProvider=${request.accessBlock.provider}`,
12646
- `crawler=${request.accessBlock.crawler.userAgent}`,
12647
- ...request.accessBlock.requestId ? [`requestId=${request.accessBlock.requestId}`] : []
12648
- ] : []
12649
- ].join("\n") : request.outcome === "failure" ? request.error : request.reason === "origin-backpressure" ? `reason=${request.reason}\nerror=${request.error}` : [
12650
- `reason=${request.reason}`,
12651
- `robotsUrl=${request.robotsTxt.url}`,
12652
- `robotsAllowed=${request.robotsTxt.allowed ?? "unknown"}`,
12653
- `robotsAvailability=${request.robotsTxt.availability}`,
12654
- ...request.robotsTxt.status !== void 0 ? [`robotsStatus=${request.robotsTxt.status}`] : [],
12655
- ...request.robotsTxt.matchedLine ? [`robotsMatchedLine=${request.robotsTxt.matchedLine}`] : [],
12656
- ...request.robotsTxt.error ? [`robotsError=${request.robotsTxt.error}`] : []
12657
- ].join("\n");
12658
- const duration = ((request.durationMs ?? 0) / 1e3).toFixed(3);
12659
- return {
12660
- xml: [
12661
- ` <testcase classname="seo.sitemap-health" name="${escapeXml(request.requestedUrl)}" time="${duration}">`,
12662
- ...failures.length ? [` <failure message="${escapeXml(failures[0] ?? "Health check failed")}">${escapeXml(failures.join("\n"))}</failure>`] : [],
12663
- ` <system-out>${escapeXml(output)}</system-out>`,
12664
- " </testcase>"
12665
- ].join("\n"),
12666
- failed: failures.length > 0
12667
- };
12668
- });
12669
- const cases = [...sitemapCases, ...requestCases];
12670
- const failures = cases.filter((item) => item.failed).length;
12671
- const elapsed = report.requests.reduce((sum, request) => sum + (request.durationMs ?? 0), 0);
12672
- return [
12673
- "<?xml version=\"1.0\" encoding=\"UTF-8\"?>",
12674
- `<testsuite name="seo sitemap health" tests="${cases.length}" failures="${failures}" errors="0" skipped="0" time="${(elapsed / 1e3).toFixed(3)}">`,
12675
- " <properties>",
12676
- ` <property name="report.id" value="${escapeXml(report.id)}"/>`,
12677
- ` <property name="report.status" value="${escapeXml(report.status)}"/>`,
12678
- ` <property name="crawl.strategy" value="${escapeXml(report.config.strategy)}"/>`,
12679
- ` <property name="crawler.userAgent" value="${escapeXml(report.access.crawler.userAgent)}"/>`,
12680
- " </properties>",
12681
- ...cases.map((item) => item.xml),
12682
- "</testsuite>",
12683
- ""
12684
- ].join("\n");
12685
- }
12686
- function renderCrawlHtml(report, fixes = topFixes(report)) {
12687
- const reviews = reviewObservations(report);
12688
- const issueRows = report.issueGroups.map((group) => {
12689
- const rule = explainRule(group.ruleId);
12690
- return `<tr><td>${escapeHtml(group.severity)}</td><td>${escapeHtml(group.title)}</td><td>${group.count}</td><td>${escapeHtml(rule?.howToFix ?? "")}</td><td>${escapeHtml(group.sampleUrls[0] ?? "")}</td></tr>`;
12691
- }).join("\n");
12692
- const fixItems = fixes.slice(0, 10).map((fix) => `<li><strong>${escapeHtml(fix.title)}</strong><br>${escapeHtml(fix.howToFix)}<br><span>Verify: ${escapeHtml(fix.howToVerify)}</span><br><code>${escapeHtml(fix.verification.command)}</code></li>`).join("\n");
12693
- const reviewItems = reviews.slice(0, 10).map((observation) => `<li><strong>${escapeHtml(observation.title)}</strong><br>${escapeHtml(observation.howToFix)}<br><span>Check before scheduling work. ${escapeHtml(observation.howToVerify)}</span></li>`).join("\n");
12694
- const caveatItems = report.caveats.map((caveat) => `<li>${escapeHtml(caveat)}</li>`).join("\n");
12695
- const warningItems = report.warnings.map((warning) => `<li>${escapeHtml(warning)}</li>`).join("\n");
12696
- const accessItems = report.access.samples.map((sample) => `<li><strong>${escapeHtml(sample.url)}</strong><br>${escapeHtml(sample.evidence.guidance.summary)}<br>${escapeHtml(sample.evidence.guidance.recommendedAction)}<br><span>${escapeHtml(sample.evidence.guidance.securityNote)}</span></li>`).join("\n");
12697
- return `<!doctype html>
12698
- <html lang="en">
12699
- <head>
12700
- <meta charset="utf-8">
12701
- <meta name="viewport" content="width=device-width, initial-scale=1">
12702
- <title>Crawl report for ${escapeHtml(report.config.url)}</title>
12703
- <style>
12704
- body { color: #17202a; font: 15px/1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; margin: 32px; max-width: 1120px; }
12705
- h1, h2 { line-height: 1.2; }
12706
- .summary { display: grid; gap: 12px; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); margin: 24px 0; }
12707
- .metric { border: 1px solid #d8dee4; border-radius: 8px; padding: 12px; }
12708
- .metric strong { display: block; font-size: 24px; }
12709
- table { border-collapse: collapse; width: 100%; }
12710
- th, td { border-bottom: 1px solid #d8dee4; padding: 10px; text-align: left; vertical-align: top; }
12711
- th { background: #f6f8fa; }
12712
- li { margin: 0 0 12px; }
12713
- span { color: #57606a; }
12714
- </style>
12715
- </head>
12716
- <body>
12717
- <h1>Crawl report</h1>
12718
- <p>${escapeHtml(report.config.url)}</p>
12719
- <p>Strategy: ${escapeHtml(report.config.strategy === "health" ? "sitemap health pass" : "full audit")}<br>Crawler: <code>${escapeHtml(report.access.crawler.userAgent)}</code></p>
12720
- <section class="summary">
12721
- <div class="metric"><span>Status</span><strong>${escapeHtml(report.status)}</strong></div>
12722
- <div class="metric"><span>Documents</span><strong>${report.summary.totalPages}</strong></div>
12723
- <div class="metric"><span>Requests</span><strong>${report.requestEvidenceStatus === "unavailable" ? "n/a" : report.requests.length}${report.requestEvidenceStatus === "partial" ? " (partial)" : ""}</strong></div>
12724
- <div class="metric"><span>Discovered</span><strong>${report.summary.discoveredUrls}</strong></div>
12725
- <div class="metric"><span>Queued</span><strong>${report.summary.queuedUrls}</strong></div>
12726
- <div class="metric"><span>Skipped</span><strong>${report.summary.skippedUrls}</strong></div>
12727
- <div class="metric"><span>Failed fetches</span><strong>${report.summary.failedUrls}</strong></div>
12728
- <div class="metric"><span>Observed internal links</span><strong>${report.summary.observedInternalLinks}</strong></div>
12729
- <div class="metric"><span>Issues</span><strong>${report.issues.length}</strong></div>
12730
- <div class="metric"><span>High</span><strong>${report.summary.highIssues}</strong></div>
12731
- </section>
12732
- ${accessItems ? `<h2>Access blocks</h2><ul>${accessItems}</ul>` : ""}
12733
- <h2>Prioritised fixes</h2>
12734
- ${fixItems ? `<ol>${fixItems}</ol>` : "<p>No prioritised fixes found.</p>"}
12735
- ${reviewItems ? `<h2>Review observations</h2>
12736
- <p>Check these before scheduling implementation work.</p>
12737
- <ul>${reviewItems}</ul>` : ""}
12738
- ${report.caveats.length ? `<h2>Caveats</h2>
12739
- <ul>${caveatItems}</ul>` : ""}
12740
- ${report.warnings.length ? `<h2>Warnings</h2>
12741
- <ul>${warningItems}</ul>` : ""}
12742
- <h2>Issue groups</h2>
12743
- <table>
12744
- <thead><tr><th>Severity</th><th>Issue</th><th>URLs</th><th>Fix</th><th>Sample URL</th></tr></thead>
12745
- <tbody>${issueRows}</tbody>
12746
- </table>
12747
- </body>
12748
- </html>
12749
- `;
12750
- }
12751
- function renderCrawlMarkdownTickets(report, fixes = topFixes(report)) {
12752
- const reviews = reviewObservations(report);
12753
- const lines = [
12754
- `# Crawl Implementation Tickets`,
12755
- "",
12756
- `Source: ${report.config.url}`,
12757
- `Report: ${report.id}`,
12758
- `Status: ${report.status}`,
12759
- `Strategy: ${report.config.strategy}`,
12760
- `Crawler: ${report.access.crawler.userAgent}`,
12761
- ""
12762
- ];
12763
- if (report.caveats.length) lines.push("## Caveats", "", ...report.caveats.map((item) => `- ${item}`), "");
12764
- if (report.warnings.length) lines.push("## Warnings", "", ...report.warnings.map((item) => `- ${item}`), "");
12765
- if (report.access.blockedRequests > 0) lines.push("## Access blocks", "", ...report.access.samples.flatMap((sample) => [
12766
- `- ${sample.url}: ${sample.evidence.guidance.summary}`,
12767
- ` - Action: ${sample.evidence.guidance.recommendedAction}`,
12768
- ` - Security: ${sample.evidence.guidance.securityNote}`
12769
- ]), "");
12770
- for (const [index, fix] of fixes.entries()) lines.push(`## ${index + 1}. ${fix.title}`, "", `- [ ] Fix ${fix.count} affected URL${fix.count === 1 ? "" : "s"}`, `- Severity: ${fix.severity}`, `- Rule: ${fix.ruleId}`, `- Why this ranks: ${fix.whyThisRanks}`, `- Plain-English fix: ${fix.howToFix}`, `- Verify: ${fix.howToVerify}`, `- Command: ${fix.verification.command}`, "", `Affected URLs:`, ...fix.sampleUrls.slice(0, 10).map((url) => `- ${url}`), "");
12771
- if (!fixes.length) lines.push("No implementation tickets were generated.");
12772
- if (reviews.length) lines.push("## Review observations", "", "These are not implementation tickets. Check the evidence before deciding whether to work on them.", "", ...reviews.slice(0, 10).flatMap((observation) => [`- ${observation.title} (${observation.severity}, ${observation.count} URL${observation.count === 1 ? "" : "s"}): ${observation.howToFix}`, ` - First URL: ${observation.sampleUrls[0] ?? "No sample URL recorded"}`]), "");
12773
- return `${lines.join("\n")}\n`;
12774
- }
12775
- function escapeHtml(value) {
12776
- return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll("\"", "&quot;").replaceAll("'", "&#39;");
12777
- }
12778
- function escapeXml(value) {
12779
- return escapeHtml(value);
12780
- }
12781
- // packages/core/src/analyze/crawler/report.ts
12782
- const MAX_CRAWL_PAGES = 1e4;
12783
- const MAX_CRAWL_DEPTH = 64;
12784
- const MAX_CRAWL_CONCURRENCY = 16;
12785
- const MAX_CRAWL_TIMEOUT_MS = 12e4;
12786
- const SENSITIVE_FIELD_NAMES = /* @__PURE__ */ new Set([
12787
- "authorization",
12788
- "cookie",
12789
- "proxy-authorization",
12790
- "set-cookie",
12791
- "www-authenticate",
12792
- "x-api-key",
12793
- "api-key",
12794
- "access-token",
12795
- "refresh-token"
12796
- ]);
12797
- const SENSITIVE_PARAM_PATTERN = /^(access_?token|auth|api_?key|client_?secret|code|jwt|key|password|refresh_?token|secret|session|sig|signature|token)$/i;
12798
- function uniqueSorted(values = []) {
12799
- return [...new Set(values.filter(Boolean))].sort();
12800
- }
12801
- function normalizeFetchRate(input, strategy) {
12802
- const defaultConcurrency = strategy === "health" ? 4 : 8;
12803
- return {
12804
- concurrency: input.fetchRate?.concurrency ?? input.concurrency ?? defaultConcurrency,
12805
- ...input.fetchRate?.intervalCap !== void 0 ? { intervalCap: input.fetchRate.intervalCap } : {},
12806
- ...input.fetchRate?.intervalMs !== void 0 ? { intervalMs: input.fetchRate.intervalMs } : {},
12807
- ...input.fetchRate?.backpressure ? { backpressure: input.fetchRate.backpressure } : {}
12808
- };
12809
- }
12810
- function normalizeCrawlConfig(input) {
12811
- const strategy = input.strategy ?? "full";
12812
- const concurrency = input.concurrency ?? (strategy === "health" ? 4 : 8);
12813
- return {
12814
- url: new URL(input.url).toString(),
12815
- mode: input.mode ?? (strategy === "health" ? "sitemap" : "site"),
12816
- strategy,
12817
- ...input.sitemapUrl ? { sitemapUrl: new URL(input.sitemapUrl).toString() } : {},
12818
- urls: uniqueSorted(input.urls ?? []),
12819
- maxPages: input.maxPages ?? 500,
12820
- maxDepth: input.maxDepth ?? 16,
12821
- concurrency,
12822
- timeoutMs: input.timeoutMs ?? 2e4,
12823
- include: uniqueSorted(input.include ?? []),
12824
- exclude: uniqueSorted(input.exclude ?? []),
12825
- respectRobots: input.respectRobots ?? true,
12826
- useSitemap: input.useSitemap ?? true,
12827
- checkExternal: strategy === "health" ? false : input.checkExternal ?? true,
12828
- js: strategy === "health" ? "off" : normalizeJavaScriptRenderingMode(input.js),
12829
- refresh: strategy === "health" ? true : input.refresh ?? false,
12830
- fetchRate: normalizeFetchRate(input, strategy)
12831
- };
12832
- }
12833
- function assertIntegerRange(value, label, minimum, maximum) {
12834
- if (!Number.isInteger(value) || value < minimum || value > maximum) throw new RangeError(`${label} must be an integer from ${minimum} to ${maximum}.`);
12835
- }
12836
- function assertCrawlConfigLimits(config) {
12837
- if (config.strategy === "health" && config.mode !== "sitemap") throw new RangeError("The health strategy requires sitemap mode.");
12838
- if (config.strategy === "health" && !config.useSitemap) throw new RangeError("The health strategy requires sitemap discovery.");
12839
- assertIntegerRange(config.maxPages, "maxPages", 1, MAX_CRAWL_PAGES);
12840
- assertIntegerRange(config.maxDepth, "maxDepth", 0, 64);
12841
- assertIntegerRange(config.concurrency, "concurrency", 1, 16);
12842
- assertIntegerRange(config.timeoutMs, "timeoutMs", 1, MAX_CRAWL_TIMEOUT_MS);
12843
- if (config.fetchRate.concurrency !== void 0) assertIntegerRange(config.fetchRate.concurrency, "fetchRate.concurrency", 1, 16);
12844
- if (config.strategy === "health" && config.mode !== "sitemap") throw new RangeError("Health strategy requires sitemap mode.");
12845
- if (config.sitemapUrl && config.mode !== "sitemap") throw new RangeError("sitemapUrl requires sitemap mode.");
12846
- return config;
12847
- }
12848
- function crawlConfigHash(config) {
12849
- const normalized = normalizeCrawlConfig(config);
12850
- return createHash("sha256").update(JSON.stringify(normalized)).digest("hex").slice(0, 16);
12851
- }
12852
- function crawlDefinitionId(input) {
12853
- const normalized = {
12854
- config: normalizeCrawlConfig(input.config),
12855
- site: input.site ?? null,
12856
- googleAnalyticsPropertyId: input.googleAnalyticsPropertyId ?? null,
12857
- ...input.analyticsConnection ? { analyticsConnection: input.analyticsConnection } : {}
12858
- };
12859
- return `crawl_def_${createHash("sha256").update(JSON.stringify(normalized)).digest("hex").slice(0, 20)}`;
12860
- }
12861
- function crawlRunId() {
12862
- return `crawl_${randomUUID().replaceAll("-", "")}`;
12863
- }
12864
- function groupCrawlIssues(issues) {
12865
- const groups = /* @__PURE__ */ new Map();
12866
- for (const issue of issues) {
12867
- const existing = groups.get(issue.ruleId) ?? {
12868
- ruleId: issue.ruleId,
12869
- title: issue.title,
12870
- category: issue.category,
12871
- severity: issue.severity,
12872
- count: 0,
12873
- sampleUrls: []
12874
- };
12875
- existing.count += 1;
12876
- if (existing.sampleUrls.length < 10) existing.sampleUrls.push(issue.url);
12877
- groups.set(issue.ruleId, existing);
12878
- }
12879
- const severityRank = {
12880
- high: 3,
12881
- medium: 2,
12882
- low: 1
12883
- };
12884
- return [...groups.values()].sort((a, b) => severityRank[b.severity] - severityRank[a.severity] || b.count - a.count);
12885
- }
12886
- function isSensitiveFieldName(key) {
12887
- const lower = key.toLowerCase();
12888
- return SENSITIVE_FIELD_NAMES.has(lower) || SENSITIVE_PARAM_PATTERN.test(lower.replace(/-/g, "_"));
12889
- }
12890
- function sanitizeUrlString(value) {
12891
- try {
12892
- const url = new URL(value);
12893
- for (const key of [...url.searchParams.keys()]) if (SENSITIVE_PARAM_PATTERN.test(key)) url.searchParams.set(key, "[redacted]");
12894
- return url.toString();
12895
- } catch {
12896
- return value;
12897
- }
12898
- }
12899
- function sanitizeTenantString(value) {
12900
- return value.replace(/https?:\/\/[^\s"'<>`]+/g, (match) => sanitizeUrlString(match)).replace(/\b(access_?token|api_?key|auth|client_?secret|jwt|password|refresh_?token|secret|session|signature|token)(\s*[=:]\s*)[^\s,;&]+/gi, (_match, key, separator) => `${key}${separator}[redacted]`).replace(/\b[A-Za-z]:\\[^\s"'<>`]+/g, "[local-path]").replace(/\/(?:Users|home|tmp|var\/folders)\/[^\s"'<>`]+/g, "[local-path]");
12901
- }
12902
- function sanitizeTenantValue(value, key) {
12903
- if (key && isSensitiveFieldName(key)) return value === "[redacted]" ? value : "[redacted]";
12904
- if (typeof value === "string") return sanitizeTenantString(value);
12905
- if (Array.isArray(value)) {
12906
- let sanitized;
12907
- for (let index = 0; index < value.length; index += 1) {
12908
- const item = sanitizeTenantValue(value[index]);
12909
- if (sanitized) sanitized.push(item);
12910
- else if (item !== value[index]) sanitized = [...value.slice(0, index), item];
12911
- }
12912
- return sanitized ?? value;
12913
- }
12914
- if (!value || typeof value !== "object") return value;
12915
- const record = value;
12916
- let sanitized;
12917
- for (const [entryKey, entry] of Object.entries(record)) {
12918
- const safeEntry = sanitizeTenantValue(entry, entryKey);
12919
- if (safeEntry === entry) continue;
12920
- sanitized ??= { ...record };
12921
- sanitized[entryKey] = safeEntry;
12922
- }
12923
- return sanitized ?? value;
12663
+ return sanitized ?? value;
12924
12664
  }
12925
12665
  function sanitizeCrawlConfig(config) {
12926
12666
  return sanitizeTenantValue(config);
@@ -12948,6 +12688,10 @@ function sanitizeMessages(values = []) {
12948
12688
  function compareText$17(left, right) {
12949
12689
  return left < right ? -1 : left > right ? 1 : 0;
12950
12690
  }
12691
+ function compareCaveats(left, right) {
12692
+ const rank = (value) => value.startsWith("Coverage incomplete:") ? 0 : 1;
12693
+ return rank(left) - rank(right) || compareText$17(left, right);
12694
+ }
12951
12695
  function sortPages(pages) {
12952
12696
  return [...pages].sort((left, right) => compareText$17(left.url, right.url));
12953
12697
  }
@@ -13182,7 +12926,7 @@ function createCrawlReport(input) {
13182
12926
  ...input.externalLinkVerification ? { externalLinkVerification: sanitizeTenantValue(input.externalLinkVerification) } : {},
13183
12927
  ...input.siteChecks ? { siteChecks: sanitizeTenantValue(input.siteChecks) } : {},
13184
12928
  warnings: sanitizeMessages(input.warnings).sort(compareText$17),
13185
- caveats: sanitizeMessages(input.caveats).sort(compareText$17)
12929
+ caveats: sanitizeMessages(input.caveats).sort(compareCaveats)
13186
12930
  };
13187
12931
  }
13188
12932
  function normalizeLoadedCrawlReport(report) {
@@ -13214,9 +12958,547 @@ function normalizeLoadedCrawlReport(report) {
13214
12958
  issues,
13215
12959
  issueGroups: groupCrawlIssues(issues),
13216
12960
  warnings: sanitizeMessages(report.warnings).sort(compareText$17),
13217
- caveats: sanitizeMessages(report.caveats).sort(compareText$17)
12961
+ caveats: sanitizeMessages(report.caveats).sort(compareCaveats)
12962
+ };
12963
+ }
12964
+ // packages/core/src/analyze/crawler/crawl-caveats.ts
12965
+ /** Reports rank a caveat with this prefix first and lead with it. */
12966
+ const CRAWL_COVERAGE_CAVEAT_PREFIX = "Coverage incomplete:";
12967
+ function rateLimitRerunCommand(config, intervalMs) {
12968
+ const scope = config.strategy === "health" ? " --health" : config.mode === "site" ? "" : ` --mode ${config.mode}`;
12969
+ const sitemap = config.sitemapUrl ? ` --sitemap-url ${config.sitemapUrl}` : "";
12970
+ const slowMs = Math.min(8e3, Math.max(2e3, intervalMs * 2));
12971
+ return `seo crawl ${config.url}${scope}${sitemap} --max-pages ${config.maxPages} --concurrency 1 --fetch-interval-cap 1 --fetch-interval-ms ${slowMs}`;
12972
+ }
12973
+ /** Set when the sitemaps list more URLs than the page limit let the crawl read. */
12974
+ function sitemapLimitCaveatInput(config, discovery, defaulted) {
12975
+ if (!discovery || discovery.urlsReturned < config.maxPages || !discovery.roots.some((root) => root.possiblyTruncated)) return;
12976
+ return {
12977
+ maxPages: config.maxPages,
12978
+ defaulted,
12979
+ ceiling: MAX_CRAWL_PAGES,
12980
+ health: config.strategy === "health",
12981
+ url: config.url
13218
12982
  };
13219
12983
  }
12984
+ function crawlCaveats(input) {
12985
+ const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
12986
+ if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
12987
+ if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
12988
+ if (input.rateLimit) {
12989
+ const { readPages, skippedUrls, rerunCommand } = input.rateLimit;
12990
+ caveats.push(`${CRAWL_COVERAGE_CAVEAT_PREFIX} read ${readPages} of ${readPages + skippedUrls} pages because the origin kept rate limiting the crawl (HTTP 429) after it backed off. Rerun slower: ${rerunCommand}`);
12991
+ }
12992
+ if (input.sitemapLimit) {
12993
+ const { maxPages, defaulted, ceiling, health, url } = input.sitemapLimit;
12994
+ caveats.push(`Checked only the first ${maxPages} sitemap URLs because the page limit is ${maxPages}${defaulted ? " by default" : ""}, and the sitemaps list more. Raise --max-pages, up to ${ceiling.toLocaleString("en-US")}, to check more: seo crawl ${url}${health ? " --health" : ""} --max-pages ${ceiling}`);
12995
+ }
12996
+ if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
12997
+ const renderingFallbackUrls = input.renderingFallbackUrls ?? 0;
12998
+ if (renderingFallbackUrls > 0) caveats.push(`JavaScript rendering could not run for ${renderingFallbackUrls} page${renderingFallbackUrls === 1 ? "" : "s"}, so ${renderingFallbackUrls === 1 ? "it was" : "they were"} audited from raw HTML. Client-rendered content, metadata and links may be missing. Install Chromium or set SEO_BROWSER_EXECUTABLE_PATH, then rerun.`);
12999
+ const queryVariantSkippedUrls = input.queryVariantSkippedUrls ?? 0;
13000
+ if (queryVariantSkippedUrls > 0) caveats.push(`Left ${queryVariantSkippedUrls} query-string variant${queryVariantSkippedUrls === 1 ? "" : "s"} unqueued after 5 variants of the same path, so tracking parameters do not fill the crawl.`);
13001
+ for (const issue of input.reachIssues ?? []) if (issue.detail) caveats.push(issue.detail);
13002
+ return caveats;
13003
+ }
13004
+ /**
13005
+ * A site crawl that found no internal links on the start page and no sitemap
13006
+ * URLs checked one page, so its other findings cannot stand for the site.
13007
+ */
13008
+ function crawlReachIssues(input) {
13009
+ const [page] = input.pages;
13010
+ if (input.crawledPages !== 1 || input.pages.length !== 1 || !page || page.outgoingInternalCount > 0 || input.sitemapUrlsReturned > 0) return [];
13011
+ const rule = explainRule("crawl_single_page_reach");
13012
+ if (!rule) throw new Error("Missing rule guidance for crawl_single_page_reach");
13013
+ return [{
13014
+ ruleId: "crawl_single_page_reach",
13015
+ title: rule.title,
13016
+ category: rule.category,
13017
+ severity: rule.defaultSeverity,
13018
+ url: page.url,
13019
+ detail: page.usedJs ? "The rendered start page had 0 internal links and no sitemap returned URLs, so the crawl checked only this page. List the pages to check with --mode list." : "The start page had 0 internal links in its HTML and no sitemap returned URLs, so the crawl checked only this page. It probably builds its links with JavaScript. Rerun with --rendering on, or list the pages with --mode list.",
13020
+ evidence: {
13021
+ startUrl: input.startUrl,
13022
+ crawledPages: 1,
13023
+ outgoingInternalCount: 0,
13024
+ sitemapUrlsReturned: 0,
13025
+ rendered: Boolean(page.usedJs)
13026
+ }
13027
+ }];
13028
+ }
13029
+ // packages/core/src/analyze/crawler/top-fixes.ts
13030
+ const SEVERITY_SCORE = {
13031
+ high: 5e3,
13032
+ medium: 1200,
13033
+ low: 100
13034
+ };
13035
+ const EFFORT_BY_CATEGORY = {
13036
+ canonical: "medium",
13037
+ content: "high",
13038
+ agents: "low",
13039
+ headings: "low",
13040
+ images: "medium",
13041
+ indexability: "medium",
13042
+ international: "low",
13043
+ metadata: "low",
13044
+ mobile: "low",
13045
+ response: "medium",
13046
+ security: "medium",
13047
+ social: "low",
13048
+ "structured-data": "medium"
13049
+ };
13050
+ const EFFORT_SCORE = {
13051
+ low: 60,
13052
+ medium: 30,
13053
+ high: 0
13054
+ };
13055
+ const AFFECTED_URL_WEIGHT = {
13056
+ high: 25,
13057
+ medium: 15,
13058
+ low: 2
13059
+ };
13060
+ const VALUE_WEIGHT = {
13061
+ high: 1,
13062
+ medium: .75,
13063
+ low: .15
13064
+ };
13065
+ const CRAWL_REACH_RULES = /* @__PURE__ */ new Set(["crawl_single_page_reach"]);
13066
+ function reachRank(ruleId) {
13067
+ return CRAWL_REACH_RULES.has(ruleId) ? 0 : 1;
13068
+ }
13069
+ function matchesPattern(pattern, value) {
13070
+ if (!pattern.includes("*")) return value.includes(pattern);
13071
+ const parts = pattern.split("*");
13072
+ let position = 0;
13073
+ for (const part of parts) {
13074
+ if (!part) continue;
13075
+ const found = value.slice(position).indexOf(part);
13076
+ if (found === -1) return false;
13077
+ position += found + part.length;
13078
+ }
13079
+ return true;
13080
+ }
13081
+ function filteredIssues(report, filters) {
13082
+ return report.issues.filter((issue) => {
13083
+ if (filters.category && issue.category !== filters.category) return false;
13084
+ if (filters.severity && issue.severity !== filters.severity) return false;
13085
+ if (filters.url && !matchesPattern(filters.url, issue.url)) return false;
13086
+ if ((filters.recommendation ?? "fix") !== recommendationForRule(issue.ruleId)) return false;
13087
+ return true;
13088
+ });
13089
+ }
13090
+ function groupIssues(issues) {
13091
+ const groups = /* @__PURE__ */ new Map();
13092
+ for (const issue of issues) {
13093
+ const existing = groups.get(issue.ruleId) ?? {
13094
+ ruleId: issue.ruleId,
13095
+ title: issue.title,
13096
+ category: issue.category,
13097
+ severity: issue.severity,
13098
+ count: 0,
13099
+ sampleUrls: []
13100
+ };
13101
+ existing.count += 1;
13102
+ if (existing.sampleUrls.length < 10) existing.sampleUrls.push(issue.url);
13103
+ groups.set(issue.ruleId, existing);
13104
+ }
13105
+ return [...groups.values()];
13106
+ }
13107
+ function searchValueForGroup(report, urls) {
13108
+ const wanted = new Set(urls);
13109
+ const pages = report.pages.filter((page) => wanted.has(page.url) || wanted.has(page.finalUrl));
13110
+ const metrics = pages.map((page) => page.searchMetrics).filter((item) => Boolean(item));
13111
+ const clicks = metrics.reduce((sum, item) => sum + item.clicks, 0);
13112
+ const impressions = metrics.reduce((sum, item) => sum + item.impressions, 0);
13113
+ const sessions = pages.reduce((sum, page) => sum + (page.analytics?.sessions ?? 0), 0);
13114
+ const observedMetrics = new Set(report.dataSources?.analytics.observedMetrics ?? [
13115
+ "sessions",
13116
+ "totalUsers",
13117
+ "conversions"
13118
+ ]);
13119
+ const totalUsers = observedMetrics.has("totalUsers") ? pages.reduce((sum, page) => sum + (page.analytics?.totalUsers ?? 0), 0) : void 0;
13120
+ const conversions = observedMetrics.has("conversions") ? pages.reduce((sum, page) => sum + (page.analytics?.conversions ?? 0), 0) : void 0;
13121
+ const avgPosition = metrics.length ? metrics.reduce((sum, item) => sum + item.position, 0) / metrics.length : void 0;
13122
+ return {
13123
+ searchVisibleUrls: metrics.length,
13124
+ clicks,
13125
+ impressions,
13126
+ sessions,
13127
+ totalUsers,
13128
+ conversions,
13129
+ avgPosition
13130
+ };
13131
+ }
13132
+ function whyThisRanks(input, recommendation, searchStatus, analyticsProvider) {
13133
+ const visibility = input.searchVisibleUrls ? `${input.searchVisibleUrls} affected URLs have GSC visibility (${input.clicks} clicks, ${input.impressions} impressions).` : searchStatus && !["joined", "none"].includes(searchStatus) ? `No joined GSC visibility is available for these affected URLs; Search Console evidence is ${searchStatus}.` : "No affected URL has joined GSC visibility yet.";
13134
+ const analyticsDetails = analyticsProvider ? analyticsProviderDetails(analyticsProvider) : analyticsProviderDetails("google");
13135
+ const analyticsLabel = analyticsDetails.label;
13136
+ const sessionLabel = analyticsDetails.landingMetricLabel;
13137
+ return `${visibility}${input.sessions ? ` ${analyticsLabel} adds ${input.sessions} ${sessionLabel}${input.conversions === void 0 ? "" : ` and ${input.conversions} conversions`} from affected pages.` : ""} ${recommendation === "review" ? "This is a review observation. Confirm it before scheduling implementation work." : `This ranks as a fix. Severity is ${input.severity}; ${input.affectedUrls} URL${input.affectedUrls === 1 ? " is" : "s are"} affected; estimated effort is ${input.effort}.`}`;
13138
+ }
13139
+ function verificationCommand(report, severity) {
13140
+ return `seo crawl ${report.config.url} --severity ${severity} --max-pages ${Math.min(report.config.maxPages, 100)}`;
13141
+ }
13142
+ function topFixes(report, filters = {}) {
13143
+ return groupIssues(filteredIssues(report, filters)).map((group) => {
13144
+ const search = searchValueForGroup(report, group.sampleUrls);
13145
+ const effort = EFFORT_BY_CATEGORY[group.category] ?? "medium";
13146
+ const scoreFactors = {
13147
+ severity: SEVERITY_SCORE[group.severity],
13148
+ affectedUrls: group.count,
13149
+ searchVisibleUrls: search.searchVisibleUrls,
13150
+ clicks: search.clicks,
13151
+ impressions: search.impressions,
13152
+ sessions: search.sessions,
13153
+ totalUsers: search.totalUsers,
13154
+ conversions: search.conversions,
13155
+ avgPosition: search.avgPosition,
13156
+ effort,
13157
+ effortScore: EFFORT_SCORE[effort]
13158
+ };
13159
+ const valueWeight = VALUE_WEIGHT[group.severity];
13160
+ const score = scoreFactors.severity + scoreFactors.affectedUrls * AFFECTED_URL_WEIGHT[group.severity] + scoreFactors.searchVisibleUrls * 100 * valueWeight + scoreFactors.clicks * 20 * valueWeight + Math.min(scoreFactors.impressions, 1e4) / 25 * valueWeight + scoreFactors.sessions * 2 * valueWeight + (scoreFactors.conversions ?? 0) * 100 * valueWeight + scoreFactors.effortScore;
13161
+ const rule = explainRule(group.ruleId);
13162
+ const recommendation = recommendationForRule(group.ruleId);
13163
+ return {
13164
+ ...group,
13165
+ recommendation,
13166
+ score: Math.round(score),
13167
+ scoreFactors,
13168
+ whyThisRanks: whyThisRanks(scoreFactors, recommendation, report.dataSources?.searchConsole.status, report.dataSources?.analytics.provider),
13169
+ howToFix: rule?.howToFix ?? "",
13170
+ howToVerify: rule?.howToVerify ?? "",
13171
+ verification: {
13172
+ command: verificationCommand(report, group.severity),
13173
+ expected: rule?.howToVerify ?? ""
13174
+ }
13175
+ };
13176
+ }).sort((a, b) => reachRank(a.ruleId) - reachRank(b.ruleId) || b.score - a.score || SEVERITY_SCORE[b.severity] - SEVERITY_SCORE[a.severity] || b.count - a.count || a.ruleId.localeCompare(b.ruleId)).slice(0, filters.limit ?? 10);
13177
+ }
13178
+ function reviewObservations(report, filters = {}) {
13179
+ return topFixes(report, {
13180
+ ...filters,
13181
+ recommendation: "review"
13182
+ });
13183
+ }
13184
+ // packages/core/src/analyze/crawler/export.ts
13185
+ const issueHeaders = [
13186
+ "rule_id",
13187
+ "title",
13188
+ "category",
13189
+ "severity",
13190
+ "url",
13191
+ "detail",
13192
+ "clicks",
13193
+ "impressions",
13194
+ "position"
13195
+ ];
13196
+ const pageHeaders = [
13197
+ "url",
13198
+ "final_url",
13199
+ "status",
13200
+ "indexable",
13201
+ "title",
13202
+ "meta_description",
13203
+ "canonical",
13204
+ "word_count",
13205
+ "content_extractor",
13206
+ "content_extractor_type",
13207
+ "content_extraction_fallback",
13208
+ "word_count_source",
13209
+ "internal_links",
13210
+ "internal_inlinks",
13211
+ "internal_authority_score",
13212
+ "external_links",
13213
+ "images_total",
13214
+ "images_missing_alt",
13215
+ "oversized_image_candidates",
13216
+ "schema_types",
13217
+ "clicks",
13218
+ "impressions",
13219
+ "sessions"
13220
+ ];
13221
+ function renderCrawlCsv(report) {
13222
+ return renderCsv(report.issues.map((issue) => ({
13223
+ rule_id: issue.ruleId,
13224
+ title: issue.title,
13225
+ category: issue.category,
13226
+ severity: issue.severity,
13227
+ url: issue.url,
13228
+ detail: issue.detail,
13229
+ clicks: issue.searchMetrics?.clicks,
13230
+ impressions: issue.searchMetrics?.impressions,
13231
+ position: issue.searchMetrics?.position
13232
+ })), issueHeaders);
13233
+ }
13234
+ function renderCrawlPagesCsv(report) {
13235
+ return renderCsv(report.pages.map((page) => ({
13236
+ url: page.url,
13237
+ final_url: page.finalUrl,
13238
+ status: page.status,
13239
+ indexable: page.indexable,
13240
+ title: page.title,
13241
+ meta_description: page.metaDescription,
13242
+ canonical: page.canonical,
13243
+ word_count: page.wordCount,
13244
+ content_extractor: page.contentExtraction?.used,
13245
+ content_extractor_type: page.contentExtraction?.extractorType,
13246
+ content_extraction_fallback: page.contentExtraction?.fallback,
13247
+ word_count_source: page.contentExtraction?.wordCountSource,
13248
+ internal_links: page.outgoingInternalCount,
13249
+ internal_inlinks: page.internalInlinkCount,
13250
+ internal_authority_score: page.internalLinkAuthorityScore,
13251
+ external_links: page.outgoingExternalCount,
13252
+ images_total: page.imagesTotal,
13253
+ images_missing_alt: page.imagesMissingAlt,
13254
+ oversized_image_candidates: page.oversizedImageCandidates?.length ?? 0,
13255
+ schema_types: page.schemaTypes?.join("|"),
13256
+ clicks: page.searchMetrics?.clicks,
13257
+ impressions: page.searchMetrics?.impressions,
13258
+ sessions: page.analytics?.sessions
13259
+ })), pageHeaders);
13260
+ }
13261
+ /** The caveat a report must lead with when the crawl read too little. */
13262
+ function crawlCoverageCaveat(report) {
13263
+ return report.caveats.find((caveat) => caveat.startsWith(CRAWL_COVERAGE_CAVEAT_PREFIX));
13264
+ }
13265
+ function renderCrawlPretty(report, fixes = topFixes(report)) {
13266
+ const reviews = reviewObservations(report);
13267
+ const requestSummary = report.requestEvidenceStatus === "available" ? `${report.requests.length} requests` : report.requestEvidenceStatus === "partial" ? `${report.requests.length} observed requests; some started requests were still in flight when the crawl stopped` : "request evidence unavailable for this legacy report";
13268
+ const coverage = crawlCoverageCaveat(report);
13269
+ const lines = [
13270
+ ...coverage ? [coverage, ""] : [],
13271
+ `Crawl report for ${report.config.url}`,
13272
+ "",
13273
+ `Status: ${report.status}`,
13274
+ `Strategy: ${report.config.strategy === "health" ? "sitemap health pass" : "full audit"}`,
13275
+ `Crawler: ${report.access.crawler.userAgent}`,
13276
+ `Documents: ${report.summary.totalPages} retained; ${requestSummary}, ${report.summary.discoveredUrls} URLs discovered, ${report.summary.failedUrls} failed, ${report.summary.skippedUrls} skipped`,
13277
+ `Issues: ${report.issues.length} (${report.summary.highIssues} high, ${report.summary.mediumIssues} medium, ${report.summary.lowIssues} low)`
13278
+ ];
13279
+ if (report.externalLinkVerification) {
13280
+ const external = report.externalLinkVerification;
13281
+ lines.push(`External links: ${external.fetchedUrls} fetched, ${external.failedUrls} request failure${external.failedUrls === 1 ? "" : "s"}, ${external.deferredUrls} deferred (${external.dataStatus})`);
13282
+ }
13283
+ if (report.access.blockedRequests > 0) {
13284
+ lines.push("", `Access blocks: ${report.access.blockedRequests}`);
13285
+ for (const sample of report.access.samples.slice(0, 3)) {
13286
+ const requestId = sample.evidence.requestId ? `, request ID ${sample.evidence.requestId}` : "";
13287
+ lines.push(`- ${sample.url}: ${sample.evidence.guidance.summary} (${sample.evidence.provider}${requestId})`, ` Action: ${sample.evidence.guidance.recommendedAction}`, ` Security: ${sample.evidence.guidance.securityNote}`);
13288
+ if (sample.evidence.guidance.documentationUrl) lines.push(` Provider docs: ${sample.evidence.guidance.documentationUrl}`);
13289
+ }
13290
+ }
13291
+ if (fixes.length) {
13292
+ lines.push("", "Prioritised fixes");
13293
+ for (const fix of fixes.slice(0, 5)) {
13294
+ lines.push(`- ${fix.title} (${fix.severity}, ${fix.count} URL${fix.count === 1 ? "" : "s"}): ${fix.howToFix}`);
13295
+ if (fix.sampleUrls[0]) lines.push(` First URL: ${fix.sampleUrls[0]}`);
13296
+ }
13297
+ } else lines.push("", "No prioritised fixes found.");
13298
+ if (reviews.length) {
13299
+ lines.push("", "Review observations (check before scheduling work)");
13300
+ for (const observation of reviews.slice(0, 5)) {
13301
+ lines.push(`- ${observation.title} (${observation.severity}, ${observation.count} URL${observation.count === 1 ? "" : "s"}): ${observation.howToFix}`);
13302
+ if (observation.sampleUrls[0]) lines.push(` First URL: ${observation.sampleUrls[0]}`);
13303
+ }
13304
+ }
13305
+ lines.push("", "Next commands", report.config.strategy === "health" ? `- seo crawl ${report.config.url} --max-pages ${report.config.maxPages} --json` : `- seo crawl ${report.config.url} --json`);
13306
+ for (const command of [...new Set(fixes.slice(0, 3).map((fix) => fix.verification.command))]) lines.push(`- ${command}`);
13307
+ if (report.warnings.length) lines.push("", "Warnings", ...report.warnings.map((item) => `- ${item}`));
13308
+ if (report.caveats.length) lines.push("", "Caveats", ...report.caveats.map((item) => `- ${item}`));
13309
+ return `${lines.join("\n")}\n`;
13310
+ }
13311
+ function renderCrawlJunit(report) {
13312
+ const issuesByUrl = /* @__PURE__ */ new Map();
13313
+ for (const issue of report.issues) {
13314
+ const existing = issuesByUrl.get(issue.url) ?? [];
13315
+ existing.push(issue);
13316
+ issuesByUrl.set(issue.url, existing);
13317
+ }
13318
+ const sitemapCases = (report.sitemapDiscovery?.roots ?? []).flatMap((root) => {
13319
+ return (root.documents.length ? root.documents : [{
13320
+ url: root.url,
13321
+ dataStatus: root.dataStatus,
13322
+ compression: "none",
13323
+ warning: root.warnings.join(" ") || void 0
13324
+ }]).map((document) => {
13325
+ const failures = [
13326
+ ...document.dataStatus !== "complete" ? [document.warning ?? `Sitemap evidence is ${document.dataStatus}.`] : [],
13327
+ ...document.redirected ? [`Sitemap redirected to ${document.finalUrl ?? "an unknown URL"}.`] : [],
13328
+ ...document.status !== void 0 && (document.status < 200 || document.status >= 300) ? [`Sitemap returned HTTP ${document.status}.`] : []
13329
+ ];
13330
+ const output = [
13331
+ `dataStatus=${document.dataStatus}`,
13332
+ ...document.status !== void 0 ? [`status=${document.status}`] : [],
13333
+ ...document.finalUrl ? [`finalUrl=${document.finalUrl}`] : [],
13334
+ ...document.root ? [`root=${document.root}`] : [],
13335
+ ...document.warning ? [`warning=${document.warning}`] : []
13336
+ ].join("\n");
13337
+ return {
13338
+ xml: [
13339
+ ` <testcase classname="seo.sitemap-document" name="${escapeXml(document.url)}" time="0.000">`,
13340
+ ...failures.length ? [` <failure message="${escapeXml(failures[0] ?? "Sitemap check failed")}">${escapeXml(failures.join("\n"))}</failure>`] : [],
13341
+ ` <system-out>${escapeXml(output)}</system-out>`,
13342
+ " </testcase>"
13343
+ ].join("\n"),
13344
+ failed: failures.length > 0
13345
+ };
13346
+ });
13347
+ });
13348
+ const requestCases = report.requests.map((request) => {
13349
+ const failures = [
13350
+ ...(issuesByUrl.get(request.requestedUrl) ?? []).map((issue) => `${issue.severity}: ${issue.title}${issue.detail ? `: ${issue.detail}` : ""}`),
13351
+ ...request.outcome === "failure" ? [`${request.failureKind}: ${request.error}`] : [],
13352
+ ...request.outcome === "skipped" ? [request.reason === "origin-backpressure" ? request.error : `${request.reason}: ${request.robotsTxt.matchedLine ?? request.robotsTxt.availability}`] : []
13353
+ ];
13354
+ const output = request.outcome === "response" ? [
13355
+ `status=${request.status}`,
13356
+ `finalUrl=${request.finalUrl}`,
13357
+ ...request.robotsTxt ? [
13358
+ `robotsUrl=${request.robotsTxt.url}`,
13359
+ `robotsAllowed=${request.robotsTxt.allowed ?? "unknown"}`,
13360
+ `robotsAvailability=${request.robotsTxt.availability}`,
13361
+ ...request.robotsTxt.status !== void 0 ? [`robotsStatus=${request.robotsTxt.status}`] : [],
13362
+ ...request.robotsTxt.matchedLine ? [`robotsMatchedLine=${request.robotsTxt.matchedLine}`] : [],
13363
+ ...request.robotsTxt.error ? [`robotsError=${request.robotsTxt.error}`] : []
13364
+ ] : [],
13365
+ ...request.accessBlock ? [
13366
+ `accessProvider=${request.accessBlock.provider}`,
13367
+ `crawler=${request.accessBlock.crawler.userAgent}`,
13368
+ ...request.accessBlock.requestId ? [`requestId=${request.accessBlock.requestId}`] : []
13369
+ ] : []
13370
+ ].join("\n") : request.outcome === "failure" ? request.error : request.reason === "origin-backpressure" ? `reason=${request.reason}\nerror=${request.error}` : [
13371
+ `reason=${request.reason}`,
13372
+ `robotsUrl=${request.robotsTxt.url}`,
13373
+ `robotsAllowed=${request.robotsTxt.allowed ?? "unknown"}`,
13374
+ `robotsAvailability=${request.robotsTxt.availability}`,
13375
+ ...request.robotsTxt.status !== void 0 ? [`robotsStatus=${request.robotsTxt.status}`] : [],
13376
+ ...request.robotsTxt.matchedLine ? [`robotsMatchedLine=${request.robotsTxt.matchedLine}`] : [],
13377
+ ...request.robotsTxt.error ? [`robotsError=${request.robotsTxt.error}`] : []
13378
+ ].join("\n");
13379
+ const duration = ((request.durationMs ?? 0) / 1e3).toFixed(3);
13380
+ return {
13381
+ xml: [
13382
+ ` <testcase classname="seo.sitemap-health" name="${escapeXml(request.requestedUrl)}" time="${duration}">`,
13383
+ ...failures.length ? [` <failure message="${escapeXml(failures[0] ?? "Health check failed")}">${escapeXml(failures.join("\n"))}</failure>`] : [],
13384
+ ` <system-out>${escapeXml(output)}</system-out>`,
13385
+ " </testcase>"
13386
+ ].join("\n"),
13387
+ failed: failures.length > 0
13388
+ };
13389
+ });
13390
+ const cases = [...sitemapCases, ...requestCases];
13391
+ const failures = cases.filter((item) => item.failed).length;
13392
+ const elapsed = report.requests.reduce((sum, request) => sum + (request.durationMs ?? 0), 0);
13393
+ return [
13394
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>",
13395
+ `<testsuite name="seo sitemap health" tests="${cases.length}" failures="${failures}" errors="0" skipped="0" time="${(elapsed / 1e3).toFixed(3)}">`,
13396
+ " <properties>",
13397
+ ` <property name="report.id" value="${escapeXml(report.id)}"/>`,
13398
+ ` <property name="report.status" value="${escapeXml(report.status)}"/>`,
13399
+ ` <property name="crawl.strategy" value="${escapeXml(report.config.strategy)}"/>`,
13400
+ ` <property name="crawler.userAgent" value="${escapeXml(report.access.crawler.userAgent)}"/>`,
13401
+ " </properties>",
13402
+ ...cases.map((item) => item.xml),
13403
+ "</testsuite>",
13404
+ ""
13405
+ ].join("\n");
13406
+ }
13407
+ function renderCrawlHtml(report, fixes = topFixes(report)) {
13408
+ const reviews = reviewObservations(report);
13409
+ const issueRows = report.issueGroups.map((group) => {
13410
+ const rule = explainRule(group.ruleId);
13411
+ return `<tr><td>${escapeHtml(group.severity)}</td><td>${escapeHtml(group.title)}</td><td>${group.count}</td><td>${escapeHtml(rule?.howToFix ?? "")}</td><td>${escapeHtml(group.sampleUrls[0] ?? "")}</td></tr>`;
13412
+ }).join("\n");
13413
+ const fixItems = fixes.slice(0, 10).map((fix) => `<li><strong>${escapeHtml(fix.title)}</strong><br>${escapeHtml(fix.howToFix)}<br><span>Verify: ${escapeHtml(fix.howToVerify)}</span><br><code>${escapeHtml(fix.verification.command)}</code></li>`).join("\n");
13414
+ const reviewItems = reviews.slice(0, 10).map((observation) => `<li><strong>${escapeHtml(observation.title)}</strong><br>${escapeHtml(observation.howToFix)}<br><span>Check before scheduling work. ${escapeHtml(observation.howToVerify)}</span></li>`).join("\n");
13415
+ const caveatItems = report.caveats.map((caveat) => `<li>${escapeHtml(caveat)}</li>`).join("\n");
13416
+ const warningItems = report.warnings.map((warning) => `<li>${escapeHtml(warning)}</li>`).join("\n");
13417
+ const accessItems = report.access.samples.map((sample) => `<li><strong>${escapeHtml(sample.url)}</strong><br>${escapeHtml(sample.evidence.guidance.summary)}<br>${escapeHtml(sample.evidence.guidance.recommendedAction)}<br><span>${escapeHtml(sample.evidence.guidance.securityNote)}</span></li>`).join("\n");
13418
+ return `<!doctype html>
13419
+ <html lang="en">
13420
+ <head>
13421
+ <meta charset="utf-8">
13422
+ <meta name="viewport" content="width=device-width, initial-scale=1">
13423
+ <title>Crawl report for ${escapeHtml(report.config.url)}</title>
13424
+ <style>
13425
+ body { color: #17202a; font: 15px/1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; margin: 32px; max-width: 1120px; }
13426
+ h1, h2 { line-height: 1.2; }
13427
+ .summary { display: grid; gap: 12px; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); margin: 24px 0; }
13428
+ .metric { border: 1px solid #d8dee4; border-radius: 8px; padding: 12px; }
13429
+ .metric strong { display: block; font-size: 24px; }
13430
+ table { border-collapse: collapse; width: 100%; }
13431
+ th, td { border-bottom: 1px solid #d8dee4; padding: 10px; text-align: left; vertical-align: top; }
13432
+ th { background: #f6f8fa; }
13433
+ li { margin: 0 0 12px; }
13434
+ span { color: #57606a; }
13435
+ </style>
13436
+ </head>
13437
+ <body>
13438
+ <h1>Crawl report</h1>
13439
+ <p>${escapeHtml(report.config.url)}</p>
13440
+ <p>Strategy: ${escapeHtml(report.config.strategy === "health" ? "sitemap health pass" : "full audit")}<br>Crawler: <code>${escapeHtml(report.access.crawler.userAgent)}</code></p>
13441
+ <section class="summary">
13442
+ <div class="metric"><span>Status</span><strong>${escapeHtml(report.status)}</strong></div>
13443
+ <div class="metric"><span>Documents</span><strong>${report.summary.totalPages}</strong></div>
13444
+ <div class="metric"><span>Requests</span><strong>${report.requestEvidenceStatus === "unavailable" ? "n/a" : report.requests.length}${report.requestEvidenceStatus === "partial" ? " (partial)" : ""}</strong></div>
13445
+ <div class="metric"><span>Discovered</span><strong>${report.summary.discoveredUrls}</strong></div>
13446
+ <div class="metric"><span>Queued</span><strong>${report.summary.queuedUrls}</strong></div>
13447
+ <div class="metric"><span>Skipped</span><strong>${report.summary.skippedUrls}</strong></div>
13448
+ <div class="metric"><span>Failed fetches</span><strong>${report.summary.failedUrls}</strong></div>
13449
+ <div class="metric"><span>Observed internal links</span><strong>${report.summary.observedInternalLinks}</strong></div>
13450
+ <div class="metric"><span>Issues</span><strong>${report.issues.length}</strong></div>
13451
+ <div class="metric"><span>High</span><strong>${report.summary.highIssues}</strong></div>
13452
+ </section>
13453
+ ${accessItems ? `<h2>Access blocks</h2><ul>${accessItems}</ul>` : ""}
13454
+ <h2>Prioritised fixes</h2>
13455
+ ${fixItems ? `<ol>${fixItems}</ol>` : "<p>No prioritised fixes found.</p>"}
13456
+ ${reviewItems ? `<h2>Review observations</h2>
13457
+ <p>Check these before scheduling implementation work.</p>
13458
+ <ul>${reviewItems}</ul>` : ""}
13459
+ ${report.caveats.length ? `<h2>Caveats</h2>
13460
+ <ul>${caveatItems}</ul>` : ""}
13461
+ ${report.warnings.length ? `<h2>Warnings</h2>
13462
+ <ul>${warningItems}</ul>` : ""}
13463
+ <h2>Issue groups</h2>
13464
+ <table>
13465
+ <thead><tr><th>Severity</th><th>Issue</th><th>URLs</th><th>Fix</th><th>Sample URL</th></tr></thead>
13466
+ <tbody>${issueRows}</tbody>
13467
+ </table>
13468
+ </body>
13469
+ </html>
13470
+ `;
13471
+ }
13472
+ function renderCrawlMarkdownTickets(report, fixes = topFixes(report)) {
13473
+ const reviews = reviewObservations(report);
13474
+ const lines = [
13475
+ `# Crawl Implementation Tickets`,
13476
+ "",
13477
+ `Source: ${report.config.url}`,
13478
+ `Report: ${report.id}`,
13479
+ `Status: ${report.status}`,
13480
+ `Strategy: ${report.config.strategy}`,
13481
+ `Crawler: ${report.access.crawler.userAgent}`,
13482
+ ""
13483
+ ];
13484
+ if (report.caveats.length) lines.push("## Caveats", "", ...report.caveats.map((item) => `- ${item}`), "");
13485
+ if (report.warnings.length) lines.push("## Warnings", "", ...report.warnings.map((item) => `- ${item}`), "");
13486
+ if (report.access.blockedRequests > 0) lines.push("## Access blocks", "", ...report.access.samples.flatMap((sample) => [
13487
+ `- ${sample.url}: ${sample.evidence.guidance.summary}`,
13488
+ ` - Action: ${sample.evidence.guidance.recommendedAction}`,
13489
+ ` - Security: ${sample.evidence.guidance.securityNote}`
13490
+ ]), "");
13491
+ for (const [index, fix] of fixes.entries()) lines.push(`## ${index + 1}. ${fix.title}`, "", `- [ ] Fix ${fix.count} affected URL${fix.count === 1 ? "" : "s"}`, `- Severity: ${fix.severity}`, `- Rule: ${fix.ruleId}`, `- Why this ranks: ${fix.whyThisRanks}`, `- Plain-English fix: ${fix.howToFix}`, `- Verify: ${fix.howToVerify}`, `- Command: ${fix.verification.command}`, "", `Affected URLs:`, ...fix.sampleUrls.slice(0, 10).map((url) => `- ${url}`), "");
13492
+ if (!fixes.length) lines.push("No implementation tickets were generated.");
13493
+ if (reviews.length) lines.push("## Review observations", "", "These are not implementation tickets. Check the evidence before deciding whether to work on them.", "", ...reviews.slice(0, 10).flatMap((observation) => [`- ${observation.title} (${observation.severity}, ${observation.count} URL${observation.count === 1 ? "" : "s"}): ${observation.howToFix}`, ` - First URL: ${observation.sampleUrls[0] ?? "No sample URL recorded"}`]), "");
13494
+ return `${lines.join("\n")}\n`;
13495
+ }
13496
+ function escapeHtml(value) {
13497
+ return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll("\"", "&quot;").replaceAll("'", "&#39;");
13498
+ }
13499
+ function escapeXml(value) {
13500
+ return escapeHtml(value);
13501
+ }
13220
13502
  // packages/core/src/analyze/crawler/site-checks-schema.ts
13221
13503
  const crawlSiteChecksSchema = z.object({ soft404: z.object({
13222
13504
  status: z.enum([
@@ -13643,6 +13925,7 @@ const crawlPageSnapshotSchema = z.object({
13643
13925
  source: z.enum([
13644
13926
  "environment",
13645
13927
  "playwright-cache",
13928
+ "agent-browser-cache",
13646
13929
  "system"
13647
13930
  ]),
13648
13931
  product: z.string(),
@@ -14346,14 +14629,6 @@ function agentReadyIssues(input) {
14346
14629
  }
14347
14630
  return issues;
14348
14631
  }
14349
- // packages/core/src/analyze/crawler/crawl-caveats.ts
14350
- function crawlCaveats(input) {
14351
- const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
14352
- if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
14353
- if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
14354
- if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
14355
- return caveats;
14356
- }
14357
14632
  // packages/core/src/analyze/crawler/crawl-control.ts
14358
14633
  const CRAWL_CANCELLED = Symbol("crawl-cancelled");
14359
14634
  function abortController(input) {
@@ -14395,83 +14670,94 @@ function crawlRssLimitBytes(totalMemoryBytes) {
14395
14670
  function crawlMemoryPressure(input) {
14396
14671
  return input.memoryUsage.heapUsed + input.memoryUsage.external >= crawlMemoryLimitBytes(input.totalMemoryBytes) || input.memoryUsage.rss >= crawlRssLimitBytes(input.totalMemoryBytes);
14397
14672
  }
14398
- // packages/core/src/analyze/crawler/crawl-url-queue.ts
14399
- function before(left, right) {
14400
- return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
14673
+ // packages/core/src/analyze/crawler/crawl-rate-limit.ts
14674
+ const RETRIES_PER_URL = 3;
14675
+ const BASE_BACKOFF_MS = 2e3;
14676
+ const MAX_BACKOFF_MS = 3e4;
14677
+ const COOLDOWN_BUDGET_MS = 18e4;
14678
+ const LIMITED_INTERVAL_MS = 1e3;
14679
+ const LIMITED_MAX_INTERVAL_MS = 4e3;
14680
+ function rateLimitEvidence(result, status, nowMs) {
14681
+ if (status !== 429 && status !== 503) return void 0;
14682
+ const header = retryAfterMs(result.page?.responseHeaders?.["retry-after"]);
14683
+ const cooldownUntil = Date.parse(result.page?.fetchDiagnostics?.backpressure?.cooldownUntil ?? "");
14684
+ const fromCooldown = Number.isFinite(cooldownUntil) ? Math.max(0, cooldownUntil - nowMs) : void 0;
14685
+ const wait = header ?? fromCooldown;
14686
+ if (status === 503 && wait === void 0) return void 0;
14687
+ return wait === void 0 ? {} : { retryAfterMs: Math.min(wait, MAX_BACKOFF_MS) };
14401
14688
  }
14402
- var CrawlUrlQueue = class {
14403
- #items = [];
14404
- #positions = /* @__PURE__ */ new Map();
14405
- get size() {
14406
- return this.#items.length;
14407
- }
14408
- has(url) {
14409
- return this.#positions.has(url);
14410
- }
14411
- push(item) {
14412
- if (this.has(item.url)) throw new Error(`URL is already queued: ${item.url}`);
14413
- const index = this.#items.length;
14414
- this.#items.push(item);
14415
- this.#positions.set(item.url, index);
14416
- this.#bubbleUp(index);
14417
- }
14418
- decreaseDepth(url, depth) {
14419
- const index = this.#positions.get(url);
14420
- if (index === void 0) return false;
14421
- const item = this.#items[index];
14422
- if (!item || depth >= item.depth) return false;
14423
- item.depth = depth;
14424
- this.#bubbleUp(index);
14425
- return true;
14426
- }
14427
- take() {
14428
- const first = this.#items[0];
14429
- if (!first) return void 0;
14430
- const last = this.#items.pop();
14431
- this.#positions.delete(first.url);
14432
- if (last && last !== first) {
14433
- this.#items[0] = last;
14434
- this.#positions.set(last.url, 0);
14435
- this.#bubbleDown(0);
14436
- }
14437
- return first;
14438
- }
14439
- #swap(left, right) {
14440
- const leftItem = this.#items[left];
14441
- const rightItem = this.#items[right];
14442
- if (!leftItem || !rightItem) return;
14443
- this.#items[left] = rightItem;
14444
- this.#items[right] = leftItem;
14445
- this.#positions.set(rightItem.url, left);
14446
- this.#positions.set(leftItem.url, right);
14447
- }
14448
- #bubbleUp(start) {
14449
- let index = start;
14450
- while (index > 0) {
14451
- const parent = Math.floor((index - 1) / 2);
14452
- const item = this.#items[index];
14453
- const parentItem = this.#items[parent];
14454
- if (!item || !parentItem || !before(item, parentItem)) break;
14455
- this.#swap(index, parent);
14456
- index = parent;
14457
- }
14689
+ /**
14690
+ * Rate-limit state for one crawl. After the first 429 the crawl drops to one
14691
+ * request at a time with a per-host interval and never speeds back up.
14692
+ */
14693
+ var CrawlRateLimit = class {
14694
+ rate;
14695
+ responses = 0;
14696
+ skippedUrls = 0;
14697
+ pendingMs = 0;
14698
+ spentMs = 0;
14699
+ #level = 0;
14700
+ #budgetSpent = false;
14701
+ #retries = /* @__PURE__ */ new Map();
14702
+ constructor(rate) {
14703
+ this.rate = rate;
14704
+ }
14705
+ get limited() {
14706
+ return this.responses > 0;
14707
+ }
14708
+ get intervalMs() {
14709
+ return this.rate.intervalMs ?? LIMITED_INTERVAL_MS;
14710
+ }
14711
+ /** The cool-down before probes that run after the crawl. */
14712
+ get settleMs() {
14713
+ return Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** this.#level);
14714
+ }
14715
+ /**
14716
+ * `retry` requeues the URL after `pendingMs`. `skip` gives it up, and
14717
+ * `stop` gives up the rest of the queue too once the budget is spent.
14718
+ */
14719
+ observe(url, result, status, nowMs) {
14720
+ const evidence = rateLimitEvidence(result, status, nowMs);
14721
+ if (!evidence) return void 0;
14722
+ this.responses += 1;
14723
+ if (this.pendingMs === 0) this.#level += 1;
14724
+ const step = 2 ** Math.max(0, this.#level - 1);
14725
+ const waitMs = evidence.retryAfterMs ?? Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * step);
14726
+ this.rate = {
14727
+ ...this.rate,
14728
+ concurrency: 1,
14729
+ intervalCap: 1,
14730
+ intervalMs: Math.min(LIMITED_MAX_INTERVAL_MS, Math.max(this.rate.intervalMs ?? 0, LIMITED_INTERVAL_MS * step))
14731
+ };
14732
+ const nextMs = Math.max(this.pendingMs, waitMs);
14733
+ if (this.spentMs + nextMs > COOLDOWN_BUDGET_MS) this.#budgetSpent = true;
14734
+ const retries = this.#retries.get(url) ?? 0;
14735
+ if (!this.#budgetSpent && retries < RETRIES_PER_URL) {
14736
+ this.#retries.set(url, retries + 1);
14737
+ this.pendingMs = nextMs;
14738
+ return "retry";
14739
+ }
14740
+ this.skippedUrls += 1;
14741
+ return this.#budgetSpent ? "stop" : "skip";
14742
+ }
14743
+ async settle(sleep, signal) {
14744
+ if (this.pendingMs <= 0) return;
14745
+ await sleep(this.pendingMs, signal);
14746
+ this.spentMs += this.pendingMs;
14747
+ this.pendingMs = 0;
14748
+ }
14749
+ caveatInput(config, readPages) {
14750
+ if (this.skippedUrls === 0) return void 0;
14751
+ return {
14752
+ readPages,
14753
+ skippedUrls: this.skippedUrls,
14754
+ rerunCommand: rateLimitRerunCommand(config, this.intervalMs)
14755
+ };
14458
14756
  }
14459
- #bubbleDown(start) {
14460
- let index = start;
14461
- while (true) {
14462
- const left = index * 2 + 1;
14463
- const right = left + 1;
14464
- let next = index;
14465
- const item = this.#items[next];
14466
- const leftItem = this.#items[left];
14467
- const rightItem = this.#items[right];
14468
- if (leftItem && item && before(leftItem, item)) next = left;
14469
- const nextItem = this.#items[next];
14470
- if (rightItem && nextItem && before(rightItem, nextItem)) next = right;
14471
- if (next === index) return;
14472
- this.#swap(index, next);
14473
- index = next;
14474
- }
14757
+ warning() {
14758
+ if (!this.limited) return void 0;
14759
+ const plural = (count, noun) => `${count} ${noun}${count === 1 ? "" : "s"}`;
14760
+ return `The origin rate limited ${plural(this.responses, "request")}. The crawl slowed to one request at a time, at least ${Math.round(this.intervalMs / 1e3)}s apart, and waited ${Math.round(this.spentMs / 1e3)}s in cool-downs${this.skippedUrls > 0 ? `, then left ${plural(this.skippedUrls, "URL")} unread` : " before reading every queued URL"}.`;
14475
14761
  }
14476
14762
  };
14477
14763
  // packages/core/src/analyze/crawler/external-link-checks.ts
@@ -14706,11 +14992,15 @@ function observationFromPage(requestedUrl, page) {
14706
14992
  };
14707
14993
  }
14708
14994
  // packages/core/src/analyze/crawler/site-checks.ts
14709
- const PROBE_COUNT = 2;
14995
+ const PROBE_COUNT = 3;
14710
14996
  function probeUrls(startUrl) {
14711
14997
  const origin = new URL(startUrl).origin;
14712
14998
  const seed = createHash("sha256").update(origin).digest("hex").slice(0, 12);
14713
- return Array.from({ length: PROBE_COUNT }, (_, index) => new URL(`/.well-known/seo-audit/not-found-${seed}-${index + 1}`, origin).toString());
14999
+ return [
15000
+ `/${seed}-not-found`,
15001
+ `/${seed}-not-found/${seed}-missing`,
15002
+ `/.well-known/seo-audit/not-found-${seed}-1`
15003
+ ].map((path) => new URL(path, origin).toString());
14714
15004
  }
14715
15005
  function probeObservation(url, result) {
14716
15006
  const request = result.request;
@@ -14730,29 +15020,40 @@ function probeObservation(url, result) {
14730
15020
  error: request.outcome === "failure" ? request.error : `Probe skipped: ${request.reason}`
14731
15021
  };
14732
15022
  }
15023
+ function isConclusive(probe) {
15024
+ return probe.status !== void 0 && !probe.accessBlocked && probe.status !== 429 && probe.status !== 503;
15025
+ }
14733
15026
  async function collectCrawlSiteChecks(input) {
15027
+ const probe = async (url) => probeObservation(url, await input.fetchStatusPage(url, {
15028
+ js: "off",
15029
+ refresh: true,
15030
+ timeoutMs: input.timeoutMs,
15031
+ signal: input.signal,
15032
+ writeCache: false,
15033
+ respectRobots: input.respectRobots,
15034
+ ...input.rate ? { rate: input.rate } : {}
15035
+ }));
14734
15036
  const probes = [];
14735
15037
  for (const url of probeUrls(input.startUrl)) {
14736
15038
  if (input.signal?.aborted) break;
14737
- const result = await input.fetchStatusPage(url, {
14738
- js: "off",
14739
- refresh: true,
14740
- timeoutMs: input.timeoutMs,
14741
- signal: input.signal,
14742
- writeCache: false,
14743
- respectRobots: input.respectRobots
14744
- });
14745
- probes.push(probeObservation(url, result));
15039
+ probes.push(await probe(url));
15040
+ }
15041
+ if (input.sleep && !input.signal?.aborted && probes.some((item) => !isConclusive(item))) {
15042
+ await input.sleep(input.retryDelayMs ?? 2e3, input.signal);
15043
+ for (const [index, item] of probes.entries()) {
15044
+ if (input.signal?.aborted) break;
15045
+ if (!isConclusive(item)) probes[index] = await probe(item.url);
15046
+ }
14746
15047
  }
14747
- const conclusive = probes.filter((probe) => probe.status !== void 0 && !probe.accessBlocked);
15048
+ const conclusive = probes.filter(isConclusive);
14748
15049
  const successful = conclusive.filter((probe) => (probe.status ?? 0) >= 200 && (probe.status ?? 0) < 400);
14749
15050
  const correctMissing = conclusive.filter((probe) => probe.status === 404 || probe.status === 410);
14750
- const status = conclusive.length !== PROBE_COUNT ? "unknown" : successful.length === PROBE_COUNT ? "warning" : correctMissing.length === PROBE_COUNT ? "pass" : "unknown";
15051
+ const status = successful.length > 0 ? "warning" : correctMissing.length === PROBE_COUNT ? "pass" : "unknown";
14751
15052
  const checks = { soft404: {
14752
15053
  status,
14753
15054
  probes,
14754
15055
  probeLimit: PROBE_COUNT,
14755
- complete: probes.length === PROBE_COUNT
15056
+ complete: conclusive.length === PROBE_COUNT
14756
15057
  } };
14757
15058
  if (status !== "warning") return {
14758
15059
  checks,
@@ -14768,7 +15069,7 @@ async function collectCrawlSiteChecks(input) {
14768
15069
  category: rule.category,
14769
15070
  severity: rule.defaultSeverity,
14770
15071
  url: new URL("/", input.startUrl).toString(),
14771
- detail: `${successful.length} known-nonexistent paths returned a successful response or redirect.`,
15072
+ detail: `${successful.length} of ${probes.length} made-up paths returned a successful response or redirect: ${successful.map((probe) => `${new URL(probe.url).pathname} (${probe.status})`).join(", ")}.`,
14772
15073
  evidence: checks.soft404
14773
15074
  }]
14774
15075
  };
@@ -17308,6 +17609,18 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
17308
17609
  }[connection.provider](connection);
17309
17610
  }
17310
17611
  // packages/core/src/analyze/crawler/site-crawl-dependencies.ts
17612
+ function sleep$2(ms, signal) {
17613
+ return new Promise((resolve) => {
17614
+ if (signal?.aborted) return resolve();
17615
+ const done = () => {
17616
+ clearTimeout(timer);
17617
+ signal?.removeEventListener("abort", done);
17618
+ resolve();
17619
+ };
17620
+ const timer = setTimeout(done, ms);
17621
+ signal?.addEventListener("abort", done, { once: true });
17622
+ });
17623
+ }
17311
17624
  function resolveCrawlSiteDependencies(dependencies = {}) {
17312
17625
  const hasInjectedPerPageSearchProvider = Boolean(dependencies.queryPageMetrics || dependencies.queryPageTopQuery);
17313
17626
  const hasInjectedLegacyBulkSearchProvider = Boolean(dependencies.queryPagesMetrics || dependencies.queryPagesTopQueries);
@@ -17328,6 +17641,7 @@ function resolveCrawlSiteDependencies(dependencies = {}) {
17328
17641
  fetchLandingPageValues: dependencies.fetchLandingPageValues ?? fetchLandingPageValues,
17329
17642
  landingValueForUrl: dependencies.landingValueForUrl ?? landingValueForUrl,
17330
17643
  now: dependencies.now ?? (() => /* @__PURE__ */ new Date()),
17644
+ sleep: dependencies.sleep ?? sleep$2,
17331
17645
  memoryUsage: dependencies.memoryUsage ?? (() => process.memoryUsage()),
17332
17646
  totalMemory: dependencies.totalMemory ?? totalmem
17333
17647
  };
@@ -17543,6 +17857,10 @@ async function crawlSite(input, dependencies) {
17543
17857
  const queueSafetyDeferredUrls = /* @__PURE__ */ new Set();
17544
17858
  const maxTrackedQueueSafetyUrls = Math.min(5e4, Math.max(1e3, config.maxPages * 5));
17545
17859
  let queueSafetyInventoryCapped = false;
17860
+ const queryVariantsByPath = /* @__PURE__ */ new Map();
17861
+ const queryVariantSkipped = /* @__PURE__ */ new Set();
17862
+ let renderingFallbackUrls = 0;
17863
+ const rateLimit = new CrawlRateLimit(config.fetchRate);
17546
17864
  let failedUrls = 0;
17547
17865
  let observedInternalLinks = 0;
17548
17866
  let memoryLimited = false;
@@ -17627,6 +17945,15 @@ async function crawlSite(input, dependencies) {
17627
17945
  });
17628
17946
  return;
17629
17947
  }
17948
+ if (isCdnUtilityUrl(normalized)) {
17949
+ recordSkip("cdn-utility-url");
17950
+ emitStatus("url_skipped", {
17951
+ url: normalized,
17952
+ depth,
17953
+ reason: "cdn_utility_url"
17954
+ });
17955
+ return;
17956
+ }
17630
17957
  if (isLikelyAsset(normalized)) {
17631
17958
  recordSkip("asset-url");
17632
17959
  emitStatus("url_skipped", {
@@ -17645,6 +17972,20 @@ async function crawlSite(input, dependencies) {
17645
17972
  });
17646
17973
  return;
17647
17974
  }
17975
+ const parsed = new URL(normalized);
17976
+ const pathKey = parsed.search ? `${parsed.origin}${parsed.pathname}` : "";
17977
+ if (pathKey && (queryVariantsByPath.get(pathKey) ?? 0) >= 5) {
17978
+ if (!queryVariantSkipped.has(normalized) && queryVariantSkipped.size < maxTrackedQueueSafetyUrls) {
17979
+ queryVariantSkipped.add(normalized);
17980
+ recordSkip("query-variant-limit");
17981
+ emitStatus("url_skipped", {
17982
+ url: normalized,
17983
+ depth,
17984
+ reason: "query_variant_limit"
17985
+ });
17986
+ }
17987
+ return;
17988
+ }
17648
17989
  if (queue.size + visited.size + inFlight.size >= config.maxPages * 5) {
17649
17990
  if (!queueSafetyDeferredUrls.has(normalized)) {
17650
17991
  if (queueSafetyDeferredUrls.size < maxTrackedQueueSafetyUrls) {
@@ -17660,6 +18001,7 @@ async function crawlSite(input, dependencies) {
17660
18001
  }
17661
18002
  return;
17662
18003
  }
18004
+ if (pathKey) queryVariantsByPath.set(pathKey, (queryVariantsByPath.get(pathKey) ?? 0) + 1);
17663
18005
  discovered.add(normalized);
17664
18006
  queuedUrls += 1;
17665
18007
  queue.push({
@@ -17716,7 +18058,7 @@ async function crawlSite(input, dependencies) {
17716
18058
  js: config.js,
17717
18059
  refresh: config.refresh,
17718
18060
  timeoutMs: config.timeoutMs,
17719
- rate: config.fetchRate,
18061
+ rate: rateLimit.rate,
17720
18062
  signal,
17721
18063
  writeCache: false,
17722
18064
  respectRobots: config.respectRobots,
@@ -17747,7 +18089,11 @@ async function crawlSite(input, dependencies) {
17747
18089
  });
17748
18090
  };
17749
18091
  while (!isCancelled() && (queue.size || inFlight.size) && (!memoryLimited || inFlight.size > 0) && pages.length < config.maxPages) {
17750
- while (!isCancelled() && !memoryLimited && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
18092
+ if (rateLimit.pendingMs > 0 && inFlight.size === 0) {
18093
+ await rateLimit.settle(deps.sleep, signal);
18094
+ if (isCancelled()) break;
18095
+ }
18096
+ while (!isCancelled() && !memoryLimited && rateLimit.pendingMs === 0 && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
17751
18097
  const item = nextQueueItem();
17752
18098
  if (!item) break;
17753
18099
  startTask(item);
@@ -17767,11 +18113,46 @@ async function crawlSite(input, dependencies) {
17767
18113
  error: result.warning ?? "Crawler returned no page snapshot.",
17768
18114
  extraction: "not-applicable"
17769
18115
  });
18116
+ const limited = rateLimit.observe(task.url, result, request.outcome === "response" ? request.status : void 0, deps.now().getTime());
18117
+ if (limited) {
18118
+ activeConcurrency = 1;
18119
+ healthyProbeStreak = 0;
18120
+ }
18121
+ if (limited === "retry") {
18122
+ visited.delete(task.url);
18123
+ if (!queue.has(task.url)) queue.push({
18124
+ url: task.url,
18125
+ depth: task.depth
18126
+ });
18127
+ emitStatus("url_queued", {
18128
+ url: task.url,
18129
+ depth: task.depth,
18130
+ reason: "rate_limited_retry",
18131
+ message: `The origin rate limited ${task.url}. Waiting ${Math.ceil(rateLimit.pendingMs / 1e3)}s, then retrying one request at a time.`
18132
+ });
18133
+ continue;
18134
+ }
18135
+ if (limited) {
18136
+ requests.push(request);
18137
+ recordSkip("origin-backpressure");
18138
+ emitStatus("page_skipped", {
18139
+ url: task.url,
18140
+ depth: task.depth,
18141
+ reason: "origin-backpressure"
18142
+ });
18143
+ if (limited === "stop") {
18144
+ let left = 0;
18145
+ while (queue.take()) left += 1;
18146
+ recordSkips("origin-backpressure", left);
18147
+ rateLimit.skippedUrls += left;
18148
+ }
18149
+ continue;
18150
+ }
17770
18151
  requests.push(request);
17771
18152
  if (healthStrategy) {
17772
18153
  if (request.outcome === "response" && request.status < 400 && !request.accessBlock) {
17773
18154
  healthyProbeStreak += 1;
17774
- activeConcurrency = Math.min(config.concurrency, 1 + Math.floor(healthyProbeStreak / 20));
18155
+ activeConcurrency = Math.min(rateLimit.limited ? 1 : config.concurrency, 1 + Math.floor(healthyProbeStreak / 20));
17775
18156
  } else {
17776
18157
  healthyProbeStreak = 0;
17777
18158
  activeConcurrency = 1;
@@ -17812,6 +18193,9 @@ async function crawlSite(input, dependencies) {
17812
18193
  continue;
17813
18194
  }
17814
18195
  const page = result.page;
18196
+ const rendering = page.fetchDiagnostics?.rendering;
18197
+ if (rendering?.status === "unavailable" && config.js === "on") throw new SeoError("INVALID_INPUT", `--rendering on needs a local Chrome or Chromium browser, and none could start. ${rendering.error ?? ""} Install Chromium, or set SEO_BROWSER_EXECUTABLE_PATH to a Chromium executable, then rerun. Use --rendering auto to accept raw HTML.`.replace(/\s+/g, " ").trim());
18198
+ if (rendering?.status === "unavailable" || rendering?.status === "failed") renderingFallbackUrls += 1;
17815
18199
  const finalUrl = normalizeUrl$1(page.finalUrl, config.url);
17816
18200
  if (!finalUrl) {
17817
18201
  failedUrls += 1;
@@ -17841,7 +18225,7 @@ async function crawlSite(input, dependencies) {
17841
18225
  const storeDocument = (urls) => {
17842
18226
  const sourceRequest = normalizeUrl$1(task.url, config.url) ?? task.url;
17843
18227
  const directlyRequested = sourceRequest === finalUrl;
17844
- const normalizedLinks = urls.map((url) => normalizeUrl$1(url, finalUrl)).filter((url) => Boolean(url));
18228
+ const normalizedLinks = urls.map((url) => normalizeUrl$1(url, finalUrl)).filter((url) => Boolean(url) && !isCdnUtilityUrl(url ?? ""));
17845
18229
  const syncLinkEvidence = (target, links) => {
17846
18230
  const sortedLinks = [...links].sort((left, right) => left < right ? -1 : left > right ? 1 : 0);
17847
18231
  target.outgoingInternalCount = sortedLinks.length;
@@ -17920,7 +18304,7 @@ async function crawlSite(input, dependencies) {
17920
18304
  if (followLinks && task.depth < config.maxDepth) {
17921
18305
  if (memoryLimited) for (const value of result.urls) {
17922
18306
  const normalized = normalizeUrl$1(value, finalUrl);
17923
- if (normalized && sameOrigin(normalized, origin) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
18307
+ if (normalized && sameOrigin(normalized, origin) && !isCdnUtilityUrl(normalized) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
17924
18308
  discovered.add(normalized);
17925
18309
  memoryDeferredUrls.add(normalized);
17926
18310
  }
@@ -17945,6 +18329,8 @@ async function crawlSite(input, dependencies) {
17945
18329
  cancelled = isCancelled();
17946
18330
  const robotsDisallowed = skipReasonCounts["robots-disallowed"] ?? 0;
17947
18331
  if (robotsDisallowed > 0) warnings.push(`${robotsDisallowed} URL${robotsDisallowed === 1 ? "" : "s"} disallowed ${SEO_CRAWLER_USER_AGENT} in robots.txt. If the audit should reach them, allow the SEO-Skill token on only those public paths.`);
18332
+ const rateLimitWarning = rateLimit.warning();
18333
+ if (rateLimitWarning) warnings.push(rateLimitWarning);
17948
18334
  if (cancelled) {
17949
18335
  warnings.push("Crawl cancelled; returning a partial report.");
17950
18336
  emitStatus("cancelled", {
@@ -17956,7 +18342,7 @@ async function crawlSite(input, dependencies) {
17956
18342
  const sitemapEvidencePartial = sitemapDiscovery?.dataStatus === "partial";
17957
18343
  const accessEvidencePartial = requests.some((request) => request.outcome === "response" && request.accessBlock);
17958
18344
  const requiredSitemapUnavailable = config.mode === "sitemap" && sitemapDiscovery?.dataStatus === "unavailable";
17959
- const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
18345
+ const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || rateLimit.skippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
17960
18346
  const failed = !cancelled && pages.length === 0 && (requiredSitemapUnavailable || requests.length > 0 && requests.every((request) => request.outcome === "failure"));
17961
18347
  const requestEvidenceStatus = cancelled && inFlight.size > 0 ? "partial" : "available";
17962
18348
  const dataSources = await crawlDataSources({
@@ -18003,7 +18389,10 @@ async function crawlSite(input, dependencies) {
18003
18389
  timeoutMs: config.timeoutMs,
18004
18390
  fetchStatusPage: deps.fetchStatusPage,
18005
18391
  respectRobots: config.respectRobots,
18006
- signal
18392
+ signal,
18393
+ rate: rateLimit.rate,
18394
+ sleep: deps.sleep,
18395
+ retryDelayMs: rateLimit.settleMs
18007
18396
  }) : void 0;
18008
18397
  for (const page of pages) {
18009
18398
  if (!page.geo || !llmsTxt) continue;
@@ -18034,6 +18423,12 @@ async function crawlSite(input, dependencies) {
18034
18423
  ...startPage.responseHeaders?.["content-signal"] ? { contentSignal: startPage.responseHeaders["content-signal"] } : {}
18035
18424
  } } : {}
18036
18425
  } : aiSignals?.signals;
18426
+ const reachIssues = followLinks && config.maxDepth > 0 && config.maxPages > 1 && !cancelled ? crawlReachIssues({
18427
+ startUrl: config.url,
18428
+ pages: pages.filter((page) => page.status >= 200 && page.status < 300),
18429
+ crawledPages: pages.length,
18430
+ sitemapUrlsReturned: sitemapDiscovery?.urlsReturned ?? 0
18431
+ }) : [];
18037
18432
  const report = createCrawlReport({
18038
18433
  config,
18039
18434
  site,
@@ -18048,6 +18443,7 @@ async function crawlSite(input, dependencies) {
18048
18443
  ...externalLinkVerification ? { externalLinkVerification } : {},
18049
18444
  ...siteCheckResult ? { siteChecks: siteCheckResult.checks } : {},
18050
18445
  additionalIssues: [
18446
+ ...reachIssues,
18051
18447
  ...siteCheckResult?.issues ?? [],
18052
18448
  ...sitemapDiscovery ? sitemapHostIssues(sitemapDiscovery, config.url) : [],
18053
18449
  ...ai ? agentReadyIssues({
@@ -18065,7 +18461,12 @@ async function crawlSite(input, dependencies) {
18065
18461
  queueSafetySkippedUrls,
18066
18462
  queueSafetyInventoryCapped,
18067
18463
  originBackpressureSkippedUrls,
18068
- memoryPressureSkippedUrls
18464
+ rateLimit: rateLimit.caveatInput(config, pages.length),
18465
+ sitemapLimit: sitemapLimitCaveatInput(config, sitemapDiscovery, input.maxPages === void 0),
18466
+ memoryPressureSkippedUrls,
18467
+ queryVariantSkippedUrls: queryVariantSkipped.size,
18468
+ renderingFallbackUrls,
18469
+ reachIssues
18069
18470
  }),
18070
18471
  stats: {
18071
18472
  discoveredUrls: discovered.size,
@@ -18620,7 +19021,7 @@ function analyzeCtrUnderperformersFromRows(input) {
18620
19021
  // packages/core/src/analyze/ctr-underperformers-report.ts
18621
19022
  const MAX_GSC_ROWS$3 = 1e5;
18622
19023
  const DEFAULT_DAYS$3 = 28;
18623
- const MAX_DAYS$2 = 548;
19024
+ const MAX_DAYS$3 = 548;
18624
19025
  function plural$8(count, singular, pluralLabel = `${singular}s`) {
18625
19026
  return count === 1 ? singular : pluralLabel;
18626
19027
  }
@@ -18639,12 +19040,12 @@ async function ctrUnderperformersReportWith(input, dependencies) {
18639
19040
  maximum: 100,
18640
19041
  label: "limit"
18641
19042
  });
18642
- const explicitRange = explicitDateRange(input, MAX_DAYS$2);
19043
+ const explicitRange = explicitDateRange(input, MAX_DAYS$3);
18643
19044
  const days = explicitRange?.days ?? integerOption$1({
18644
19045
  value: input.days,
18645
19046
  fallback: DEFAULT_DAYS$3,
18646
19047
  minimum: 1,
18647
- maximum: MAX_DAYS$2,
19048
+ maximum: MAX_DAYS$3,
18648
19049
  label: "Days"
18649
19050
  });
18650
19051
  const now = dependencies.now();
@@ -21885,7 +22286,7 @@ function quickWinTemplateSummaries(items) {
21885
22286
  }
21886
22287
  // packages/core/src/analyze/site-diagnostics/quick-wins-report.ts
21887
22288
  const DEFAULT_DAYS$2 = 28;
21888
- const MAX_DAYS$1 = 548;
22289
+ const MAX_DAYS$2 = 548;
21889
22290
  const DEFAULT_VERIFY_LIMIT$1 = 5;
21890
22291
  const MAX_VERIFY_LIMIT = 100;
21891
22292
  const MAX_SOURCE_ROWS$2 = 1e5;
@@ -21903,12 +22304,12 @@ function applyVerification$1(item, coverage) {
21903
22304
  };
21904
22305
  }
21905
22306
  async function quickWinsReportWith(input, dependencies) {
21906
- const explicitRange = explicitDateRange(input, MAX_DAYS$1);
22307
+ const explicitRange = explicitDateRange(input, MAX_DAYS$2);
21907
22308
  const days = explicitRange?.days ?? integerOption$1({
21908
22309
  value: input.days,
21909
22310
  fallback: DEFAULT_DAYS$2,
21910
22311
  minimum: 1,
21911
- maximum: MAX_DAYS$1,
22312
+ maximum: MAX_DAYS$2,
21912
22313
  label: "Days"
21913
22314
  });
21914
22315
  const verifyLimit = integerOption$1({
@@ -22327,7 +22728,7 @@ function analyzeStrikingDistanceRows(input) {
22327
22728
  }
22328
22729
  // packages/core/src/analyze/striking-distance-report.ts
22329
22730
  const DEFAULT_DAYS$1 = 28;
22330
- const MAX_DAYS = 548;
22731
+ const MAX_DAYS$1 = 548;
22331
22732
  const DEFAULT_VERIFY_LIMIT = 5;
22332
22733
  const MAX_SOURCE_ROWS$1 = 1e5;
22333
22734
  function boundedInteger$2(value, fallback, minimum, maximum) {
@@ -22336,7 +22737,7 @@ function boundedInteger$2(value, fallback, minimum, maximum) {
22336
22737
  }
22337
22738
  function reportDays(value) {
22338
22739
  if (value === void 0) return DEFAULT_DAYS$1;
22339
- if (!Number.isInteger(value) || value < 1 || value > MAX_DAYS) throw new SeoError("INVALID_INPUT", `Days must be a whole number between 1 and ${MAX_DAYS}.`);
22740
+ if (!Number.isInteger(value) || value < 1 || value > MAX_DAYS$1) throw new SeoError("INVALID_INPUT", `Days must be a whole number between 1 and ${MAX_DAYS$1}.`);
22340
22741
  return value;
22341
22742
  }
22342
22743
  function dateRange(days, now) {
@@ -22376,7 +22777,7 @@ function verdict$1(analysis) {
22376
22777
  return `${countLabel(analysis.summary.eligibleRows, "eligible query/page row")} found; ${countLabel(analysis.summary.returnedRows, "row")} returned for review.`;
22377
22778
  }
22378
22779
  async function strikingDistanceWith(input, dependencies) {
22379
- const explicitRange = explicitDateRange(input, MAX_DAYS);
22780
+ const explicitRange = explicitDateRange(input, MAX_DAYS$1);
22380
22781
  const days = explicitRange?.days ?? reportDays(input.days);
22381
22782
  const generatedAt = dependencies.now().toISOString();
22382
22783
  const range = explicitRange?.range ?? dateRange(days, new Date(generatedAt));
@@ -31789,6 +32190,7 @@ const wait = (ms) => new Promise((done) => setTimeout(done, ms));
31789
32190
  function requireInput(input) {
31790
32191
  const needsKeywords = input.action === "keyword_metrics" || input.action === "serp";
31791
32192
  if (needsKeywords && input.keywords.length === 0) throw new CommandError("INVALID_INPUT", `${input.action} needs keywords.`);
32193
+ if (input.action === "link_gap" && !input.target) throw new CommandError("INVALID_INPUT", "link_gap needs a target domain and competitors.");
31792
32194
  if (!needsKeywords && !input.target) throw new CommandError("INVALID_INPUT", `${input.action} needs a target domain or URL.`);
31793
32195
  }
31794
32196
  async function eachKeyword(keywords, missed, rowsFor) {
@@ -31806,14 +32208,17 @@ async function eachKeyword(keywords, missed, rowsFor) {
31806
32208
  }
31807
32209
  async function research(provider, headers, input) {
31808
32210
  requireInput(input);
32211
+ if (input.action === "link_gap") return linkGap(provider, headers, input);
31809
32212
  const keys = need(headers, provider);
31810
32213
  if (provider === "dataforseo") return dataForSeo(keys, input);
31811
32214
  const missed = [];
32215
+ const rows = await ahrefs(keys[0] ?? "", input, missed);
31812
32216
  return {
31813
- rows: await ahrefs(keys[0] ?? "", input, missed),
32217
+ rows,
31814
32218
  costUsd: null,
31815
32219
  partial: false,
31816
- missed
32220
+ missed,
32221
+ noData: withoutData(input, rows, missed, 100)
31817
32222
  };
31818
32223
  }
31819
32224
  function countryName(code) {
@@ -31865,7 +32270,8 @@ async function dataForSeoTask([login, password], path, task) {
31865
32270
  if (result?.status_code === NO_RESULTS) return {
31866
32271
  cost: result.cost ?? 0,
31867
32272
  result: null,
31868
- partial: false
32273
+ partial: false,
32274
+ noResults: true
31869
32275
  };
31870
32276
  if (!result || result.status_code >= 4e4 && result.status_code !== PARTIAL) throw Object.assign(new CommandError("PROVIDER_ERROR", `dataforseo: ${result?.status_message ?? "no task result"}`), { cost: result?.cost ?? 0 });
31871
32277
  return {
@@ -31906,18 +32312,29 @@ async function dataForSeo(keys, input) {
31906
32312
  partial: false,
31907
32313
  missed: []
31908
32314
  };
32315
+ const rows = await dataForSeoRows(keys, input, spent);
31909
32316
  return {
31910
- rows: await dataForSeoRows(keys, input, spent),
32317
+ rows,
31911
32318
  costUsd: Math.round(spent.cost * 1e4) / 1e4,
31912
32319
  partial: spent.partial,
31913
- missed: spent.missed
32320
+ missed: spent.missed,
32321
+ noData: withoutData(input, rows, spent.missed, 700)
31914
32322
  };
31915
32323
  }
32324
+ function withoutData(input, rows, missed, keywordLimit) {
32325
+ if (input.action === "keyword_metrics" || input.action === "serp") {
32326
+ const max = input.action === "serp" ? 5 : keywordLimit;
32327
+ const answered = new Set(rows.map((row) => String(row.keyword ?? "").toLowerCase()));
32328
+ const failed = new Set(missed);
32329
+ return input.keywords.slice(0, max).filter((keyword) => !failed.has(keyword) && !answered.has(keyword.toLowerCase()));
32330
+ }
32331
+ return [];
32332
+ }
31916
32333
  async function dataForSeoRows(keys, input, spent) {
31917
32334
  const language = dataForSeoLanguage(input.country);
31918
32335
  if (!language && input.action !== "serp" && input.action !== "backlinks") throw new CommandError("INVALID_INPUT", `DataForSEO has no ${input.action} data for country "${input.country}".`);
31919
32336
  const market = dataForSeoMarket(input.country, language);
31920
- const post = async (path, task) => {
32337
+ const post = async (path, task, attempts = 1) => {
31921
32338
  const done = await dataForSeoTask(keys, path, {
31922
32339
  ...market,
31923
32340
  ...task
@@ -31927,6 +32344,7 @@ async function dataForSeoRows(keys, input, spent) {
31927
32344
  });
31928
32345
  spent.cost += done.cost;
31929
32346
  spent.partial ||= done.partial;
32347
+ if (done.noResults && attempts > 1) return post(path, task, attempts - 1);
31930
32348
  return done.result?.items ?? [];
31931
32349
  };
31932
32350
  const get = (value, path) => path.split(".").reduce((current, key) => current && typeof current === "object" ? current[key] : void 0, value);
@@ -31956,7 +32374,7 @@ async function dataForSeoRows(keys, input, spent) {
31956
32374
  return (await post("v3/serp/google/organic/live/advanced", {
31957
32375
  keyword,
31958
32376
  depth: Math.min(input.limit, 100)
31959
- })).filter((item) => item.type === "organic").map((item) => {
32377
+ }, 2)).filter((item) => item.type === "organic").map((item) => {
31960
32378
  const row = pick(item, {
31961
32379
  position: "rank_absolute",
31962
32380
  url: "url",
@@ -31964,9 +32382,11 @@ async function dataForSeoRows(keys, input, spent) {
31964
32382
  title: "title"
31965
32383
  });
31966
32384
  if (/^https?:\/\/(www\.)?google\.[a-z.]+\/goto\b/.test(String(row.url))) {
31967
- const site = /^https?:\/\/([^/\s›]+)/.exec(String(item.breadcrumb ?? ""));
32385
+ const crumb = String(item.breadcrumb ?? "");
32386
+ const site = /^https?:\/\/([^/\s›]+)/.exec(crumb);
31968
32387
  row.url = null;
31969
32388
  row.domain = site?.[1] ?? null;
32389
+ row.url_hint = site ? crumb.replace(/^https?:\/\//, "") : null;
31970
32390
  }
31971
32391
  return {
31972
32392
  keyword,
@@ -31996,6 +32416,93 @@ async function dataForSeoRows(keys, input, spent) {
31996
32416
  organic_etv: "full_domain_metrics.organic.etv"
31997
32417
  }));
31998
32418
  }
32419
+ const GAP_DOMAINS = 1e3;
32420
+ const GAP_COMPETITORS = 5;
32421
+ const SHORTENERS = [
32422
+ "bit.ly",
32423
+ "tinyurl.com",
32424
+ "t.co",
32425
+ "ow.ly",
32426
+ "buff.ly",
32427
+ "is.gd",
32428
+ "cutt.ly",
32429
+ "rebrand.ly",
32430
+ "shorturl.at",
32431
+ "rb.gy",
32432
+ "lnkd.in"
32433
+ ];
32434
+ const FREE_APP_HOSTS = [
32435
+ "web.app",
32436
+ "firebaseapp.com",
32437
+ "pages.dev"
32438
+ ];
32439
+ const LINK_LIST_NAME = /(subdomain|whois|login|signin|coupon|promo|voucher|discount|sitelike|similarsite|sitesimilar|siteprice|sitevalue|siteworth|websiteworth|hypestat|statshow|websiteoutlook|siteinfo|domaininfo|domainstat|domaintools|ipaddress|dnslookup|linkdirectory|webdirectory|alternativeto-)/;
32440
+ function isLinkList(domain) {
32441
+ const within = (name) => domain === name || domain.endsWith(`.${name}`);
32442
+ if (SHORTENERS.some(within)) return true;
32443
+ if (FREE_APP_HOSTS.some((name) => domain.endsWith(`.${name}`))) return true;
32444
+ return LINK_LIST_NAME.test(domain.replace(/[-.]/g, ""));
32445
+ }
32446
+ async function linkGap(provider, headers, input) {
32447
+ const target = bareTarget(input.target).replace(/^www\./, "");
32448
+ const competitors = [...new Set((input.competitors ?? []).map((site) => bareTarget(site).replace(/^www\./, "")))].filter((site) => site && site !== target);
32449
+ if (competitors.length < 2 || competitors.length > GAP_COMPETITORS) throw new CommandError("INVALID_INPUT", `link_gap needs a target and 2 to ${GAP_COMPETITORS} competitors.`);
32450
+ const score = provider === "ahrefs" ? "domain_rating" : "rank";
32451
+ const sites = [target, ...competitors];
32452
+ const lists = [];
32453
+ let cost = 0;
32454
+ let partial = false;
32455
+ for (const site of sites) {
32456
+ const got = await research(provider, headers, {
32457
+ ...input,
32458
+ action: "backlinks",
32459
+ target: site,
32460
+ limit: GAP_DOMAINS
32461
+ });
32462
+ cost += got.costUsd ?? 0;
32463
+ partial ||= got.partial;
32464
+ lists.push(got.rows);
32465
+ }
32466
+ const domainOf = (row) => String(row.domain ?? "").toLowerCase().replace(/^www\./, "");
32467
+ const authority = (row) => Number(row[score] ?? 0);
32468
+ const floors = lists.filter((rows) => rows.length >= GAP_DOMAINS).map((rows) => Math.min(...rows.map(authority)));
32469
+ const floor = floors.length ? Math.max(...floors) : null;
32470
+ const own = new Set((lists[0] ?? []).map(domainOf));
32471
+ const found = /* @__PURE__ */ new Map();
32472
+ competitors.forEach((site, index) => {
32473
+ for (const row of lists[index + 1] ?? []) {
32474
+ const domain = domainOf(row);
32475
+ if (!domain || own.has(domain) || domain === target) continue;
32476
+ if (domain.endsWith(`.${target}`)) continue;
32477
+ const entry = found.get(domain) ?? {
32478
+ value: 0,
32479
+ sites: []
32480
+ };
32481
+ entry.value = Math.max(entry.value, authority(row));
32482
+ if (!entry.sites.includes(site)) entry.sites.push(site);
32483
+ found.set(domain, entry);
32484
+ }
32485
+ });
32486
+ const gap = [...found.entries()].filter(([, entry]) => entry.sites.length >= 2 && (floor === null || entry.value >= floor));
32487
+ const linkLists = gap.map(([domain]) => domain).filter(isLinkList).sort((a, b) => a.localeCompare(b));
32488
+ return {
32489
+ rows: gap.filter(([domain]) => !isLinkList(domain)).sort(([a, left], [b, right]) => right.value - left.value || right.sites.length - left.sites.length || a.localeCompare(b)).map(([domain, entry]) => ({
32490
+ domain,
32491
+ [score]: entry.value,
32492
+ competitors_linking: entry.sites.length,
32493
+ links_to: entry.sites.join(" ")
32494
+ })),
32495
+ costUsd: provider === "dataforseo" ? Math.round(cost * 1e4) / 1e4 : null,
32496
+ partial,
32497
+ missed: [],
32498
+ noData: [],
32499
+ ...floor !== null ? { floor: {
32500
+ field: score,
32501
+ value: floor
32502
+ } } : {},
32503
+ ...linkLists.length ? { leftOut: linkLists } : {}
32504
+ };
32505
+ }
31999
32506
  function isPage(target) {
32000
32507
  return /\/./.test(bareTarget(target));
32001
32508
  }
@@ -32086,7 +32593,7 @@ const DIMENSIONS = [
32086
32593
  "device",
32087
32594
  "searchAppearance"
32088
32595
  ];
32089
- const ROWS = 25e3;
32596
+ const ROWS$1 = 25e3;
32090
32597
  async function runRows(input, deps) {
32091
32598
  const dimensions = input.dimensions?.length ? input.dimensions : ["query"];
32092
32599
  const filters = input.filters ?? [];
@@ -32095,7 +32602,7 @@ async function runRows(input, deps) {
32095
32602
  const query = {
32096
32603
  dimensions,
32097
32604
  type: "web",
32098
- maxRows: ROWS,
32605
+ maxRows: ROWS$1,
32099
32606
  ...filters.length ? { dimensionFilterGroups: [{
32100
32607
  groupType: "and",
32101
32608
  filters
@@ -32152,9 +32659,9 @@ async function runRows(input, deps) {
32152
32659
  const sum = (key) => rows.reduce((total, row) => total + Number(row[key] ?? 0), 0);
32153
32660
  notes.push(`Totals: clicks ${sum("clicks")}, impressions ${sum("impressions")}; previous period clicks ${sum("previous_clicks")}.`);
32154
32661
  }
32155
- notes.push(`Compared with ${previous.startDate} to ${previous.endDate}${before.rowsFetched >= ROWS ? " (earlier period hit the row limit)" : ""}. 0 clicks with no position means the row had no impressions in the current period.`);
32662
+ notes.push(`Compared with ${previous.startDate} to ${previous.endDate}${before.rowsFetched >= ROWS$1 ? " (earlier period hit the row limit)" : ""}. 0 clicks with no position means the row had no impressions in the current period.`);
32156
32663
  if (pageAt >= 0) notes.push("Page URLs with #fragments are merged into their page.");
32157
- if (current.rowsFetched >= ROWS) notes.push(`Search Console has more than ${ROWS} rows; add a filter to narrow it.`);
32664
+ if (current.rowsFetched >= ROWS$1) notes.push(`Search Console has more than ${ROWS$1} rows; add a filter to narrow it.`);
32158
32665
  notes.push("Search Console hides some rare queries for privacy, so query rows do not add up to the site totals.");
32159
32666
  const detail = input.detail ?? "summary";
32160
32667
  return {
@@ -32263,6 +32770,22 @@ const hostOf$2 = (url) => {
32263
32770
  return null;
32264
32771
  }
32265
32772
  };
32773
+ const NOT_SOURCES = [
32774
+ "dataforseo.com",
32775
+ "images.openai.com",
32776
+ "gstatic.com",
32777
+ "googleusercontent.com",
32778
+ "ggpht.com",
32779
+ "example.com",
32780
+ "example.org",
32781
+ "example.net"
32782
+ ];
32783
+ const IMAGE = /\.(png|jpe?g|gif|webp|avif|svg)(\?|#|$)/i;
32784
+ const isSource = (url) => {
32785
+ const host = hostOf$2(url);
32786
+ if (!host || IMAGE.test(url)) return false;
32787
+ return !NOT_SOURCES.some((name) => host === name || host.endsWith(`.${name}`));
32788
+ };
32266
32789
  const empty = {
32267
32790
  cited_url: null,
32268
32791
  sources: [],
@@ -32291,7 +32814,7 @@ function judge(engine, task, site) {
32291
32814
  const cited = all.find((url) => onSite(url, site.domain)) ?? null;
32292
32815
  const base = {
32293
32816
  cited_url: cited,
32294
- sources: [...new Set(all.filter((url) => !onSite(url, site.domain)).map(hostOf$2).filter((host) => Boolean(host)))].slice(0, MAX_SOURCES),
32817
+ sources: [...new Set(all.filter((url) => !onSite(url, site.domain) && isSource(url)).map(hostOf$2).filter((host) => Boolean(host)))].slice(0, MAX_SOURCES),
32295
32818
  model: shown.model,
32296
32819
  answered_at
32297
32820
  };
@@ -32344,6 +32867,7 @@ async function runAiAnswers(input, headers, deps, more) {
32344
32867
  question,
32345
32868
  impressions,
32346
32869
  task: null,
32870
+ ref: null,
32347
32871
  cost_usd: 0,
32348
32872
  ...empty
32349
32873
  });
@@ -32355,7 +32879,8 @@ async function runAiAnswers(input, headers, deps, more) {
32355
32879
  ...blank("", null),
32356
32880
  engine: engine.name,
32357
32881
  result: "pending",
32358
- task: value
32882
+ task: value,
32883
+ ref: value
32359
32884
  };
32360
32885
  rows.push(row);
32361
32886
  waiting.push({
@@ -32408,6 +32933,7 @@ async function runAiAnswers(input, headers, deps, more) {
32408
32933
  }
32409
32934
  row.result = "pending";
32410
32935
  row.task = `${engine.name}:${task.id}`;
32936
+ row.ref = row.task;
32411
32937
  waiting.push({
32412
32938
  row,
32413
32939
  engine,
@@ -32448,6 +32974,7 @@ async function runAiAnswers(input, headers, deps, more) {
32448
32974
  const cost = usd(rows.reduce((sum, row) => sum + row.cost_usd, 0));
32449
32975
  const count = (result) => rows.filter((row) => row.result === result).length;
32450
32976
  const left = rows.filter((row) => row.task).map((row) => row.task);
32977
+ const refs = rows.flatMap((row) => row.ref ? [row.ref] : []);
32451
32978
  const cites = /* @__PURE__ */ new Map();
32452
32979
  for (const row of rows) for (const host of row.sources) cites.set(host, (cites.get(host) ?? 0) + 1);
32453
32980
  const topCited = [...cites].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10);
@@ -32481,7 +33008,7 @@ async function runAiAnswers(input, headers, deps, more) {
32481
33008
  limit: 50,
32482
33009
  detail: input.detail,
32483
33010
  notes,
32484
- more
33011
+ more: (offset) => more(offset, refs)
32485
33012
  }),
32486
33013
  data: {
32487
33014
  provider: "dataforseo",
@@ -32495,6 +33022,7 @@ async function runAiAnswers(input, headers, deps, more) {
32495
33022
  questions: [...new Set(rows.map((row) => row.question))],
32496
33023
  costUsd: cost,
32497
33024
  pendingTasks: left,
33025
+ tasks: refs,
32498
33026
  citedSites: topCited.map(([host, answers]) => ({
32499
33027
  host,
32500
33028
  answers
@@ -32614,7 +33142,7 @@ function ga4MatchReason(match, site) {
32614
33142
  }
32615
33143
  // packages/core/src/commands/analytics-site.ts
32616
33144
  const STREAM_CHECKS = 30;
32617
- const AT_ONCE = 5;
33145
+ const AT_ONCE$1 = 5;
32618
33146
  const TIED = 3;
32619
33147
  const hostOf$1 = (site) => {
32620
33148
  const value = site.replace(/^sc-domain:/, "");
@@ -32679,8 +33207,8 @@ async function lookProperty(access, site) {
32679
33207
  }
32680
33208
  const order = ranked(site, candidates).slice(0, STREAM_CHECKS);
32681
33209
  let failures = 0;
32682
- for (let start = 0; start < order.length; start += AT_ONCE) {
32683
- const batch = order.slice(start, start + AT_ONCE);
33210
+ for (let start = 0; start < order.length; start += AT_ONCE$1) {
33211
+ const batch = order.slice(start, start + AT_ONCE$1);
32684
33212
  const streams = await Promise.all(batch.map((candidate) => listAnalyticsDataStreams(candidate.google, candidate.property, { signIn: access.signIn }).catch(() => {
32685
33213
  failures += 1;
32686
33214
  return [];
@@ -32881,7 +33409,7 @@ async function totals$1(found, access, range, before, pages) {
32881
33409
  notes
32882
33410
  };
32883
33411
  }
32884
- const change$1 = (value) => value === null ? "n/a" : `${value > 0 ? "+" : ""}${value}%`;
33412
+ const change$2 = (value) => value === null ? "n/a" : `${value > 0 ? "+" : ""}${value}%`;
32885
33413
  function engineText(rows) {
32886
33414
  const shown = rows.filter((row) => row.sessions || row.previousSessions);
32887
33415
  if (shown.length < 2) return "";
@@ -32891,7 +33419,7 @@ function aiText(ai) {
32891
33419
  const list = (rows, key) => rows.slice(0, AI_ROWS).map((row) => `${row[key]} ${row.sessions} (was ${row.previousSessions})`).join(", ");
32892
33420
  if (!ai.sessions && !ai.previousSessions) return "Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): none in either period.";
32893
33421
  return [
32894
- `Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): ${ai.sessions} sessions (was ${ai.previousSessions}, ${change$1(ai.sessionsChangePct)}). By source: ${list(ai.bySource, "source")}.`,
33422
+ `Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): ${ai.sessions} sessions (was ${ai.previousSessions}, ${change$2(ai.sessionsChangePct)}). By source: ${list(ai.bySource, "source")}.`,
32895
33423
  ai.topLandingPages.length ? `Top landing pages from AI assistants: ${list(ai.topLandingPages, "page")}.` : "",
32896
33424
  "Only visits that pass a referrer or utm_source count; AI apps that send none show as Direct."
32897
33425
  ].filter(Boolean).join(" ");
@@ -33008,7 +33536,7 @@ async function analyticsBeside(access, site, range, before, pages) {
33008
33536
  const keyEvents = organic.keyEvents || organic.previousKeyEvents ? `, Organic Search key events ${organic.keyEvents} (was ${organic.previousKeyEvents})` : "";
33009
33537
  return {
33010
33538
  text: [
33011
- `Google Analytics, property ${data.propertyName} (${data.property}), matched by its web stream ${data.webStream}${pages ? `, landing pages whose path contains "${pages}"` : ""}: Organic Search sessions from Google ${google.sessions} (was ${google.previousSessions}, ${change$1(google.sessionsChangePct)}), from every search engine ${organic.sessions} (was ${organic.previousSessions}, ${change$1(organic.sessionsChangePct)})${keyEvents}, sessions from all channels ${all.sessions} (was ${all.previousSessions}, ${change$1(all.sessionsChangePct)}).`,
33539
+ `Google Analytics, property ${data.propertyName} (${data.property}), matched by its web stream ${data.webStream}${pages ? `, landing pages whose path contains "${pages}"` : ""}: Organic Search sessions from Google ${google.sessions} (was ${google.previousSessions}, ${change$2(google.sessionsChangePct)}), from every search engine ${organic.sessions} (was ${organic.previousSessions}, ${change$2(organic.sessionsChangePct)})${keyEvents}, sessions from all channels ${all.sessions} (was ${all.previousSessions}, ${change$2(all.sessionsChangePct)}).`,
33012
33540
  engineText(data.organicSearchByEngine),
33013
33541
  "Sessions are Google Analytics’ own count, not Search Console clicks.",
33014
33542
  ...data.notes,
@@ -33235,7 +33763,7 @@ function auditPreview(snapshot) {
33235
33763
  }
33236
33764
  const WEEK_MS = 6048e5;
33237
33765
  const day$1 = (ms) => ms === null ? "" : new Date(ms).toISOString().slice(0, 10);
33238
- const count$2 = (value) => value.toLocaleString("en-US");
33766
+ const count$3 = (value) => value.toLocaleString("en-US");
33239
33767
  function relative$2(value, origin) {
33240
33768
  if (!value) return "";
33241
33769
  return origin && value.startsWith(origin) ? value.slice(origin.length) || "/" : value;
@@ -33256,7 +33784,7 @@ function rowsTable(rows, checks) {
33256
33784
  changed(row) ? row.prev_coverage ?? "" : "",
33257
33785
  changed(row) ? day$1(row.changed_at) : "",
33258
33786
  day$1(row.checked_at),
33259
- `${count$2(row.clicks)} (${count$2(row.previous_clicks)})${row.lost ? " lost" : ""}`,
33787
+ `${count$3(row.clicks)} (${count$3(row.previous_clicks)})${row.lost ? " lost" : ""}`,
33260
33788
  row.in_sitemap ? "yes" : "no",
33261
33789
  row.google_canonical && row.google_canonical !== row.url ? relative$2(row.google_canonical, checks.origin) : "",
33262
33790
  row.error ?? ""
@@ -33264,12 +33792,12 @@ function rowsTable(rows, checks) {
33264
33792
  }
33265
33793
  function head(site, checks) {
33266
33794
  const { queued } = checks;
33267
- const quota = checks.properties.map((item) => `${item.property} (${count$2(item.urls)} URLs: ${count$2(item.top)} top pages checked daily, the rest every ${item.cycle_days === 1 ? "day" : `${item.cycle_days} days`}; ${count$2(item.used_today)} of ${count$2(checks.per_day)} checks used today)`).join(", ");
33795
+ const quota = checks.properties.map((item) => `${item.property} (${count$3(item.urls)} URLs: ${count$3(item.top)} top pages checked daily, the rest every ${item.cycle_days === 1 ? "day" : `${item.cycle_days} days`}; ${count$3(item.used_today)} of ${count$3(checks.per_day)} checks used today)`).join(", ");
33268
33796
  return [
33269
33797
  `Google indexing of ${site}, from the URL Inspection results this server keeps for it (as of ${new Date(checks.as_of).toISOString().slice(0, 16).replace("T", " ")} UTC).`,
33270
- checks.dropped_this_week || checks.returned_this_week ? `In the last 7 days ${count$2(checks.dropped_this_week)} ${checks.dropped_this_week === 1 ? "URL" : "URLs"} dropped out of Google's index and ${count$2(checks.returned_this_week)} came back; they are listed first. A top page's drop counts only after a second check an hour later.` : null,
33271
- `Covered: ${count$2(checks.covered)} of ${count$2(checks.known)} known URLs have a current result. Known URLs are the sitemap's, the crawl's and the Search Console pages with clicks or impressions. Each property spends its full ${count$2(checks.per_day)} checks a day: its top pages (lost clicks, then clicks, then impressions, up to ${count$2(checks.top_per_day)}) daily, the rest as often as the quota allows, and a page the weekly crawl saw change at once.`,
33272
- queued.total ? `Queued: ${count$2(queued.total)} (${count$2(queued.lost)} pages that lost their clicks, ${count$2(queued.search)} with clicks or impressions, ${count$2(queued.sitemap)} sitemap URLs never checked, ${count$2(queued.stale)} older or changed), checked in that order. Time left: ${etaText(checks.eta_hours)}, at up to ${count$2(checks.per_day)} checks a day per property, spread over the day.` : "Queued: nothing; every known URL has a current result.",
33798
+ checks.dropped_this_week || checks.returned_this_week ? `In the last 7 days ${count$3(checks.dropped_this_week)} ${checks.dropped_this_week === 1 ? "URL" : "URLs"} dropped out of Google's index and ${count$3(checks.returned_this_week)} came back; they are listed first. A top page's drop counts only after a second check an hour later.` : null,
33799
+ `Covered: ${count$3(checks.covered)} of ${count$3(checks.known)} known URLs have a current result. Known URLs are the sitemap's, the crawl's and the Search Console pages with clicks or impressions. Each property spends its full ${count$3(checks.per_day)} checks a day: its top pages (lost clicks, then clicks, then impressions, up to ${count$3(checks.top_per_day)}) daily, the rest as often as the quota allows, and a page the weekly crawl saw change at once.`,
33800
+ queued.total ? `Queued: ${count$3(queued.total)} (${count$3(queued.lost)} pages that lost their clicks, ${count$3(queued.search)} with clicks or impressions, ${count$3(queued.sitemap)} sitemap URLs never checked, ${count$3(queued.stale)} older or changed), checked in that order. Time left: ${etaText(checks.eta_hours)}, at up to ${count$3(checks.per_day)} checks a day per property, spread over the day.` : "Queued: nothing; every known URL has a current result.",
33273
33801
  quota ? `Checked through ${quota}.` : null,
33274
33802
  checks.stopped ? `Checks stopped early: ${checks.stopped}` : null,
33275
33803
  "A URL without a result has none here, so this is not an all-clear for it."
@@ -33278,9 +33806,9 @@ function head(site, checks) {
33278
33806
  function suggestionsText(checks) {
33279
33807
  if (!checks.suggestions.length) return null;
33280
33808
  return [
33281
- `The sitemap has more URLs than these properties can check every 7 days. Each Search Console property gets its own ${count$2(checks.per_day)} checks a day, so adding these folders as URL-prefix properties (on the same Google account) would check more:`,
33809
+ `The sitemap has more URLs than these properties can check every 7 days. Each Search Console property gets its own ${count$3(checks.per_day)} checks a day, so adding these folders as URL-prefix properties (on the same Google account) would check more:`,
33282
33810
  "property to add | sitemap URLs in it | more URLs checked a week",
33283
- ...checks.suggestions.map((item) => `${item.property} | ${count$2(item.urls)} | ${count$2(item.gain)}`),
33811
+ ...checks.suggestions.map((item) => `${item.property} | ${count$3(item.urls)} | ${count$3(item.gain)}`),
33284
33812
  "The next day the checks find a new property in Search Console and use its quota for the URLs in that folder."
33285
33813
  ].join("\n");
33286
33814
  }
@@ -33295,7 +33823,7 @@ function runAuditChecks(input, checks, call) {
33295
33823
  data: checks
33296
33824
  };
33297
33825
  if (!checks.started) return {
33298
- text: `${site} is in the weekly site audit, and its Google checks are starting now: pages that lost their clicks first, then pages by clicks and impressions, then sitemap URLs. ${checks.known ? `${count$2(checks.known)} URLs are known so far. ` : ""}Run ${call("index", { site })} again in a few minutes for the coverage and the time left.`,
33826
+ text: `${site} is in the weekly site audit, and its Google checks are starting now: pages that lost their clicks first, then pages by clicks and impressions, then sitemap URLs. ${checks.known ? `${count$3(checks.known)} URLs are known so far. ` : ""}Run ${call("index", { site })} again in a few minutes for the coverage and the time left.`,
33299
33827
  data: checks
33300
33828
  };
33301
33829
  const more = (offset) => input.paged ? `More: ${call("index", {
@@ -33319,7 +33847,7 @@ function runAuditChecks(input, checks, call) {
33319
33847
  const reasons = checks.reasons.length ? [
33320
33848
  "Google's reason at each URL's latest check:",
33321
33849
  "reason | verdict | urls | changed to it this week",
33322
- ...checks.reasons.map((item) => `${item.coverage ?? ""} | ${item.verdict ?? ""} | ${count$2(item.urls)} | ${count$2(item.changed_this_week)}`)
33850
+ ...checks.reasons.map((item) => `${item.coverage ?? ""} | ${item.verdict ?? ""} | ${count$3(item.urls)} | ${count$3(item.changed_this_week)}`)
33323
33851
  ].join("\n") : "No URL has a result yet.";
33324
33852
  const shown = checks.rows.slice(0, 20);
33325
33853
  return {
@@ -33376,7 +33904,7 @@ async function bingFor(deps, property) {
33376
33904
  missing: site ? `${site.url} is in the connected Bing account but is not verified, so Bing gives no data for it. Verify it at ${BING_HOME}.` : `The connected Bing account does not have ${property.replace(/^sc-domain:/, "")}. Add and verify it at ${BING_HOME} (it can import sites from Search Console). New sites take a few days to show data.`
33377
33905
  };
33378
33906
  }
33379
- function days(range) {
33907
+ function days$1(range) {
33380
33908
  return Math.round((Date.parse(range.endDate) - Date.parse(range.startDate)) / 864e5) + 1;
33381
33909
  }
33382
33910
  async function bingTotals(source, range, before) {
@@ -33389,7 +33917,7 @@ async function bingTotals(source, range, before) {
33389
33917
  clicks: rows.reduce((total, row) => total + row.clicks, 0),
33390
33918
  impressions: rows.reduce((total, row) => total + row.impressions, 0),
33391
33919
  days: new Set(rows.map((row) => row.date)).size,
33392
- expectedDays: days(window)
33920
+ expectedDays: days$1(window)
33393
33921
  };
33394
33922
  };
33395
33923
  return {
@@ -33399,14 +33927,14 @@ async function bingTotals(source, range, before) {
33399
33927
  invalidRows: traffic.invalidRows
33400
33928
  };
33401
33929
  }
33402
- const change = (now, before) => {
33930
+ const change$1 = (now, before) => {
33403
33931
  const value = pct(now, before);
33404
33932
  return value === null ? "n/a" : `${value > 0 ? "+" : ""}${value}%`;
33405
33933
  };
33406
33934
  function enginesText(google, bing) {
33407
33935
  if ("missing" in bing) return `Bing: ${bing.missing}`;
33408
33936
  const { current: now, previous: then } = bing;
33409
- const lines = [`Search engines, same dates. Google (Search Console): clicks ${google.clicks} (was ${google.previousClicks}, ${change(google.clicks, google.previousClicks)}), impressions ${google.impressions}. Bing (${bing.site}): clicks ${now.clicks} (was ${then.clicks}, ${change(now.clicks, then.clicks)}), impressions ${now.impressions} (was ${then.impressions}).`];
33937
+ const lines = [`Search engines, same dates. Google (Search Console): clicks ${google.clicks} (was ${google.previousClicks}, ${change$1(google.clicks, google.previousClicks)}), impressions ${google.impressions}. Bing (${bing.site}): clicks ${now.clicks} (was ${then.clicks}, ${change$1(now.clicks, then.clicks)}), impressions ${now.impressions} (was ${then.impressions}).`];
33410
33938
  const both = google.clicks + now.clicks;
33411
33939
  if (both > 0) lines.push(`Share of these clicks: Google ${Math.round(google.clicks / both * 100)}%, Bing ${Math.round(now.clicks / both * 100)}%.`);
33412
33940
  if (now.days < now.expectedDays || then.days < then.expectedDays) lines.push(`Bing has data for ${now.days} of ${now.expectedDays} days (earlier period ${then.days} of ${then.expectedDays}), so its totals are partial.`);
@@ -33560,7 +34088,7 @@ async function runBingRows(input, deps) {
33560
34088
  }
33561
34089
  };
33562
34090
  }
33563
- function moved(rows, range, before, name, limit) {
34091
+ function moved$1(rows, range, before, name, limit) {
33564
34092
  const now = sumWeeks(rows, range);
33565
34093
  const then = sumWeeks(rows, before);
33566
34094
  const old = new Map(then.rows.map((row) => [row.key, row]));
@@ -33598,8 +34126,8 @@ async function bingOverview(source, range, before) {
33598
34126
  ]);
33599
34127
  if ("missing" in totals) throw new CommandError("NOT_CONNECTED", totals.missing);
33600
34128
  const { current: now, previous: then } = totals;
33601
- const q = moved(queries.rows, range, before, "query", 15);
33602
- const p = moved(pages.rows, range, before, "page", 15);
34129
+ const q = moved$1(queries.rows, range, before, "query", 15);
34130
+ const p = moved$1(pages.rows, range, before, "page", 15);
33603
34131
  return {
33604
34132
  bingSite: source.site,
33605
34133
  range,
@@ -33650,7 +34178,7 @@ function bingOverviewText(report, heading, origin, rowsPerList) {
33650
34178
  };
33651
34179
  return [
33652
34180
  `${heading}, ${range.startDate} to ${range.endDate} against ${before.startDate} to ${before.endDate}.`,
33653
- t.days || t.previousDays ? `Bing totals: clicks ${t.clicks} (was ${t.previousClicks}, ${change(t.clicks, t.previousClicks)}), impressions ${t.impressions} (was ${t.previousImpressions}, ${change(t.impressions, t.previousImpressions)}).` : "Bing totals: unknown. Bing has no daily data for these dates.",
34181
+ t.days || t.previousDays ? `Bing totals: clicks ${t.clicks} (was ${t.previousClicks}, ${change$1(t.clicks, t.previousClicks)}), impressions ${t.impressions} (was ${t.previousImpressions}, ${change$1(t.impressions, t.previousImpressions)}).` : "Bing totals: unknown. Bing has no daily data for these dates.",
33654
34182
  group("pages", "page"),
33655
34183
  group("queries", "query"),
33656
34184
  `Limits:\n${report.caveats.map((line) => `- ${line}`).join("\n")}`
@@ -33727,13 +34255,13 @@ function clickyRange(input) {
33727
34255
  endDate: input.endDate
33728
34256
  };
33729
34257
  }
33730
- const count$1 = (item) => {
34258
+ const count$2 = (item) => {
33731
34259
  const value = Number(item.value);
33732
34260
  return Number.isSafeInteger(value) && value >= 0 ? value : null;
33733
34261
  };
33734
34262
  const totals = (items) => items.map((item) => ({
33735
34263
  name: item.title ?? "",
33736
- visits: count$1(item)
34264
+ visits: count$2(item)
33737
34265
  })).filter((item) => Boolean(item.name) && item.visits !== null);
33738
34266
  const line = (items) => {
33739
34267
  const top = items.slice(0, 10).map((item) => `${item.name} ${item.visits}`);
@@ -33764,7 +34292,7 @@ async function runClickyRows(input, keys, options) {
33764
34292
  const entrances = stats["pages-entrance"] ?? [];
33765
34293
  const visitsByPage = /* @__PURE__ */ new Map();
33766
34294
  for (const item of entrances) {
33767
- const visits = count$1(item);
34295
+ const visits = count$2(item);
33768
34296
  if (visits === null || !item.url) continue;
33769
34297
  let page;
33770
34298
  try {
@@ -33813,7 +34341,8 @@ async function runClickyRows(input, keys, options) {
33813
34341
  // packages/core/src/commands/domain-rating.ts
33814
34342
  const DR_ATTRIBUTION = "domain_rating: Domain Rating by Ahrefs (https://ahrefs.com/).";
33815
34343
  const ENDPOINT = "https://api.ahrefs.com/v3/public/domain-rating-free";
33816
- const MAX_DOMAINS = 25;
34344
+ const MAX_DOMAINS = 50;
34345
+ const AT_ONCE = 25;
33817
34346
  const TTL_SECONDS = 86400;
33818
34347
  const DOMAIN_FIELDS = [
33819
34348
  "domain",
@@ -33844,23 +34373,26 @@ async function lookup(domain, key) {
33844
34373
  }
33845
34374
  return body.domain_rating?.domain_rating ?? null;
33846
34375
  }
33847
- async function domainRatings(domains, key) {
34376
+ async function domainRatings(domains, key, max = MAX_DOMAINS) {
33848
34377
  const found = /* @__PURE__ */ new Map();
33849
34378
  if (!key) return found;
33850
- const unique = [...new Set(domains.map(bareDomain).filter(Boolean))].slice(0, MAX_DOMAINS);
33851
- const values = await Promise.all(unique.map((domain) => lookup(domain, key).catch(() => null)));
33852
- unique.forEach((domain, index) => {
33853
- found.set(domain, values[index] ?? null);
33854
- });
34379
+ const unique = [...new Set(domains.map(bareDomain).filter(Boolean))].slice(0, max);
34380
+ for (let start = 0; start < unique.length; start += AT_ONCE) {
34381
+ const batch = unique.slice(start, start + AT_ONCE);
34382
+ const values = await Promise.all(batch.map((domain) => lookup(domain, key).catch(() => null)));
34383
+ batch.forEach((domain, index) => {
34384
+ found.set(domain, values[index] ?? null);
34385
+ });
34386
+ }
33855
34387
  return found;
33856
34388
  }
33857
- async function withDomainRating(rows, key) {
34389
+ async function withDomainRating(rows, key, start = 0) {
33858
34390
  const field = DOMAIN_FIELDS.find((name) => rows.some((row) => typeof row[name] === "string"));
33859
34391
  if (!key || !field || rows.some((row) => "domain_rating" in row)) return {
33860
34392
  rows,
33861
34393
  added: false
33862
34394
  };
33863
- const ratings = await domainRatings(rows.map((row) => String(row[field] ?? "")), key);
34395
+ const ratings = await domainRatings(rows.slice(start).map((row) => String(row[field] ?? "")), key);
33864
34396
  return {
33865
34397
  rows: rows.map((row) => {
33866
34398
  const next = {};
@@ -35035,6 +35567,265 @@ function temporaryRedirectNote(pages) {
35035
35567
  if (!temporary.length) return null;
35036
35568
  return `Note: 302 and 307 are temporary redirects; a permanent move usually uses 301 or 308. Temporary redirects here: ${temporary.map((page) => `${page.url} (${page.status} to ${page.redirects_to})`).join("; ")}.`;
35037
35569
  }
35570
+ // packages/core/src/commands/updates.ts
35571
+ const UPDATES_SECTION = "google-updates";
35572
+ const UPDATES_SECTION_TEXT = "One Google ranking update (core, spam and similar, from Google’s Search Status Dashboard) laid over the site’s Search Console data: clicks and impressions from the day it started against the same number of days just before (whole weeks, up to 28 days), whether that change is outside the site’s own movement in the 8 comparisons before it (a heuristic), and the folders, pages and queries that gained or lost most. The latest update by default; with end_date, the latest update that started on or before it. Timing only, not cause.";
35573
+ const MAX_DAYS = 28;
35574
+ const BASELINE = 8;
35575
+ const ROWS = 25e3;
35576
+ const LISTED = 10;
35577
+ const SHOWN = 5;
35578
+ const MIN_CLICKS = 50;
35579
+ const days = (from, to) => Math.round((Date.parse(to) - Date.parse(from)) / 864e5) + 1;
35580
+ const asImpactUpdate = (update) => ({
35581
+ id: update.id,
35582
+ name: update.name,
35583
+ start: update.start,
35584
+ end: update.end ?? null,
35585
+ url: update.sourceUrl
35586
+ });
35587
+ function sum(rows, from, to) {
35588
+ let clicks = 0;
35589
+ let impressions = 0;
35590
+ for (const row of rows) {
35591
+ const day = row.keys[0] ?? "";
35592
+ if (day < from || day > to) continue;
35593
+ clicks += row.clicks;
35594
+ impressions += row.impressions;
35595
+ }
35596
+ return {
35597
+ clicks,
35598
+ impressions
35599
+ };
35600
+ }
35601
+ const change = (now, before) => before > 0 ? (now - before) / before * 100 : null;
35602
+ function joined(before, after) {
35603
+ const byKey = /* @__PURE__ */ new Map();
35604
+ const at = (key) => {
35605
+ let row = byKey.get(key);
35606
+ if (!row) {
35607
+ row = {
35608
+ key,
35609
+ clicks: 0,
35610
+ previous: 0,
35611
+ impressions: 0,
35612
+ previous_impressions: 0
35613
+ };
35614
+ byKey.set(key, row);
35615
+ }
35616
+ return row;
35617
+ };
35618
+ for (const row of before) {
35619
+ const item = at(row.keys[0] ?? "");
35620
+ item.previous += row.clicks;
35621
+ item.previous_impressions += row.impressions;
35622
+ }
35623
+ for (const row of after) {
35624
+ const item = at(row.keys[0] ?? "");
35625
+ item.clicks += row.clicks;
35626
+ item.impressions += row.impressions;
35627
+ }
35628
+ return [...byKey.values()];
35629
+ }
35630
+ const moved = (row) => row.clicks - row.previous;
35631
+ const byKey = (a, b) => a.key < b.key ? -1 : a.key > b.key ? 1 : 0;
35632
+ function extremes(rows) {
35633
+ return {
35634
+ lost: rows.filter((row) => moved(row) < 0).sort((a, b) => moved(a) - moved(b) || byKey(a, b)).slice(0, LISTED),
35635
+ gained: rows.filter((row) => moved(row) > 0).sort((a, b) => moved(b) - moved(a) || byKey(a, b)).slice(0, LISTED)
35636
+ };
35637
+ }
35638
+ function folderOf(url) {
35639
+ try {
35640
+ const parts = new URL(url).pathname.split("/").filter(Boolean);
35641
+ return parts.length > 1 ? `/${parts[0]}/` : "/";
35642
+ } catch {
35643
+ return "/";
35644
+ }
35645
+ }
35646
+ function folders(pages) {
35647
+ const byFolder = /* @__PURE__ */ new Map();
35648
+ for (const page of pages) {
35649
+ const key = folderOf(page.key);
35650
+ const row = byFolder.get(key) ?? {
35651
+ key,
35652
+ clicks: 0,
35653
+ previous: 0,
35654
+ impressions: 0,
35655
+ previous_impressions: 0
35656
+ };
35657
+ row.clicks += page.clicks;
35658
+ row.previous += page.previous;
35659
+ row.impressions += page.impressions;
35660
+ row.previous_impressions += page.previous_impressions;
35661
+ byFolder.set(key, row);
35662
+ }
35663
+ return [...byFolder.values()].filter((row) => moved(row) !== 0).sort((a, b) => Math.abs(moved(b)) - Math.abs(moved(a)) || byKey(a, b)).slice(0, LISTED);
35664
+ }
35665
+ function windowDays(update, latest) {
35666
+ const start = update.start.slice(0, 10);
35667
+ const end = update.end?.slice(0, 10) ?? null;
35668
+ const wanted = Math.min(MAX_DAYS, Math.max(7, Math.ceil(days(start, end ?? latest) / 7) * 7));
35669
+ const covered = Math.floor(Math.max(0, days(start, latest)) / 7) * 7;
35670
+ return {
35671
+ start,
35672
+ end,
35673
+ wanted,
35674
+ n: Math.min(wanted, covered)
35675
+ };
35676
+ }
35677
+ async function updateImpact(site, update, read, now) {
35678
+ const { start, end, wanted, n } = windowDays(update, latestFinalGscDate(now));
35679
+ const empty = {
35680
+ update,
35681
+ site,
35682
+ before: null,
35683
+ after: null,
35684
+ normal_range: null,
35685
+ verdict: "not-enough-data",
35686
+ capped: {
35687
+ pages: false,
35688
+ queries: false
35689
+ },
35690
+ folders: [],
35691
+ pages: {
35692
+ lost: [],
35693
+ gained: []
35694
+ },
35695
+ queries: {
35696
+ lost: [],
35697
+ gained: []
35698
+ }
35699
+ };
35700
+ if (n < 7) return {
35701
+ ...empty,
35702
+ window: "waiting"
35703
+ };
35704
+ const after = {
35705
+ startDate: start,
35706
+ endDate: shiftIsoDate(start, n - 1)
35707
+ };
35708
+ const before = {
35709
+ startDate: shiftIsoDate(start, -n),
35710
+ endDate: shiftIsoDate(start, -1)
35711
+ };
35712
+ const first = Math.ceil(n / 7);
35713
+ const pairs = Array.from({ length: BASELINE }, (_, i) => {
35714
+ const back = 7 * (first + i);
35715
+ return {
35716
+ after: [shiftIsoDate(start, -back), shiftIsoDate(start, n - 1 - back)],
35717
+ before: [shiftIsoDate(start, -back - n), shiftIsoDate(start, -back - 1)]
35718
+ };
35719
+ });
35720
+ const shared = sharedReader(read);
35721
+ const call = (range, dimension) => shared(site, {
35722
+ ...range,
35723
+ dimensions: [dimension],
35724
+ type: "web",
35725
+ dataState: "final",
35726
+ ...dimension === "date" ? {} : { maxRows: ROWS }
35727
+ });
35728
+ const [daily, pagesBefore, pagesAfter, queriesBefore, queriesAfter] = await Promise.all([
35729
+ call({
35730
+ startDate: pairs.at(-1)?.before[0],
35731
+ endDate: after.endDate
35732
+ }, "date"),
35733
+ call(before, "page"),
35734
+ call(after, "page"),
35735
+ call(before, "query"),
35736
+ call(after, "query")
35737
+ ]);
35738
+ const totals = (range) => ({
35739
+ ...range,
35740
+ ...sum(daily.rows, range.startDate, range.endDate)
35741
+ });
35742
+ const was = totals(before);
35743
+ const is = totals(after);
35744
+ const seen = pairs.map((pair) => change(sum(daily.rows, pair.after[0], pair.after[1]).clicks, sum(daily.rows, pair.before[0], pair.before[1]).clicks)).filter((value) => value !== null);
35745
+ const normal = seen.length ? {
35746
+ comparisons: seen.length,
35747
+ clicks_pct: Math.round(Math.max(...seen.map((v) => Math.abs(v))) * 10) / 10
35748
+ } : null;
35749
+ const moves = change(is.clicks, was.clicks);
35750
+ const verdict = !normal || normal.comparisons < BASELINE / 2 || was.clicks < MIN_CLICKS || moves === null ? "not-enough-data" : Math.abs(moves) > normal.clicks_pct ? "outside-normal" : "within-normal";
35751
+ const pageRows = joined(pagesBefore.rows, pagesAfter.rows);
35752
+ return {
35753
+ ...empty,
35754
+ window: end && n === wanted ? "full" : "partial",
35755
+ before: was,
35756
+ after: is,
35757
+ normal_range: normal,
35758
+ verdict,
35759
+ capped: {
35760
+ pages: pagesBefore.rows.length >= ROWS || pagesAfter.rows.length >= ROWS,
35761
+ queries: queriesBefore.rows.length >= ROWS || queriesAfter.rows.length >= ROWS
35762
+ },
35763
+ folders: folders(pageRows),
35764
+ pages: extremes(pageRows),
35765
+ queries: extremes(joined(queriesBefore.rows, queriesAfter.rows))
35766
+ };
35767
+ }
35768
+ const count$1 = (n) => n.toLocaleString("en-US");
35769
+ const dayLabel$1 = (date) => (/* @__PURE__ */ new Date(`${date}T00:00:00Z`)).toLocaleDateString("en-GB", {
35770
+ day: "numeric",
35771
+ month: "short",
35772
+ year: "numeric",
35773
+ timeZone: "UTC"
35774
+ });
35775
+ function trend(now, before) {
35776
+ const pct = change(now, before);
35777
+ if (pct === null) return `from ${count$1(before)}`;
35778
+ const rounded = Math.round(pct);
35779
+ return rounded === 0 ? `level with ${count$1(before)}` : `${rounded > 0 ? "up" : "down"} ${Math.abs(rounded)}% on ${count$1(before)}`;
35780
+ }
35781
+ function rowText(row, origin) {
35782
+ const name = row.key.startsWith(origin) ? row.key.slice(origin.length) || "/" : row.key;
35783
+ const diff = moved(row);
35784
+ return `${name} ${count$1(row.clicks)} (${diff > 0 ? "+" : ""}${count$1(diff)})`;
35785
+ }
35786
+ function impactLines(impact) {
35787
+ const { update, before, after } = impact;
35788
+ const origin = originOf(impact.site);
35789
+ const dates = `${dayLabel$1(update.start.slice(0, 10))} to ${update.end ? dayLabel$1(update.end.slice(0, 10)) : "now, still rolling out"}`;
35790
+ const lines = [`${update.name} (${dates}). [Google’s notice](${update.url})`];
35791
+ if (!before || !after) return [...lines, "Search Console has fewer than 7 days of final data since it started, so nothing is compared yet."];
35792
+ const n = days(after.startDate, after.endDate);
35793
+ lines.push(`Compared: ${dayLabel$1(after.startDate)} to ${dayLabel$1(after.endDate)} against the ${n} days before (same weekdays)${update.end && update.end.slice(0, 10) > after.endDate && impact.window === "full" ? `, the first ${n} days of the update` : ""}.${impact.window === "partial" ? " Final data does not cover the whole update yet, so this is part of it." : ""}`, `Clicks ${count$1(after.clicks)}, ${trend(after.clicks, before.clicks)}. Impressions ${count$1(after.impressions)}, ${trend(after.impressions, before.impressions)}.`);
35794
+ const normal = impact.normal_range;
35795
+ lines.push(impact.verdict === "not-enough-data" ? `Not enough data to say whether this is unusual for the site${before.clicks < MIN_CLICKS ? ` (${count$1(before.clicks)} clicks before the update)` : ""}.` : `${impact.verdict === "outside-normal" ? "Outside" : "Within"} the site’s normal movement: in the ${normal?.comparisons} earlier ${n}-day comparisons, clicks moved by at most ${normal?.clicks_pct}% (a heuristic).`);
35796
+ const list = (title, rows) => rows.length ? [`${title}: ${rows.slice(0, SHOWN).map((row) => rowText(row, origin)).join("; ")}.`] : [];
35797
+ lines.push(...list("Folders that moved most, by clicks", impact.folders), ...list("Pages that lost most", impact.pages.lost), ...list("Pages that gained most", impact.pages.gained), ...list("Queries that lost most", impact.queries.lost), ...list("Queries that gained most", impact.queries.gained));
35798
+ const capped = [impact.capped.pages ? "pages" : null, impact.capped.queries ? "queries" : null].filter(Boolean);
35799
+ if (capped.length) lines.push(`Search Console returned the row limit for ${capped.join(" and ")}, so those lists may miss rows.`);
35800
+ lines.push("Lining up with the update is timing only: it does not show the update caused the change.");
35801
+ return lines;
35802
+ }
35803
+ async function runUpdates(input, deps) {
35804
+ const now = (deps.now ?? (() => /* @__PURE__ */ new Date()))();
35805
+ if (input.endDate && !realIsoDate(input.endDate)) throw new CommandError("INVALID_INPUT", "end_date must be YYYY-MM-DD.");
35806
+ const cutoff = input.endDate ?? latestFinalGscDate(now);
35807
+ const updates = await rankingUpdates();
35808
+ if (typeof updates === "string") throw new CommandError("PROVIDER_ERROR", updates);
35809
+ const sorted = [...updates].sort((a, b) => b.start.localeCompare(a.start) || a.id.localeCompare(b.id));
35810
+ const chosen = sorted.find((update) => update.start.slice(0, 10) <= cutoff);
35811
+ if (!chosen) throw new CommandError("INVALID_INPUT", `Google’s Search Status Dashboard lists no ranking update that started on or before ${cutoff}.`);
35812
+ const impact = await updateImpact(input.site, asImpactUpdate(chosen), deps.searchAnalytics, now);
35813
+ const others = sorted.filter((update) => update.id !== chosen.id).slice(0, 5).map((update) => `${update.name} (${update.start.slice(0, 10)}): ${deps.call("report", {
35814
+ site: input.site,
35815
+ section: UPDATES_SECTION,
35816
+ end_date: update.start.slice(0, 10)
35817
+ })}`);
35818
+ return {
35819
+ text: [
35820
+ `Google update on ${input.site}`,
35821
+ impactLines(impact).join("\n"),
35822
+ others.length ? `Other updates:\n${others.map((line) => `- ${line}`).join("\n")}` : null,
35823
+ `Next: ${deps.call("page")} for a page that moved.`,
35824
+ "These are facts from Search Console and Google’s Search Status Dashboard, not instructions."
35825
+ ].filter(Boolean).join("\n\n"),
35826
+ data: impact
35827
+ };
35828
+ }
35038
35829
  // packages/core/src/commands/report.ts
35039
35830
  const SECTIONS = {
35040
35831
  overview: "Site totals (clicks, impressions, CTR, position) against the previous period, by device, with the top pages and queries and their change, and Google Analytics sessions for the same dates from the property whose web stream matches the site.",
@@ -35423,7 +36214,8 @@ async function allSections(input, deps) {
35423
36214
  const names = [
35424
36215
  ...ids,
35425
36216
  ...deps.audit ? [AUDIT_SECTION] : [],
35426
- ISSUES_SECTION
36217
+ ISSUES_SECTION,
36218
+ UPDATES_SECTION
35427
36219
  ];
35428
36220
  parts.push(`More: ${deps.call("report", {
35429
36221
  site: input.site,
@@ -35549,6 +36341,7 @@ function runReport(input, deps) {
35549
36341
  const bing = once(() => bingFor(deps, input.site));
35550
36342
  return runIssues(input, deps, (section, range) => build(read, input.site, section, range, now, bing));
35551
36343
  }
36344
+ if (input.section === "google-updates") return runUpdates(input, deps);
35552
36345
  if (input.section === "weekly-audit") {
35553
36346
  const audit = deps.audit;
35554
36347
  if (!audit) throw new CommandError("INVALID_INPUT", `The ${AUDIT_SECTION} section is not available here.`);
@@ -35587,10 +36380,11 @@ const RESEARCH_ACTIONS = [
35587
36380
  "serp",
35588
36381
  "competitors",
35589
36382
  "backlinks",
36383
+ "link_gap",
35590
36384
  "ai_answers"
35591
36385
  ];
35592
36386
  const RESEARCH_PROVIDERS = ["dataforseo", "ahrefs"];
35593
- const RESEARCH_HELP = `keyword_metrics (keywords: volume, difficulty, CPC), ranked_keywords (target: keywords a domain or URL ranks for), serp (keywords, up to 5: the ranking pages), competitors (target: domains that share keywords), backlinks (target: referring domains), ai_answers (DataForSEO only; target: the site. Without keywords it takes the site’s top 10 chat-style and long questions from Search Console’s AI-shaped queries itself. With up to 10 questions in keywords it asks those instead, and the target can be any domain, such as a competitor's. Each answer row also lists the other sites it cites, and the notes rank them. Answers come from DataForSEO's queue: answers not ready after about 40 seconds are returned as tasks, already paid for; call again with the same target and those tasks to read them at no cost. It asks ChatGPT search, Google AI Mode and Google AI Overviews each one and says whether the answer cites or mentions the site. DataForSEO’s queue prices are about $0.0072 a question across the three engines, so about $0.07 a run: tell the person before running it). Rows with a domain, and the target, get a free Ahrefs domain_rating column. Numbers are provider estimates, not Search Console data.`;
36387
+ const RESEARCH_HELP = `keyword_metrics (keywords: volume, difficulty, CPC), ranked_keywords (target: keywords a domain or URL ranks for), serp (keywords, up to 5: the ranking pages), competitors (target: domains that share keywords), backlinks (target: referring domains), link_gap (target and 2 to 5 competitors: referring domains that link to two or more competitors and not the target, highest authority first, from each site's top 1,000 referring domains; costs one backlinks call per site), ai_answers (DataForSEO only; target: the site. Without keywords it takes the site’s top 10 chat-style and long questions from Search Console’s AI-shaped queries itself. With up to 10 questions in keywords it asks those instead, and the target can be any domain, such as a competitor's. Each answer row also lists the other sites it cites, and the notes rank them. Answers come from DataForSEO's queue: answers not ready after about 40 seconds are returned as tasks, already paid for; call again with the same target and those tasks to read them at no cost. It asks ChatGPT search, Google AI Mode and Google AI Overviews each one and says whether the answer cites or mentions the site. DataForSEO’s queue prices are about $0.0072 a question across the three engines, so about $0.07 a run: tell the person before running it). Rows with a domain, and the target, get a free Ahrefs domain_rating column. Numbers are provider estimates, not Search Console data.`;
35594
36388
  const GENERIC_PLATFORMS = [
35595
36389
  "youtube.com",
35596
36390
  "facebook.com",
@@ -35634,10 +36428,11 @@ async function runResearch(input, keys, options) {
35634
36428
  const offset = input.offset ?? 0;
35635
36429
  const language = input.provider === "dataforseo" ? dataForSeoLanguage(country) : null;
35636
36430
  if (input.action === "ai_answers") throw new CommandError("INVALID_INPUT", "ai_answers runs on its own.");
35637
- const { rows: found, costUsd, partial, missed } = await research(input.provider, keys, {
36431
+ const { rows: found, costUsd, partial, missed, noData, floor, leftOut } = await research(input.provider, keys, {
35638
36432
  action: input.action,
35639
36433
  keywords: input.keywords ?? [],
35640
36434
  target,
36435
+ competitors: input.competitors ?? [],
35641
36436
  country,
35642
36437
  limit: Math.min(1e3, Math.max(100, offset + 50))
35643
36438
  });
@@ -35645,8 +36440,10 @@ async function runResearch(input, keys, options) {
35645
36440
  rows: found,
35646
36441
  note: null
35647
36442
  };
36443
+ const unresolved = rows.filter((row) => row.url === null && row.url_hint !== void 0).length;
35648
36444
  const drKey = keys.get("x-ahrefs-key") ?? options.drKey;
35649
- const rated = await withDomainRating(rows, drKey);
36445
+ const byDr = input.action === "link_gap" && input.provider === "dataforseo" && drKey;
36446
+ const rated = byDr ? await sortedByDomainRating(rows, drKey) : await withDomainRating(rows, drKey, offset);
35650
36447
  const targetDr = target ? (await domainRatings([target], drKey)).get(bareDomain(target)) : void 0;
35651
36448
  const hasTargetDr = targetDr !== void 0 && targetDr !== null;
35652
36449
  return {
@@ -35661,6 +36458,11 @@ async function runResearch(input, keys, options) {
35661
36458
  ...costUsd !== null ? [`DataForSEO cost: $${costUsd}.`] : [],
35662
36459
  ...partial ? ["Partial: DataForSEO could not fetch some pages of results, so rows may be missing."] : [],
35663
36460
  ...missed.length ? [`Partial: ${input.provider} failed for ${missed.join(", ")}, so those keywords have no rows. Ask for them again.`] : [],
36461
+ ...noData.length ? [noDataNote(input, noData)] : [],
36462
+ ...byDr ? [rows.length > GAP_RATED ? `Sorted by Domain Rating for the top ${GAP_RATED} rows by DataForSEO rank; the ${rows.length - GAP_RATED} rows after them follow by rank, with no Domain Rating.` : "Sorted by Domain Rating, then DataForSEO rank."] : [],
36463
+ ...leftOut?.length ? [`Left out ${leftOut.length} link lists, URL shorteners and directories that link to most sites (a guess from the host name): ${leftOut.join(", ")}.`] : [],
36464
+ ...floor ? [`Cut at ${floor.field} ${floor.value}: at least one site has more than 1,000 referring domains, and only its top 1,000 were read, so domains below ${floor.field} ${floor.value} are left out rather than guessed.`] : [],
36465
+ ...unresolved ? [unresolvedNote(input, unresolved, rows.length)] : [],
35664
36466
  ...note ? [note] : [],
35665
36467
  ...hasTargetDr ? [`${bareDomain(target)} domain_rating: ${targetDr}.`] : [],
35666
36468
  ...rated.added || hasTargetDr ? [DR_ATTRIBUTION] : []
@@ -35679,12 +36481,44 @@ async function runResearch(input, keys, options) {
35679
36481
  costUsd,
35680
36482
  partial,
35681
36483
  missedKeywords: missed,
36484
+ noDataKeywords: noData,
36485
+ ...input.action === "link_gap" ? {
36486
+ competitors: input.competitors ?? [],
36487
+ cutAt: floor ?? null,
36488
+ leftOut: leftOut ?? []
36489
+ } : {},
35682
36490
  total: rated.rows.length,
35683
36491
  offset,
35684
36492
  rows: rated.rows.slice(offset, offset + 50)
35685
36493
  }
35686
36494
  };
35687
36495
  }
36496
+ function noDataNote(input, keywords) {
36497
+ const list = keywords.join(", ");
36498
+ if (input.action === "serp") return `No results: ${input.provider} got an empty results page for ${list}${input.provider === "dataforseo" ? ", twice" : ""}. Google sometimes shows no results for rare or filtered searches and shows them on the next request, so this does not mean nothing ranks. Ask again later${input.provider === "dataforseo" ? ", or use provider ahrefs" : ""}.`;
36499
+ return `No data: ${input.provider} has no metrics for ${keywords.length} of the keywords asked for (${list}). Their volume is unknown, not zero.`;
36500
+ }
36501
+ function unresolvedNote(input, count, total) {
36502
+ return `${count} of ${total} results have no exact page: ${input.provider} returned Google redirect links for them. Their domain and url_hint come from Google's breadcrumb, which is shortened.${input.provider === "dataforseo" ? " For exact pages, use provider ahrefs." : ""}`;
36503
+ }
36504
+ const GAP_RATED = 200;
36505
+ async function sortedByDomainRating(rows, key) {
36506
+ const top = rows.slice(0, GAP_RATED);
36507
+ const ratings = await domainRatings(top.map((row) => String(row.domain ?? "")), key, GAP_RATED);
36508
+ const withDr = (row, rated) => {
36509
+ const { domain = null, ...rest } = row;
36510
+ return {
36511
+ domain,
36512
+ domain_rating: rated ? ratings.get(bareDomain(String(domain ?? ""))) ?? null : null,
36513
+ ...rest
36514
+ };
36515
+ };
36516
+ const dr = (row) => Number(row.domain_rating ?? -1);
36517
+ return {
36518
+ rows: [...top.map((row) => withDr(row, true)).sort((a, b) => dr(b) - dr(a) || Number(b.rank ?? 0) - Number(a.rank ?? 0) || String(a.domain).localeCompare(String(b.domain))), ...rows.slice(GAP_RATED).map((row) => withDr(row, false))],
36519
+ added: true
36520
+ };
36521
+ }
35688
36522
  // packages/core/src/commands/sitemap-urls.ts
35689
36523
  const MAX_FILES = 20;
35690
36524
  const MAX_URLS = 5e4;
@@ -36234,6 +37068,23 @@ function siteLines(site, startMs, endMs) {
36234
37068
  } else lines.push("Search clicks: not available this week.");
36235
37069
  return lines;
36236
37070
  }
37071
+ function updateLines(site) {
37072
+ return (site.google_updates ?? []).map((update) => {
37073
+ const when = update.end ? "ran" : "has been rolling out";
37074
+ const result = update.result;
37075
+ if (!result && update.ready) return `- Google update: the ${update.name} ran ${update.start.slice(0, 10)} to ${update.end?.slice(0, 10) ?? "now"}. Ask for the report's google-updates section on this site for what moved.`;
37076
+ if (!result) return `- Google update: the ${update.name} ${when}${update.end ? ` (${update.start.slice(0, 10)} to ${update.end.slice(0, 10)})` : ` since ${update.start.slice(0, 10)}`}. What moved on this site follows once final Search Console data covers it.`;
37077
+ const diff = result.clicks - result.previous;
37078
+ const pct = result.previous > 0 ? Math.round(diff / result.previous * 100) : null;
37079
+ const moved = pct === null ? `${count(result.clicks)} clicks from ${count(result.previous)}` : pct === 0 ? `clicks level at ${count(result.clicks)}` : `clicks ${pct > 0 ? "up" : "down"} ${Math.abs(pct)}% (${count(result.clicks)} from ${count(result.previous)})`;
37080
+ const verdict = {
37081
+ "outside-normal": "more than this site usually moves",
37082
+ "within-normal": "within this site’s normal movement",
37083
+ "not-enough-data": "too little data to say if that is unusual"
37084
+ }[result.verdict];
37085
+ return `- Google update: the ${update.name}, ${moved} over the ${result.days} days from its start against the ${result.days} before, ${verdict} (timing only, not cause).`;
37086
+ });
37087
+ }
36237
37088
  function opportunityLines(site) {
36238
37089
  const found = site.opportunities;
36239
37090
  if (!found) return [];
@@ -36286,7 +37137,7 @@ function weekMarkdown(data) {
36286
37137
  "",
36287
37138
  attention.length ? total === 1 ? "Your site had changes worth a look this week." : `${attention.length} of your ${total} sites had changes worth a look this week.` : quiet.length ? total === 1 ? "Your site had a quiet week." : quiet.length === total ? `All ${total} of your sites had a quiet week.` : `${quiet.length} of your ${total} sites had a quiet week.` : "Your sites are still on their first checks."
36288
37139
  ];
36289
- for (const { site, found } of rows) out.push("", `## ${siteName(site.property)}${found.length ? `: ${list(found.map((item) => item.heading))}` : ""}`, "", ...siteLines(site, startMs, endMs).map((line) => `- ${line}`), ...opportunityLines(site), ...measuredLines(site), ...site.link ? [`- [View site](${site.link})`] : []);
37140
+ for (const { site, found } of rows) out.push("", `## ${siteName(site.property)}${found.length ? `: ${list(found.map((item) => item.heading))}` : ""}`, "", ...siteLines(site, startMs, endMs).map((line) => `- ${line}`), ...opportunityLines(site), ...measuredLines(site), ...updateLines(site), ...site.link ? [`- [View site](${site.link})`] : []);
36290
37141
  return out.join("\n");
36291
37142
  }
36292
37143
  async function runAllSites(deps, sites) {
@@ -36389,10 +37240,14 @@ const SHARED_TOOLS = [
36389
37240
  name: "seo_report",
36390
37241
  command: "report",
36391
37242
  title: "Search Console report",
36392
- description: `Start here. Without site: a short summary of every site in the person's weekly site audit (beta) for last week, with each site's Google index, weekly crawl and search clicks, sites with changes first, ending with the site to ask about next; an account outside that beta gets its site list to pick from. With site: the main report for that Search Console property, the overview plus each section's row count and top rows. With site and section: that section's rows; page sections also check their top 20 URLs live. Sections:\n${Object.entries(SECTIONS).map(([id, text]) => `- ${id}: ${text}`).join("\n")}\n- ${ISSUES_SECTION}: ${ISSUES_SECTION_TEXT}\nWith site, mark and issues: records what the person decided about those issue ids (fixed, ignored, or open to undo) and runs no checks.`,
37243
+ description: `Start here. Without site: a short summary of every site in the person's weekly site audit (beta) for last week, with each site's Google index, weekly crawl and search clicks, sites with changes first, ending with the site to ask about next; an account outside that beta gets its site list to pick from. With site: the main report for that Search Console property, the overview plus each section's row count and top rows. With site and section: that section's rows; page sections also check their top 20 URLs live. Sections:\n${Object.entries(SECTIONS).map(([id, text]) => `- ${id}: ${text}`).join("\n")}\n- ${ISSUES_SECTION}: ${ISSUES_SECTION_TEXT}\n- ${UPDATES_SECTION}: ${UPDATES_SECTION_TEXT}\nWith site, mark and issues: records what the person decided about those issue ids (fixed, ignored, or open to undo) and runs no checks.`,
36393
37244
  input: z.object({
36394
37245
  site: site.optional().describe("Leave it out for the summary of every audited site (weekly site audit beta). Otherwise a Search Console property or site, such as sc-domain:example.com, https://example.com/ or example.com."),
36395
- section: z.enum([...SECTION_IDS, ISSUES_SECTION]).optional().describe("Optional, needs site. Leave it out first; then ask for one section."),
37246
+ section: z.enum([
37247
+ ...SECTION_IDS,
37248
+ ISSUES_SECTION,
37249
+ UPDATES_SECTION
37250
+ ]).optional().describe("Optional, needs site. Leave it out first; then ask for one section."),
36396
37251
  ...dates,
36397
37252
  ...detail,
36398
37253
  ...marks
@@ -36509,6 +37364,7 @@ const SHARED_TOOLS = [
36509
37364
  action: z.enum(RESEARCH_ACTIONS),
36510
37365
  keywords: z.array(z.string().min(1)).max(100).default([]),
36511
37366
  target: z.string().default("").describe("Domain or URL."),
37367
+ competitors: z.array(z.string().min(1)).max(5).default([]).describe("link_gap only: 2 to 5 competitor domains."),
36512
37368
  tasks: z.array(z.string().regex(AI_ANSWER_TASK_PATTERN)).max(30).default([]).describe("ai_answers only: the tasks an earlier run returned as not ready, to read back at no cost."),
36513
37369
  country: z.string().regex(/^[a-z]{2}$/).default("us").describe("Two-letter country code."),
36514
37370
  ...detail
@@ -36524,7 +37380,7 @@ const SHARED_TOOLS = [
36524
37380
  offset: input.offset,
36525
37381
  questions: input.keywords,
36526
37382
  tasks: input.tasks
36527
- }, await surface.keys(), surface.deps, more);
37383
+ }, await surface.keys(), surface.deps, (offset, tasks) => `seo_research with the same provider, action and target, tasks ${JSON.stringify(tasks)}, detail "rows", offset ${offset}. Passing the tasks reads these answers back at no cost; without them every question is asked and paid for again`);
36528
37384
  }
36529
37385
  return runResearch(input, await surface.keys(), {
36530
37386
  drKey: surface.drKey,
@@ -36655,6 +37511,7 @@ SHARED_TOOLS.map((shared) => shared.name === "seo_index" ? indexFromStore(shared
36655
37511
  input: shared.input.extend({ section: z.enum([
36656
37512
  ...SECTION_IDS,
36657
37513
  ISSUES_SECTION,
37514
+ UPDATES_SECTION,
36658
37515
  AUDIT_SECTION
36659
37516
  ]).optional().describe("Optional. Leave it out first; then ask for one section.") })
36660
37517
  } : shared);
@@ -39367,6 +40224,6 @@ function isoDate(daysAgo = 0) {
39367
40224
  date.setUTCDate(date.getUTCDate() - daysAgo);
39368
40225
  return date.toISOString().slice(0, 10);
39369
40226
  }
39370
- export { AHREFS_API_KEY_ENV, AHREFS_API_KEY_SECRET, AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, ANALYTICS_PROVIDER_IDS, AUDIT_PAGE_RULE_IDS, AhrefsClient, BING_API_KEY_ENV, BING_API_KEY_SECRET, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_IMPRESSIONS, CANNIBAL_MINIMUM_PAGE_SHARE, CLICKY_SITEKEY_ENV, CLICKY_SITE_KEYS_SECRET, CLOUD_SIGN_IN, CLOUD_URL, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DATAFORSEO_CREDENTIAL_SECRET, DATAFORSEO_LOGIN_ENV, DATAFORSEO_PASSWORD_ENV, DEFAULT_MAX_PAGES, DEFAULT_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, DEFAULT_UNREACHED_PAGE_LIMIT, DataForSeoClient, DecisionModelError, DecisionRunner, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, GoogleApiError, INCIDENTS_URL, JsonFileStorageAdapter, LINK_ROLES, LINT_MAX_FILES, LOCAL_BING_CONNECT, LOCAL_SIGN_IN, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MANAGED_PROVIDER_SECRET_NAMES, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, MAX_MAX_PAGES, MAX_SEARCH_CONSOLE_EXPORT_FILES, MAX_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, PAGE_TYPES, PROVIDER_ERROR_CODES, PROVIDER_SECRET_NAMES, ProviderError, ProviderExtensionRegistry, RESEARCH_IMPORT_COLUMN_KEYS, ResponseSizeLimitError, SEARCH_INTENTS, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_PROVIDER_API_VERSION, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SITEMAP_FILE_URL_LIMIT, SITE_CHECK_MAX_ISSUES, SITE_CHECK_MAX_PAGES, STEP_CHANGE_RULES, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_FAILURE_CONTEXTS, TELEMETRY_FAILURE_REASONS, TELEMETRY_NOTICE, TELEMETRY_OPERATIONS, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, TYPESAFE_API_KEY_ENV, agentActionsView, aiReferralSourceForValue, aiReferralsReport, analyticsConnectionLabel, analyticsConnectionProviderId, analyticsConnectionSchema, analyticsProviderDetails, analyticsProviderSchema, analyzeAiReferralRows, analyzeCannibalImportRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, auditCrawlPages, auditCrawlRequests, auditPage, authStatus, bearerFetch, boundedSitemapInventory, buildAgentFindings, buildQueryClusterReportFromRows, buildTelemetryPayload, builtPagePath, cacheDb, cannibalDataStatus, cannibalImportReport, cannibalReport, cannibalReportWith, changeWindow, checkBuiltSite, checkDatabaseReadiness, classifyTelemetryFailure, clearCache, cloudAccountFor, cloudBing, cloudCredentials, cloudGoogleEmails, cloudGoogleToken, cloudKeys, cloudLinked, cloudLogout, cloudWeek, clusterQueryRows, combineRobotsValues, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, ctrUnderperformersReportWith, decayClusterDrafts, decayComparisonRange, decayingReport, decayingReportWith, deleteAhrefsApiKey, deleteBingApiKey, deleteClickySiteKey, deleteDataForSeoCredentials, deleteManagedProviderSecrets, deleteProviderExtensionAccount, deleteProviderExtensionCredentials, deleteProviderSecret, deleteTokens, deriveBrandTerms, describeInstalledProviderExtension, detectAgentName, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensurePrivateDatabaseFiles, ensureSeoCliDirs, estimateSerpTitleWidth, expectedCtrForPosition, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fileSize, finalGscDateRange, findOverlappingSearchUpdates, finishCloudLogin, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, getAuthModeStatus, getCacheStats, getClientConfig, getDb, getPrivacySnapshot, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleJson, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoverageSignals, inferTemplateMovement, initializeTelemetry, inspectProviderPackage, inspectSearchConsoleUrl, inspectUrl, installProviderPackage, installedProviderPackageSchema, installedProviderPackagesSchema, internalLinksReport, internalLinksReportWith, interpretUpdateCorrelation, isBrandQuery, isNewerVersion, isSkippableReportError, isTelemetryCi, isTelemetryOperation, isTelemetryReport, isoDate, isoWeek, latestFinalGscDate, linkRecover, linkRecoverRecommendation, lintPaths, listAnalyticsAccounts, listAnalyticsDataStreams, listGa4AccountSummaries, listGa4DataStreams, listGoogleAccounts, listInstalledProviderExtensions, listRules, listSearchConsoleSites, listSearchUpdates, listSites, loadInstalledProviderExtensions, loadProviderExtensions, loadProviderPackage, loadSearchConsoleExport, localAccounts, localBing, localCredentialsProvider, localError, localGoogleFetch, localInternalLinksDependencies, localRefreshClicky, localResearchKeys, loginWithLoopback, makeRunId, matchGa4WebStreams, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, nodeCommandDeps, normalizeBrandText, normalizeClickySiteId, normalizeClickySiteKey, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, normalizeProviderExtensionAccount, normalizedResearchColumnName, notSignedIn, noteCacheWrite, openCloudLink, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, pendingCloudLogin, performanceAudit, performanceReportIsCacheable, plural, providerCapabilitySchema, providerConnectionFieldSchema, providerExactVersionSchema, providerExtensionIdSchema, providerIdSchema, providerIntegritySchema, providerKindSchema, providerPackageManifestSchema, providerPackageNameSchema, providerPreferenceSchema, providerRegistrationSchema, providerRequestJson, providerRequestText, pruneLogs, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, quickWinsReportWith, readAhrefsApiKey, readAllTokens, readBingApiKey, readBoundedResponseText, readClickySiteKey, readClickySiteKeys, readConfig, readDataForSeoCredentials, readInstalledProviderPackages, readJson, readJsonOrUndefined, readLintFiles, readOauthClient, readProviderExtensionAccount, readProviderExtensionCredentials, readProviderSecret, readTokens, recommendationForRule, reconcileExportPagesWithCrawl, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, registrableDomain, removeInstalledProviderPackage, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderWorkflowMarkdown, reportNarrative, requestHeaders, requirePageVerification, researchImportColumns, resetSeoData, resolveLocalSite, resolveSegmentRanges, resolveSeoCliPaths, reviewObservations, roundedPosition, runAnalyticsProviderLandingPages, runAnalyticsReport, runDoctor, runGa4Report, runProviderExtensionAction, runtimeForProvider, safeProviderAdapterError, saveInstalledProviderPackage, searchAnalyticsRows, searchUpdatesFromIncidents, secondPage, secondPageWith, segmentComparisonRange, segmentImpact, segmentRangeDays, seoErrorEnvelope, sessionSourceInListFilter, setActiveGoogleAccount, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteStartUrl, snapshotFetchedPage, startCloudLogin, strikingDistance, strikingDistanceWith, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, telemetryErrorCategory, toSeoError, tokenSchema, tokenStoreSchema, topFixes, trackTelemetryCommandFailed, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, trafficAnomalyWith, typesafeDecisionModel, unavailableSegmentImpactReport, unicodeTokens, uninstallProviderExtension, updateCorrelation, updateNotice, updatePostmortemCsvFiles, updatePostmortemWorkflow, urlInspectionQuota, validateResearchImportColumnSources, verifyAnalyticsProvider, verifyProviderExtension, verifyQueryContent, verifyQueryContentWith, withAgentReportContract, withRankingHistory, workflowPresentation, writeAhrefsApiKey, writeBingApiKey, writeClickySiteKey, writeConfig, writeDataForSeoCredentials, writeInstalledProviderPackages, writeOauthClient, writeProviderExtensionAccount, writeProviderExtensionCredentials, writeProviderSecret, writeToCache, writeTokens, xRobotsDirectives };
40227
+ export { AHREFS_API_KEY_ENV, AHREFS_API_KEY_SECRET, AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, ANALYTICS_PROVIDER_IDS, AUDIT_PAGE_RULE_IDS, AhrefsClient, BING_API_KEY_ENV, BING_API_KEY_SECRET, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_IMPRESSIONS, CANNIBAL_MINIMUM_PAGE_SHARE, CLICKY_SITEKEY_ENV, CLICKY_SITE_KEYS_SECRET, CLOUD_SIGN_IN, CLOUD_URL, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DATAFORSEO_CREDENTIAL_SECRET, DATAFORSEO_LOGIN_ENV, DATAFORSEO_PASSWORD_ENV, DEFAULT_MAX_PAGES, DEFAULT_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, DEFAULT_UNREACHED_PAGE_LIMIT, DataForSeoClient, DecisionModelError, DecisionRunner, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, GoogleApiError, INCIDENTS_URL, JsonFileStorageAdapter, LINK_ROLES, LINT_MAX_FILES, LOCAL_BING_CONNECT, LOCAL_SIGN_IN, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MANAGED_PROVIDER_SECRET_NAMES, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, MAX_MAX_PAGES, MAX_SEARCH_CONSOLE_EXPORT_FILES, MAX_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, PAGE_TYPES, PROVIDER_ERROR_CODES, PROVIDER_SECRET_NAMES, ProviderError, ProviderExtensionRegistry, RESEARCH_IMPORT_COLUMN_KEYS, ResponseSizeLimitError, SEARCH_INTENTS, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_PROVIDER_API_VERSION, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SITEMAP_FILE_URL_LIMIT, SITE_CHECK_MAX_ISSUES, SITE_CHECK_MAX_PAGES, STEP_CHANGE_RULES, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_FAILURE_CONTEXTS, TELEMETRY_FAILURE_REASONS, TELEMETRY_NOTICE, TELEMETRY_OPERATIONS, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, TYPESAFE_API_KEY_ENV, agentActionsView, aiReferralSourceForValue, aiReferralsReport, analyticsConnectionLabel, analyticsConnectionProviderId, analyticsConnectionSchema, analyticsProviderDetails, analyticsProviderSchema, analyzeAiReferralRows, analyzeCannibalImportRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, auditCrawlPages, auditCrawlRequests, auditPage, authStatus, bearerFetch, boundedSitemapInventory, buildAgentFindings, buildQueryClusterReportFromRows, buildTelemetryPayload, builtPagePath, cacheDb, cannibalDataStatus, cannibalImportReport, cannibalReport, cannibalReportWith, changeWindow, checkBuiltSite, checkDatabaseReadiness, classifyTelemetryFailure, clearCache, cloudAccountFor, cloudBing, cloudCredentials, cloudGoogleEmails, cloudGoogleToken, cloudKeys, cloudLinked, cloudLogout, cloudWeek, clusterQueryRows, combineRobotsValues, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlCoverageCaveat, crawlDefinitionId, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, ctrUnderperformersReportWith, decayClusterDrafts, decayComparisonRange, decayingReport, decayingReportWith, deleteAhrefsApiKey, deleteBingApiKey, deleteClickySiteKey, deleteDataForSeoCredentials, deleteManagedProviderSecrets, deleteProviderExtensionAccount, deleteProviderExtensionCredentials, deleteProviderSecret, deleteTokens, deriveBrandTerms, describeInstalledProviderExtension, detectAgentName, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensurePrivateDatabaseFiles, ensureSeoCliDirs, estimateSerpTitleWidth, expectedCtrForPosition, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fileSize, finalGscDateRange, findOverlappingSearchUpdates, finishCloudLogin, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, getAuthModeStatus, getCacheStats, getClientConfig, getDb, getPrivacySnapshot, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleJson, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoverageSignals, inferTemplateMovement, initializeTelemetry, inspectProviderPackage, inspectSearchConsoleUrl, inspectUrl, installProviderPackage, installedProviderPackageSchema, installedProviderPackagesSchema, internalLinksReport, internalLinksReportWith, interpretUpdateCorrelation, isBrandQuery, isNewerVersion, isSkippableReportError, isTelemetryCi, isTelemetryOperation, isTelemetryReport, isoDate, isoWeek, latestFinalGscDate, linkRecover, linkRecoverRecommendation, lintPaths, listAnalyticsAccounts, listAnalyticsDataStreams, listGa4AccountSummaries, listGa4DataStreams, listGoogleAccounts, listInstalledProviderExtensions, listRules, listSearchConsoleSites, listSearchUpdates, listSites, loadInstalledProviderExtensions, loadProviderExtensions, loadProviderPackage, loadSearchConsoleExport, localAccounts, localBing, localCredentialsProvider, localError, localGoogleFetch, localInternalLinksDependencies, localRefreshClicky, localResearchKeys, loginWithLoopback, makeRunId, matchGa4WebStreams, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, nodeCommandDeps, normalizeBrandText, normalizeClickySiteId, normalizeClickySiteKey, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, normalizeProviderExtensionAccount, normalizedResearchColumnName, notSignedIn, noteCacheWrite, openCloudLink, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, pendingCloudLogin, performanceAudit, performanceReportIsCacheable, plural, providerCapabilitySchema, providerConnectionFieldSchema, providerExactVersionSchema, providerExtensionIdSchema, providerIdSchema, providerIntegritySchema, providerKindSchema, providerPackageManifestSchema, providerPackageNameSchema, providerPreferenceSchema, providerRegistrationSchema, providerRequestJson, providerRequestText, pruneLogs, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, quickWinsReportWith, readAhrefsApiKey, readAllTokens, readBingApiKey, readBoundedResponseText, readClickySiteKey, readClickySiteKeys, readConfig, readDataForSeoCredentials, readInstalledProviderPackages, readJson, readJsonOrUndefined, readLintFiles, readOauthClient, readProviderExtensionAccount, readProviderExtensionCredentials, readProviderSecret, readTokens, recommendationForRule, reconcileExportPagesWithCrawl, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, registrableDomain, removeInstalledProviderPackage, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderWorkflowMarkdown, reportNarrative, requestHeaders, requirePageVerification, researchImportColumns, resetSeoData, resolveLocalSite, resolveSegmentRanges, resolveSeoCliPaths, reviewObservations, roundedPosition, runAnalyticsProviderLandingPages, runAnalyticsReport, runDoctor, runGa4Report, runProviderExtensionAction, runtimeForProvider, safeProviderAdapterError, saveInstalledProviderPackage, searchAnalyticsRows, searchUpdatesFromIncidents, secondPage, secondPageWith, segmentComparisonRange, segmentImpact, segmentRangeDays, seoErrorEnvelope, sessionSourceInListFilter, setActiveGoogleAccount, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteStartUrl, snapshotFetchedPage, startCloudLogin, strikingDistance, strikingDistanceWith, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, telemetryErrorCategory, toSeoError, tokenSchema, tokenStoreSchema, topFixes, trackTelemetryCommandFailed, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, trafficAnomalyWith, typesafeDecisionModel, unavailableSegmentImpactReport, unicodeTokens, uninstallProviderExtension, updateCorrelation, updateNotice, updatePostmortemCsvFiles, updatePostmortemWorkflow, urlInspectionQuota, validateResearchImportColumnSources, verifyAnalyticsProvider, verifyProviderExtension, verifyQueryContent, verifyQueryContentWith, withAgentReportContract, withRankingHistory, workflowPresentation, writeAhrefsApiKey, writeBingApiKey, writeClickySiteKey, writeConfig, writeDataForSeoCredentials, writeInstalledProviderPackages, writeOauthClient, writeProviderExtensionAccount, writeProviderExtensionCredentials, writeProviderSecret, writeToCache, writeTokens, xRobotsDirectives };
39371
40228
 
39372
40229
  //# sourceMappingURL=index.js.map