seo 0.2.9 → 0.2.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/mcp.js CHANGED
@@ -1,9 +1,9 @@
1
1
  import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
2
- import { n as load, t as version } from "./package-C51oI-6o.js";
2
+ import { n as load, t as version } from "./package-B-M-4PZ_.js";
3
3
  import { Headers, fetch as fetch$1 } from "undici";
4
- import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
5
- import { arch, homedir, platform } from "node:os";
6
- import { dirname, join } from "node:path";
4
+ import { chmodSync, createReadStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
5
+ import { arch, homedir, platform, totalmem } from "node:os";
6
+ import { dirname, extname, join, resolve } from "node:path";
7
7
  import { z } from "zod";
8
8
  import lockfile from "proper-lockfile";
9
9
  import { createHash, randomUUID, sign } from "node:crypto";
@@ -21,9 +21,9 @@ import { gunzipSync } from "node:zlib";
21
21
  import { execFile } from "node:child_process";
22
22
  import { fileURLToPath } from "node:url";
23
23
  import { promisify } from "node:util";
24
+ import { readFile, stat } from "node:fs/promises";
24
25
  import "pino";
25
26
  import "pino-pretty";
26
- import "node:fs/promises";
27
27
  import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
28
28
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
29
29
  import * as z$1 from "zod/v4";
@@ -86,12 +86,12 @@ var SeoError = class extends Error {
86
86
  this.retryable = errorMetadata[code].retryable;
87
87
  }
88
88
  };
89
- function errorMessage$2(error) {
89
+ function errorMessage$3(error) {
90
90
  return error instanceof Error ? error.message : String(error);
91
91
  }
92
92
  function toSeoError(error) {
93
93
  if (error instanceof SeoError) return error;
94
- const message = errorMessage$2(error);
94
+ const message = errorMessage$3(error);
95
95
  if (/not logged in/i.test(message)) return new SeoError("AUTH_REQUIRED", message);
96
96
  if (/oauth client config.*missing|shared seo google app/i.test(message)) return new SeoError("AUTH_CONFIG_REQUIRED", message);
97
97
  if (/invalid_grant|refresh token.*no longer valid/i.test(message)) return new SeoError("AUTH_EXPIRED", message);
@@ -148,6 +148,7 @@ function resolveSeoCliPaths(input = {}) {
148
148
  configFile: join(configDir, "config.json"),
149
149
  tokensFile: join(configDir, "tokens.json"),
150
150
  oauthClientFile: join(configDir, "oauth-client.json"),
151
+ providerSecretsFile: join(configDir, "provider-secrets.json"),
151
152
  telemetryStateFile: join(configDir, "telemetry.json"),
152
153
  cacheDbFile: join(cacheDir, "cache.db")
153
154
  };
@@ -227,6 +228,7 @@ const clientProfileSchema = z.object({
227
228
  watchUrls: z.array(z.string()).default([]),
228
229
  brandTerms: z.array(z.string()).default([]),
229
230
  analytics: analyticsConnectionsSchema,
231
+ searchEngines: z.object({ bing: z.object({ siteUrl: z.string() }).optional() }).optional(),
230
232
  reportDay: z.number().int().min(1).max(31).optional(),
231
233
  technicalWeekday: z.number().int().min(0).max(7).optional(),
232
234
  isDefault: z.boolean().optional(),
@@ -305,7 +307,7 @@ async function deleteKeyringPassword(service, account) {
305
307
  }
306
308
  //#endregion
307
309
  //#region packages/core/dist/storage/config.js
308
- const KEYRING_SERVICE = "seo";
310
+ const KEYRING_SERVICE$1 = "seo";
309
311
  const PRIVATE_FILE_MODE = 384;
310
312
  function tightenConfigPermissions(path) {
311
313
  try {
@@ -336,7 +338,7 @@ function tokenMetadata(tokens) {
336
338
  }
337
339
  async function readKeyringTokens(tokens) {
338
340
  const account = getKeyringAccount(tokens);
339
- const [accessToken, refreshToken] = await Promise.all([getKeyringPassword(KEYRING_SERVICE, `${account}:access`), getKeyringPassword(KEYRING_SERVICE, `${account}:refresh`)]);
341
+ const [accessToken, refreshToken] = await Promise.all([getKeyringPassword(KEYRING_SERVICE$1, `${account}:access`), getKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`)]);
340
342
  return {
341
343
  accessToken: accessToken ?? void 0,
342
344
  refreshToken: refreshToken ?? void 0
@@ -344,14 +346,14 @@ async function readKeyringTokens(tokens) {
344
346
  }
345
347
  async function writeKeyringTokens(tokens) {
346
348
  const account = getKeyringAccount(tokens);
347
- if (tokens.access_token) await setKeyringPassword(KEYRING_SERVICE, `${account}:access`, tokens.access_token);
348
- else await deleteKeyringPassword(KEYRING_SERVICE, `${account}:access`);
349
- if (tokens.refresh_token) await setKeyringPassword(KEYRING_SERVICE, `${account}:refresh`, tokens.refresh_token);
350
- else await deleteKeyringPassword(KEYRING_SERVICE, `${account}:refresh`);
349
+ if (tokens.access_token) await setKeyringPassword(KEYRING_SERVICE$1, `${account}:access`, tokens.access_token);
350
+ else await deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:access`);
351
+ if (tokens.refresh_token) await setKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`, tokens.refresh_token);
352
+ else await deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`);
351
353
  }
352
354
  async function deleteKeyringTokens(tokens) {
353
355
  const account = getKeyringAccount(tokens);
354
- await Promise.all([deleteKeyringPassword(KEYRING_SERVICE, `${account}:access`), deleteKeyringPassword(KEYRING_SERVICE, `${account}:refresh`)]);
356
+ await Promise.all([deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:access`), deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`)]);
355
357
  }
356
358
  async function readTokens() {
357
359
  ensureSeoCliDirs();
@@ -769,14 +771,14 @@ async function createGoogleAccessTokenClient() {
769
771
  }
770
772
  //#endregion
771
773
  //#region packages/core/dist/storage/cache-maintenance.js
772
- const MEBIBYTE$1 = 1024 * 1024;
774
+ const MEBIBYTE$2 = 1024 * 1024;
773
775
  const CACHE_MAX_AGE_MS = 720 * 60 * 60 * 1e3;
774
776
  const CACHE_LIMITS = {
775
- gsc_cache: 48 * MEBIBYTE$1,
776
- ga4_cache: 24 * MEBIBYTE$1,
777
- semrush_cache: 16 * MEBIBYTE$1,
778
- http_cache: 160 * MEBIBYTE$1,
779
- performance_reports: 8 * MEBIBYTE$1
777
+ gsc_cache: 48 * MEBIBYTE$2,
778
+ ga4_cache: 24 * MEBIBYTE$2,
779
+ semrush_cache: 16 * MEBIBYTE$2,
780
+ http_cache: 160 * MEBIBYTE$2,
781
+ performance_reports: 8 * MEBIBYTE$2
780
782
  };
781
783
  const CACHE_POLICIES = [
782
784
  {
@@ -2571,7 +2573,7 @@ function isValidStructuredDate(value) {
2571
2573
  const parsed = new Date(trimmed);
2572
2574
  return Number.isFinite(parsed.getTime()) && parsed.toISOString().slice(0, 10) === datePart;
2573
2575
  }
2574
- function parseJsonLd(blocks, baseUrl) {
2576
+ function parseJsonLd(blocks, baseUrl, retainJsonLd) {
2575
2577
  const jsonLd = [];
2576
2578
  const invalidJsonLdSamples = [];
2577
2579
  const unrecognizedJsonLdTypes = [];
@@ -2643,7 +2645,7 @@ function parseJsonLd(blocks, baseUrl) {
2643
2645
  blocks.forEach((block, index) => {
2644
2646
  try {
2645
2647
  const parsed = JSON.parse(block);
2646
- jsonLd.push(parsed);
2648
+ if (retainJsonLd) jsonLd.push(parsed);
2647
2649
  visit(parsed, { prefixes: /* @__PURE__ */ new Map() }, index, "$");
2648
2650
  } catch (error) {
2649
2651
  invalidJsonLdSamples.push({
@@ -2704,8 +2706,8 @@ function usableElementValue($, selector) {
2704
2706
  return ($(element).attr("content") ?? $(element).attr("datetime") ?? $(element).text()).trim().length > 0;
2705
2707
  });
2706
2708
  }
2707
- function extractStructuredData($, baseUrl) {
2708
- const parsed = parseJsonLd($("script[type=\"application/ld+json\"]").toArray().map((element) => $(element).html() ?? ""), baseUrl);
2709
+ function extractStructuredData($, baseUrl, options = {}) {
2710
+ const parsed = parseJsonLd($("script[type=\"application/ld+json\"]").toArray().map((element) => $(element).html() ?? ""), baseUrl, options.retainJsonLd ?? true);
2709
2711
  const microdata = microdataTypes($);
2710
2712
  const rdfa = rdfaTypes($);
2711
2713
  const schemaTypes = [.../* @__PURE__ */ new Set([
@@ -2739,6 +2741,16 @@ function extractStructuredData($, baseUrl) {
2739
2741
  }
2740
2742
  //#endregion
2741
2743
  //#region packages/core/dist/extract/page-extractor.js
2744
+ const SOCIAL_PROFILE_HOSTS = [
2745
+ "facebook.com",
2746
+ "instagram.com",
2747
+ "linkedin.com",
2748
+ "pinterest.com",
2749
+ "tiktok.com",
2750
+ "twitter.com",
2751
+ "x.com",
2752
+ "youtube.com"
2753
+ ];
2742
2754
  function safeText$1(value) {
2743
2755
  const trimmed = value?.replace(/\s+/g, " ").trim();
2744
2756
  return trimmed ? trimmed : void 0;
@@ -2755,7 +2767,7 @@ function absoluteUrl$2(value, base) {
2755
2767
  return;
2756
2768
  }
2757
2769
  }
2758
- function httpUrl(value, base) {
2770
+ function httpUrl$1(value, base) {
2759
2771
  const resolved = absoluteUrl$2(value, base);
2760
2772
  if (!resolved) return void 0;
2761
2773
  const protocol = new URL(resolved).protocol;
@@ -2811,6 +2823,28 @@ function imageDimensionsFromUrl(src) {
2811
2823
  height
2812
2824
  };
2813
2825
  }
2826
+ function addAnchorSample(samples, seen, href, text) {
2827
+ if (samples.length >= 25 || !text) return;
2828
+ const sampleText = text.length > 120 ? text.slice(0, 120).trimEnd() : text;
2829
+ const key = `${href}\n${sampleText}`;
2830
+ if (seen.has(key)) return;
2831
+ seen.add(key);
2832
+ samples.push({
2833
+ href,
2834
+ text: sampleText
2835
+ });
2836
+ }
2837
+ function addSocialProfile(profiles, href) {
2838
+ if (profiles.size >= 50) return;
2839
+ try {
2840
+ const url = new URL(href);
2841
+ const host = url.hostname.replace(/^www\./, "").toLowerCase();
2842
+ if (SOCIAL_PROFILE_HOSTS.some((value) => host === value || host.endsWith(`.${value}`))) {
2843
+ url.hash = "";
2844
+ profiles.add(url.toString());
2845
+ }
2846
+ } catch {}
2847
+ }
2814
2848
  async function extractPage(fetchResult, extractor = "defuddle", dependencies = {}) {
2815
2849
  const $ = extractor === "crawler" && /<html(?:\s|>)/i.test(fetchResult.html) ? load(fetchResult.html, { xml: {
2816
2850
  xmlMode: false,
@@ -2818,65 +2852,106 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
2818
2852
  } }) : load(fetchResult.html);
2819
2853
  const url = new URL(fetchResult.finalUrl);
2820
2854
  const canonicalEvidence = extractCanonicalEvidence($, fetchResult.headers, fetchResult.finalUrl);
2821
- const headings = $("h1, h2, h3, h4, h5, h6").toArray().map((element) => ({
2822
- level: Number.parseInt(element.tagName.slice(1), 10),
2823
- text: $(element).text().replace(/\s+/g, " ").trim()
2824
- })).filter((heading) => heading.text.length > 0);
2855
+ const headings = [];
2856
+ let crawlerH1;
2857
+ let crawlerH1Count = 0;
2858
+ let crawlerH2Count = 0;
2859
+ let crawlerH3Count = 0;
2860
+ let questionHeadings = 0;
2861
+ $("h1, h2, h3, h4, h5, h6").each((_index, element) => {
2862
+ const level = Number.parseInt(element.tagName.slice(1), 10);
2863
+ const text = $(element).text().replace(/\s+/g, " ").trim();
2864
+ if (!text) return;
2865
+ if (text.trimEnd().endsWith("?")) questionHeadings += 1;
2866
+ if (level === 1) {
2867
+ crawlerH1 ??= text;
2868
+ crawlerH1Count += 1;
2869
+ } else if (level === 2) crawlerH2Count += 1;
2870
+ else if (level === 3) crawlerH3Count += 1;
2871
+ if (extractor !== "crawler") headings.push({
2872
+ level,
2873
+ text
2874
+ });
2875
+ });
2825
2876
  const links = [];
2877
+ const crawlerInternalLinks = /* @__PURE__ */ new Set();
2878
+ const crawlerExternalLinks = /* @__PURE__ */ new Set();
2879
+ const internalAnchorSamples = [];
2880
+ const externalAnchorSamples = [];
2881
+ const internalAnchorKeys = /* @__PURE__ */ new Set();
2882
+ const externalAnchorKeys = /* @__PURE__ */ new Set();
2883
+ const socialProfileLinks = /* @__PURE__ */ new Set();
2826
2884
  let invalidLinkCount = 0;
2827
2885
  let unsupportedLinkCount = 0;
2828
- for (const element of $("a[href]").toArray()) {
2886
+ $("a[href]").each((_index, element) => {
2829
2887
  const href = $(element).attr("href") ?? "";
2830
2888
  try {
2831
2889
  const absolute = new URL(href, fetchResult.finalUrl);
2832
2890
  if (!["http:", "https:"].includes(absolute.protocol)) {
2833
2891
  unsupportedLinkCount += 1;
2834
- continue;
2892
+ return;
2835
2893
  }
2836
- links.push({
2837
- href: absolute.toString(),
2838
- text: $(element).text().replace(/\s+/g, " ").trim(),
2894
+ const absoluteHref = absolute.toString();
2895
+ const text = $(element).text().replace(/\s+/g, " ").trim();
2896
+ const internal = absolute.host === url.host;
2897
+ if (extractor === "crawler") {
2898
+ if (absolute.origin === url.origin) {
2899
+ absolute.hash = "";
2900
+ crawlerInternalLinks.add(absolute.toString());
2901
+ }
2902
+ if (!internal) {
2903
+ crawlerExternalLinks.add(absoluteHref);
2904
+ addSocialProfile(socialProfileLinks, absoluteHref);
2905
+ }
2906
+ addAnchorSample(internal ? internalAnchorSamples : externalAnchorSamples, internal ? internalAnchorKeys : externalAnchorKeys, absoluteHref, text);
2907
+ } else links.push({
2908
+ href: absoluteHref,
2909
+ text,
2839
2910
  rel: ($(element).attr("rel") ?? "").split(/\s+/).filter(Boolean),
2840
- internal: absolute.host === url.host,
2911
+ internal,
2841
2912
  location: $(element).closest("nav, header").length ? "navigation" : $(element).closest("footer").length ? "footer" : $(element).closest("main, article").length ? "main-content" : "other"
2842
2913
  });
2843
2914
  } catch {
2844
2915
  invalidLinkCount += 1;
2845
2916
  }
2846
- }
2917
+ });
2847
2918
  const hreflang = [];
2848
2919
  let invalidHreflangCount = 0;
2849
2920
  for (const element of $("link[rel~=\"alternate\"][hreflang][href]").toArray()) {
2850
2921
  const language = $(element).attr("hreflang") ?? "";
2851
- const href = httpUrl($(element).attr("href"), fetchResult.finalUrl);
2922
+ const href = httpUrl$1($(element).attr("href"), fetchResult.finalUrl);
2852
2923
  if (language && href) hreflang.push({
2853
2924
  hreflang: language,
2854
2925
  href
2855
2926
  });
2856
2927
  else invalidHreflangCount += 1;
2857
2928
  }
2858
- const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
2929
+ const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl$1($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
2859
2930
  const openGraph = Object.fromEntries($("meta[property^=\"og:\"]").toArray().map((element) => [$(element).attr("property") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
2860
2931
  const twitter = Object.fromEntries($("meta[name^=\"twitter:\"]").toArray().map((element) => [$(element).attr("name") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
2861
- const structuredData = extractStructuredData($, fetchResult.finalUrl);
2932
+ const structuredData = extractStructuredData($, fetchResult.finalUrl, { retainJsonLd: extractor !== "crawler" });
2862
2933
  const hasDate = structuredData.hasDate || isValidStructuredDate($("meta[property=\"article:published_time\"]").attr("content")) || isValidStructuredDate($("meta[property=\"article:modified_time\"]").attr("content")) || $("article time[datetime], main time[datetime]").toArray().some((element) => isValidStructuredDate($(element).attr("datetime")));
2863
2934
  const author = safeText$1($("meta[name=\"author\"]").attr("content"));
2864
2935
  const hasAuthor = Boolean(author) || structuredData.hasAuthor || $("[rel~=author], .author, .byline").toArray().some((element) => Boolean(safeText$1($(element).text())));
2865
2936
  const semanticHtml = $("main, article").length > 0;
2866
- const questionHeadings = headings.filter((heading) => heading.text.trimEnd().endsWith("?")).length;
2867
2937
  const listCount = $("ul, ol").length;
2868
2938
  const tableCount = $("table").length;
2869
2939
  const structuredBlocks = listCount + tableCount;
2870
2940
  const answerable = $("main p, article p, p").toArray().slice(0, 3).some((element) => $(element).text().replace(/\s+/g, " ").trim().split(/\s+/).filter(Boolean).length >= 25);
2871
- const imageElements = $("img").toArray();
2872
- const oversizedImageCandidates = imageElements.map((element) => {
2941
+ let imagesTotal = 0;
2942
+ let imagesMissingAlt = 0;
2943
+ const oversizedImageCandidates = [];
2944
+ $("img").each((_index, element) => {
2945
+ imagesTotal += 1;
2946
+ if ($(element).attr("alt") === void 0) imagesMissingAlt += 1;
2947
+ if (oversizedImageCandidates.length >= 25) return;
2873
2948
  const src = absoluteUrl$2($(element).attr("src"), fetchResult.finalUrl);
2874
- if (!src) return void 0;
2949
+ if (!src) return;
2875
2950
  const width = numericAttribute($(element).attr("width"));
2876
2951
  const height = numericAttribute($(element).attr("height"));
2877
2952
  const srcsetWidth = largestSrcsetWidth($(element).attr("srcset"));
2878
2953
  const filenameDimensions = imageDimensionsFromUrl(src);
2879
- if (Math.max(width ?? 0, height ?? 0, srcsetWidth ?? 0, filenameDimensions?.width ?? 0, filenameDimensions?.height ?? 0) < 2e3) return void 0;
2954
+ if (Math.max(width ?? 0, height ?? 0, srcsetWidth ?? 0, filenameDimensions?.width ?? 0, filenameDimensions?.height ?? 0) < 2e3) return;
2880
2955
  const detectedFrom = [
2881
2956
  width && width >= 2e3 ? "width" : void 0,
2882
2957
  height && height >= 2e3 ? "height" : void 0,
@@ -2885,13 +2960,13 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
2885
2960
  ].filter(Boolean).join(",");
2886
2961
  const candidateWidth = Math.max(width ?? 0, srcsetWidth ?? 0, filenameDimensions?.width ?? 0);
2887
2962
  const candidateHeight = Math.max(height ?? 0, filenameDimensions?.height ?? 0);
2888
- return {
2963
+ oversizedImageCandidates.push({
2889
2964
  src,
2890
2965
  detectedFrom,
2891
2966
  ...candidateWidth > 0 ? { width: candidateWidth } : {},
2892
2967
  ...candidateHeight > 0 ? { height: candidateHeight } : {}
2893
- };
2894
- }).filter((candidate) => Boolean(candidate));
2968
+ });
2969
+ });
2895
2970
  const mixedContentUrls = url.protocol === "https:" ? [...new Set($("img[src], script[src], iframe[src], source[src], video[src], audio[src], link[href]").toArray().map((element) => absoluteUrl$2($(element).attr("src") ?? $(element).attr("href"), fetchResult.finalUrl)).filter((value) => typeof value === "string" && new URL(value).protocol === "http:"))] : [];
2896
2971
  const metaRobots = combineRobotsValues($("meta[name]").toArray().filter((element) => {
2897
2972
  const name = $(element).attr("name")?.trim().toLowerCase();
@@ -2915,6 +2990,18 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
2915
2990
  hasViewport: Boolean($("meta[name=\"viewport\"]").attr("content")),
2916
2991
  headings,
2917
2992
  links,
2993
+ ...extractor === "crawler" ? { crawlerEvidence: {
2994
+ h1: crawlerH1,
2995
+ h1Count: crawlerH1Count,
2996
+ h2Count: crawlerH2Count,
2997
+ h3Count: crawlerH3Count,
2998
+ questionHeadings,
2999
+ internalLinks: [...crawlerInternalLinks],
3000
+ externalLinks: [...crawlerExternalLinks],
3001
+ internalAnchorSamples,
3002
+ externalAnchorSamples,
3003
+ socialProfileLinks: [...socialProfileLinks]
3004
+ } } : {},
2918
3005
  markdownAlternates,
2919
3006
  hreflang,
2920
3007
  jsonLd: structuredData.jsonLd,
@@ -2923,19 +3010,17 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
2923
3010
  unrecognizedJsonLdTypes: structuredData.unrecognizedJsonLdTypes.slice(0, 25),
2924
3011
  structuredDataFormats: structuredData.formats,
2925
3012
  googleRichResults: structuredData.googleRichResults,
2926
- schemaSameAsEvidence: structuredData.sameAs,
2927
- invalidSchemaSameAs: structuredData.invalidSameAs,
3013
+ schemaSameAsEvidence: extractor === "crawler" ? structuredData.sameAs.slice(0, 50) : structuredData.sameAs,
3014
+ invalidSchemaSameAs: extractor === "crawler" ? structuredData.invalidSameAs.slice(0, 25) : structuredData.invalidSameAs,
2928
3015
  schemaTypes: structuredData.schemaTypes,
2929
3016
  openGraph,
2930
3017
  twitter,
2931
3018
  author,
2932
3019
  hasAuthor,
2933
3020
  hasDate,
2934
- imagesTotal: imageElements.length,
2935
- imagesMissingAlt: imageElements.filter((element) => {
2936
- return $(element).attr("alt") === void 0;
2937
- }).length,
2938
- oversizedImageCandidates: oversizedImageCandidates.slice(0, 25),
3021
+ imagesTotal,
3022
+ imagesMissingAlt,
3023
+ oversizedImageCandidates,
2939
3024
  mixedContentUrls,
2940
3025
  semanticHtml,
2941
3026
  questionHeadings,
@@ -2943,7 +3028,7 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
2943
3028
  tableCount,
2944
3029
  structuredBlocks,
2945
3030
  answerable,
2946
- contentText: text.replace(/\s+/g, " ").trim(),
3031
+ contentText: extractor === "crawler" ? text : text.replace(/\s+/g, " ").trim(),
2947
3032
  excerpt,
2948
3033
  wordCount: content.wordCount,
2949
3034
  contentExtraction: content.diagnostics,
@@ -3473,7 +3558,7 @@ function decodePageFetchCacheEvidence(value) {
3473
3558
  return;
3474
3559
  }
3475
3560
  }
3476
- async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, respectRobots = false) {
3561
+ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, respectRobots = false, writeCache = true) {
3477
3562
  const startedAt = Date.now();
3478
3563
  const db = getDb();
3479
3564
  const key = hashKey(["page", url]);
@@ -3585,7 +3670,7 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
3585
3670
  warnings: [...robots.allowed === null ? [`robots.txt availability is unknown${robots.status ? ` after HTTP ${robots.status}` : ""}; this fetch does not prove crawler access is allowed.`] : [], ...accessBlock ? [`${accessBlock.guidance.summary} Identify the crawler as ${accessBlock.crawler.userAgent}.`] : []],
3586
3671
  robotsTxt: pageRobotsEvidence(robots)
3587
3672
  };
3588
- if (!accessBlock) {
3673
+ if (!accessBlock && writeCache) {
3589
3674
  db.prepare(`INSERT OR REPLACE INTO http_cache
3590
3675
  (url_hash, url, status, headers_json, body_blob, metadata_json, etag, fetched_at, expires_at)
3591
3676
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(key, url, response.response.status, JSON.stringify(headerMap), Buffer.from(html), encodePageFetchCacheEvidence(result), response.response.headers.get("etag"), Date.now(), Date.now() + 36e5);
@@ -4153,7 +4238,7 @@ async function fetchPage(url, opts = {}, dependencies = {}) {
4153
4238
  const makeRenderer = dependencies.createPageRenderer ?? createPageRenderer;
4154
4239
  const rate = normalizeRateControls(opts.rate);
4155
4240
  return await queueForHost(new URL(url).host, rate).add(async () => {
4156
- const first = await plainFetch(url, opts.refresh, opts.timeoutMs, rate, opts.signal, opts.respectRobots);
4241
+ const first = await plainFetch(url, opts.refresh, opts.timeoutMs, rate, opts.signal, opts.respectRobots, opts.writeCache);
4157
4242
  if (!(!first.diagnostics.accessBlock && (renderingMode === "on" || renderingMode === "auto" && looksLikeSpa(first.html))) || opts.signal?.aborted) return withRenderingState(first, renderingMode, opts.signal?.aborted ? "skipped" : renderingMode === "off" ? "not-requested" : "not-needed");
4158
4243
  const ownsRenderer = !opts.renderer;
4159
4244
  const renderer = opts.renderer ?? makeRenderer();
@@ -4258,7 +4343,7 @@ function urlInspectionQuotaDate(now) {
4258
4343
  function urlInspectionQuotaResetAt(now) {
4259
4344
  return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1));
4260
4345
  }
4261
- function boundedLimit(value) {
4346
+ function boundedLimit$1(value) {
4262
4347
  const limit = value ?? 2e3;
4263
4348
  if (!Number.isInteger(limit) || limit < 1 || limit > 2e3) throw new SeoError("INVALID_INPUT", "URL Inspection daily limit must be an integer from 1 to 2000.");
4264
4349
  return limit;
@@ -4291,7 +4376,7 @@ function reserveUrlInspectionQuota(input) {
4291
4376
  const database = input.database ?? getDb();
4292
4377
  const now = input.now ?? /* @__PURE__ */ new Date();
4293
4378
  const nowMs = now.getTime();
4294
- const requestedLimit = boundedLimit(input.limit);
4379
+ const requestedLimit = boundedLimit$1(input.limit);
4295
4380
  const quotaDate = urlInspectionQuotaDate(now);
4296
4381
  const resetAt = urlInspectionQuotaResetAt(now);
4297
4382
  const result = database.transaction(() => {
@@ -6074,13 +6159,13 @@ function isLowActionabilityQuery(query) {
6074
6159
  //#endregion
6075
6160
  //#region packages/core/dist/analyze/page-opportunities-selection.js
6076
6161
  const DEFAULT_MIN_IMPRESSIONS$3 = 10;
6077
- const DEFAULT_LIMIT$3 = 25;
6078
- const MAX_LIMIT$3 = 100;
6162
+ const DEFAULT_LIMIT$4 = 25;
6163
+ const MAX_LIMIT$4 = 100;
6079
6164
  function integerInRange$2(value, fallback, minimum, maximum) {
6080
6165
  if (value === void 0 || !Number.isFinite(value)) return fallback;
6081
6166
  return Math.min(maximum, Math.max(minimum, Math.floor(value)));
6082
6167
  }
6083
- function isHttpUrl$1(value) {
6168
+ function isHttpUrl$2(value) {
6084
6169
  try {
6085
6170
  const parsed = new URL(value);
6086
6171
  return parsed.protocol === "http:" || parsed.protocol === "https:";
@@ -6102,7 +6187,7 @@ function isBrandRow(input) {
6102
6187
  function normalizePageOpportunityOptions(input) {
6103
6188
  return {
6104
6189
  minImpressions: integerInRange$2(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$3, 1, Number.MAX_SAFE_INTEGER),
6105
- limit: integerInRange$2(input.limit, DEFAULT_LIMIT$3, 1, MAX_LIMIT$3)
6190
+ limit: integerInRange$2(input.limit, DEFAULT_LIMIT$4, 1, MAX_LIMIT$4)
6106
6191
  };
6107
6192
  }
6108
6193
  function selectTargetRows(input) {
@@ -6121,7 +6206,7 @@ function selectTargetRows(input) {
6121
6206
  for (const row of input.rows) {
6122
6207
  const query = row.keys[0]?.trim() ?? "";
6123
6208
  const page = row.keys[1] ?? "";
6124
- if (row.keys.length < 2 || !query || !isHttpUrl$1(page) || !hasValidMetrics(row)) {
6209
+ if (row.keys.length < 2 || !query || !isHttpUrl$2(page) || !hasValidMetrics(row)) {
6125
6210
  selection.invalidRows++;
6126
6211
  continue;
6127
6212
  }
@@ -6157,7 +6242,7 @@ function selectTargetRows(input) {
6157
6242
  function selectBenchmarkRows$1(input) {
6158
6243
  return input.rows.filter((row) => {
6159
6244
  const query = row.keys[0]?.trim() ?? "";
6160
- return row.keys.length >= 2 && isHttpUrl$1(row.keys[1] ?? "") && hasValidMetrics(row) && row.position >= 1 && row.position <= 10 && Boolean(query) && !isLowActionabilityQuery(query) && !isBrandRow({
6245
+ return row.keys.length >= 2 && isHttpUrl$2(row.keys[1] ?? "") && hasValidMetrics(row) && row.position >= 1 && row.position <= 10 && Boolean(query) && !isLowActionabilityQuery(query) && !isBrandRow({
6161
6246
  query,
6162
6247
  site: input.site,
6163
6248
  brandTerms: input.brandTerms,
@@ -6339,7 +6424,7 @@ const defaultDependencies$14 = {
6339
6424
  extract: extractPage,
6340
6425
  now: () => /* @__PURE__ */ new Date()
6341
6426
  };
6342
- function errorMessage$1(error) {
6427
+ function errorMessage$2(error) {
6343
6428
  return error instanceof Error ? error.message : String(error);
6344
6429
  }
6345
6430
  function unique$4(values) {
@@ -6360,7 +6445,7 @@ async function verifyPage(input) {
6360
6445
  });
6361
6446
  warnings.push(...fetched.warnings);
6362
6447
  } catch (error) {
6363
- const reason = `Page fetch failed: ${errorMessage$1(error)}`;
6448
+ const reason = `Page fetch failed: ${errorMessage$2(error)}`;
6364
6449
  return {
6365
6450
  verification: {
6366
6451
  status: "failed",
@@ -6382,7 +6467,7 @@ async function verifyPage(input) {
6382
6467
  warnings: unique$4(warnings)
6383
6468
  };
6384
6469
  } catch (error) {
6385
- const reason = `Page extraction failed: ${errorMessage$1(error)}`;
6470
+ const reason = `Page extraction failed: ${errorMessage$2(error)}`;
6386
6471
  return {
6387
6472
  fetched,
6388
6473
  verification: {
@@ -7574,7 +7659,7 @@ function check$1(section, input) {
7574
7659
  ...input
7575
7660
  };
7576
7661
  }
7577
- function section$1(id, title, checks) {
7662
+ function section$2(id, title, checks) {
7578
7663
  return {
7579
7664
  id,
7580
7665
  title,
@@ -8012,19 +8097,19 @@ function agentReadiness(report) {
8012
8097
  headline: "No agent-discovery evidence is stored in this crawl. Run a fresh focused check.",
8013
8098
  profileApplicability: profiles,
8014
8099
  summary: summariseChecks(checks),
8015
- sections: [section$1("collection", "Evidence collection", [missing]), section$1("profile", "Profile scope", scopeChecks)],
8100
+ sections: [section$2("collection", "Evidence collection", [missing]), section$2("profile", "Profile scope", scopeChecks)],
8016
8101
  checks,
8017
8102
  topActions: [missing],
8018
8103
  caveats: ["Not evaluated does not mean absent or broken.", "This report does not measure rankings, AI mentions, citations, or selection."]
8019
8104
  };
8020
8105
  }
8021
8106
  const sections = [
8022
- section$1("profile", "Profile scope", profileChecks(discovery)),
8023
- section$1("access", "Crawler and protocol access", accessChecks(report, discovery)),
8024
- section$1("representations", "HTML and Markdown representations", markdownChecks(discovery)),
8025
- section$1("discovery", "Agent discovery files", discoveryChecks(discovery, report.ai?.robotsTxt?.contentSignals)),
8026
- section$1("endpoints", "Agent endpoints and auth discovery", endpointChecks(discovery)),
8027
- section$1("identity", "Identity evidence", identityChecks(report))
8107
+ section$2("profile", "Profile scope", profileChecks(discovery)),
8108
+ section$2("access", "Crawler and protocol access", accessChecks(report, discovery)),
8109
+ section$2("representations", "HTML and Markdown representations", markdownChecks(discovery)),
8110
+ section$2("discovery", "Agent discovery files", discoveryChecks(discovery, report.ai?.robotsTxt?.contentSignals)),
8111
+ section$2("endpoints", "Agent endpoints and auth discovery", endpointChecks(discovery)),
8112
+ section$2("identity", "Identity evidence", identityChecks(report))
8028
8113
  ];
8029
8114
  const checks = sections.flatMap((item) => item.checks);
8030
8115
  const actionable = checks.filter((item) => [
@@ -8211,7 +8296,7 @@ function status(score, maxScore) {
8211
8296
  if (ratio >= .5) return "warning";
8212
8297
  return "fail";
8213
8298
  }
8214
- function section(id, title, checks) {
8299
+ function section$1(id, title, checks) {
8215
8300
  return {
8216
8301
  id,
8217
8302
  title,
@@ -8269,7 +8354,7 @@ function aiReadiness(report) {
8269
8354
  const blockedSnippets = restrictedSnippets.filter(({ control }) => control.status === "blocked");
8270
8355
  const limitedSnippets = restrictedSnippets.filter(({ control }) => control.status === "limited");
8271
8356
  const sections = [
8272
- section("agent-access", "Agent access", [
8357
+ section$1("agent-access", "Agent access", [
8273
8358
  check({
8274
8359
  id: "robots-ai-bots",
8275
8360
  section: "agent-access",
@@ -8312,7 +8397,7 @@ function aiReadiness(report) {
8312
8397
  evidence: { resources: report.ai?.agentResources ?? [] }
8313
8398
  })
8314
8399
  ]),
8315
- section("machine-readable", "Machine-readable signals", [check({
8400
+ section$1("machine-readable", "Machine-readable signals", [check({
8316
8401
  id: "llms-txt",
8317
8402
  section: "machine-readable",
8318
8403
  maxScore: 0,
@@ -8337,7 +8422,7 @@ function aiReadiness(report) {
8337
8422
  action: invalidJsonLdPages.length ? "Fix invalid JSON-LD before relying on its structured-data claims." : "Use structured data only where it accurately represents the visible page and a supported use case.",
8338
8423
  urls: invalidJsonLdPages.slice(0, 10).map((page) => page.finalUrl)
8339
8424
  })]),
8340
- section("content-clarity", "Content clarity", [
8425
+ section$1("content-clarity", "Content clarity", [
8341
8426
  check({
8342
8427
  id: "answerable-content",
8343
8428
  section: "content-clarity",
@@ -8387,7 +8472,7 @@ function aiReadiness(report) {
8387
8472
  action: "Set the html lang attribute so crawlers and assistive tech understand the page language."
8388
8473
  })
8389
8474
  ]),
8390
- section("entity-signals", "Entity signals", [check({
8475
+ section$1("entity-signals", "Entity signals", [check({
8391
8476
  id: "entity-schema",
8392
8477
  section: "entity-signals",
8393
8478
  maxScore: 0,
@@ -8406,7 +8491,7 @@ function aiReadiness(report) {
8406
8491
  plainEnglish: "Author and date signals help agents judge freshness and accountability.",
8407
8492
  action: "Add visible authors, reviewed dates, and updated dates to content where trust and freshness matter."
8408
8493
  })]),
8409
- section("technical-ux", "Technical UX", [
8494
+ section$1("technical-ux", "Technical UX", [
8410
8495
  check({
8411
8496
  id: "snippet-controls",
8412
8497
  section: "technical-ux",
@@ -8444,7 +8529,7 @@ function aiReadiness(report) {
8444
8529
  action: "Fix 4xx/5xx pages, accidental noindex, and blocked internal URLs before expanding AI discovery work."
8445
8530
  })
8446
8531
  ]),
8447
- section("agent-content", "Agent content observations", [
8532
+ section$1("agent-content", "Agent content observations", [
8448
8533
  check({
8449
8534
  id: "markdown-alternates",
8450
8535
  section: "agent-content",
@@ -8484,7 +8569,7 @@ function aiReadiness(report) {
8484
8569
  evidence: agentDiscovery?.profileApplicability
8485
8570
  })
8486
8571
  ]),
8487
- section("crawl-completeness", "Crawl completeness", [check({
8572
+ section$1("crawl-completeness", "Crawl completeness", [check({
8488
8573
  id: "crawl-depth",
8489
8574
  section: "crawl-completeness",
8490
8575
  maxScore: 8,
@@ -10431,6 +10516,7 @@ const CRAWL_SKIP_REASONS = [
10431
10516
  "configured-exclusion",
10432
10517
  "duplicate-normalization",
10433
10518
  "invalid-url",
10519
+ "memory-pressure",
10434
10520
  "off-origin",
10435
10521
  "origin-backpressure",
10436
10522
  "queue-safety-limit",
@@ -10443,6 +10529,7 @@ const SKIP_IMPACT = {
10443
10529
  "configured-exclusion": "non-impacting",
10444
10530
  "duplicate-normalization": "non-impacting",
10445
10531
  "invalid-url": "non-impacting",
10532
+ "memory-pressure": "coverage-affecting",
10446
10533
  "off-origin": "non-impacting",
10447
10534
  "origin-backpressure": "coverage-affecting",
10448
10535
  "queue-safety-limit": "coverage-affecting",
@@ -10620,17 +10707,44 @@ function sanitizeTenantString(value) {
10620
10707
  return value.replace(/https?:\/\/[^\s"'<>`]+/g, (match) => sanitizeUrlString(match)).replace(/\b(access_?token|api_?key|auth|client_?secret|jwt|password|refresh_?token|secret|session|signature|token)(\s*[=:]\s*)[^\s,;&]+/gi, (_match, key, separator) => `${key}${separator}[redacted]`).replace(/\b[A-Za-z]:\\[^\s"'<>`]+/g, "[local-path]").replace(/\/(?:Users|home|tmp|var\/folders)\/[^\s"'<>`]+/g, "[local-path]");
10621
10708
  }
10622
10709
  function sanitizeTenantValue(value, key) {
10623
- if (key && isSensitiveFieldName(key)) return "[redacted]";
10710
+ if (key && isSensitiveFieldName(key)) return value === "[redacted]" ? value : "[redacted]";
10624
10711
  if (typeof value === "string") return sanitizeTenantString(value);
10625
- if (Array.isArray(value)) return value.map((item) => sanitizeTenantValue(item));
10712
+ if (Array.isArray(value)) {
10713
+ let sanitized;
10714
+ for (let index = 0; index < value.length; index += 1) {
10715
+ const item = sanitizeTenantValue(value[index]);
10716
+ if (sanitized) sanitized.push(item);
10717
+ else if (item !== value[index]) sanitized = [...value.slice(0, index), item];
10718
+ }
10719
+ return sanitized ?? value;
10720
+ }
10626
10721
  if (!value || typeof value !== "object") return value;
10627
- return Object.fromEntries(Object.entries(value).map(([entryKey, entry]) => [entryKey, sanitizeTenantValue(entry, entryKey)]));
10722
+ const record = value;
10723
+ let sanitized;
10724
+ for (const [entryKey, entry] of Object.entries(record)) {
10725
+ const safeEntry = sanitizeTenantValue(entry, entryKey);
10726
+ if (safeEntry === entry) continue;
10727
+ sanitized ??= { ...record };
10728
+ sanitized[entryKey] = safeEntry;
10729
+ }
10730
+ return sanitized ?? value;
10628
10731
  }
10629
10732
  function sanitizeCrawlConfig(config) {
10630
10733
  return sanitizeTenantValue(config);
10631
10734
  }
10632
10735
  function sanitizePages(pages) {
10633
- return sanitizeTenantValue(pages).map(({ seoScore: _seoScore, geoScore: _geoScore, ...page }) => page);
10736
+ const safePages = sanitizeTenantValue(pages);
10737
+ let normalized;
10738
+ for (let index = 0; index < safePages.length; index += 1) {
10739
+ const page = safePages[index];
10740
+ if (!page) continue;
10741
+ if ("seoScore" in page || "geoScore" in page) {
10742
+ normalized ??= safePages.slice(0, index);
10743
+ const { seoScore: _seoScore, geoScore: _geoScore, ...current } = page;
10744
+ normalized.push(current);
10745
+ } else if (normalized) normalized.push(page);
10746
+ }
10747
+ return normalized ?? safePages;
10634
10748
  }
10635
10749
  function sanitizeIssues(issues) {
10636
10750
  return sanitizeTenantValue(issues);
@@ -10659,18 +10773,28 @@ function summarizeCrawlReport(input) {
10659
10773
  const byCategory = {};
10660
10774
  let responseMs = 0;
10661
10775
  let responseCount = 0;
10776
+ let statusOnlyPages = 0;
10777
+ let indexablePages = 0;
10778
+ let nonIndexablePages = 0;
10779
+ let statusErrors = 0;
10662
10780
  for (const page of input.pages) {
10663
10781
  byStatus[String(page.status)] = (byStatus[String(page.status)] ?? 0) + 1;
10664
10782
  if (typeof page.responseTimeMs === "number") {
10665
10783
  responseMs += page.responseTimeMs;
10666
10784
  responseCount += 1;
10667
10785
  }
10786
+ if (page.auditScope === "status") statusOnlyPages += 1;
10787
+ else if (page.indexable) indexablePages += 1;
10788
+ else nonIndexablePages += 1;
10789
+ if (page.status === 0 || page.status >= 400) statusErrors += 1;
10668
10790
  }
10669
10791
  const requests = input.requests ?? [];
10670
- const failedRequests = requests.filter((request) => request.outcome === "failure" && request.failureKind !== "aborted");
10671
- const abortedRequests = requests.filter((request) => request.outcome === "failure" && request.failureKind === "aborted");
10672
10792
  let requestMs = 0;
10673
10793
  let requestMsCount = 0;
10794
+ let responseRequests = 0;
10795
+ let failedRequests = 0;
10796
+ let abortedRequests = 0;
10797
+ let extractionFailures = 0;
10674
10798
  for (const request of requests) {
10675
10799
  const status = request.outcome === "response" ? String(request.status) : request.outcome === "skipped" ? request.reason : request.failureKind === "aborted" ? "aborted" : "no-response";
10676
10800
  requestByStatus[status] = (requestByStatus[status] ?? 0) + 1;
@@ -10678,27 +10802,39 @@ function summarizeCrawlReport(input) {
10678
10802
  requestMs += request.durationMs;
10679
10803
  requestMsCount += 1;
10680
10804
  }
10805
+ if (request.outcome === "response") {
10806
+ responseRequests += 1;
10807
+ if (request.extraction === "failed") extractionFailures += 1;
10808
+ } else if (request.outcome === "failure") if (request.failureKind === "aborted") abortedRequests += 1;
10809
+ else failedRequests += 1;
10810
+ }
10811
+ let highIssues = 0;
10812
+ let mediumIssues = 0;
10813
+ let lowIssues = 0;
10814
+ for (const issue of input.issues) {
10815
+ byCategory[issue.category] = (byCategory[issue.category] ?? 0) + 1;
10816
+ if (issue.severity === "high") highIssues += 1;
10817
+ else if (issue.severity === "medium") mediumIssues += 1;
10818
+ else lowIssues += 1;
10681
10819
  }
10682
- for (const issue of input.issues) byCategory[issue.category] = (byCategory[issue.category] ?? 0) + 1;
10683
10820
  const runStats = crawlRunStats(input.pages, input.stats);
10684
- const indexabilityPages = input.pages.filter((page) => page.auditScope !== "status");
10685
10821
  return {
10686
10822
  totalPages: input.pages.length,
10687
- statusOnlyPages: input.pages.filter((page) => page.auditScope === "status").length,
10688
- indexablePages: indexabilityPages.filter((page) => page.indexable).length,
10689
- nonIndexablePages: indexabilityPages.filter((page) => !page.indexable).length,
10690
- statusErrors: input.pages.filter((page) => page.status === 0 || page.status >= 400).length,
10823
+ statusOnlyPages,
10824
+ indexablePages,
10825
+ nonIndexablePages,
10826
+ statusErrors,
10691
10827
  ...runStats,
10692
10828
  attemptedRequests: input.requestEvidenceStatus === "unavailable" ? 0 : requests.length,
10693
- responseRequests: requests.filter((request) => request.outcome === "response").length,
10694
- failedRequests: failedRequests.length,
10695
- abortedRequests: abortedRequests.length,
10696
- extractionFailures: requests.filter((request) => request.outcome === "response" && request.extraction === "failed").length,
10829
+ responseRequests,
10830
+ failedRequests,
10831
+ abortedRequests,
10832
+ extractionFailures,
10697
10833
  requestByStatus,
10698
10834
  avgRequestMs: requestMsCount ? Math.round(requestMs / requestMsCount) : void 0,
10699
- highIssues: input.issues.filter((issue) => issue.severity === "high").length,
10700
- mediumIssues: input.issues.filter((issue) => issue.severity === "medium").length,
10701
- lowIssues: input.issues.filter((issue) => issue.severity === "low").length,
10835
+ highIssues,
10836
+ mediumIssues,
10837
+ lowIssues,
10702
10838
  avgResponseMs: responseCount ? Math.round(responseMs / responseCount) : void 0,
10703
10839
  byStatus,
10704
10840
  byCategory
@@ -10706,42 +10842,55 @@ function summarizeCrawlReport(input) {
10706
10842
  }
10707
10843
  const ACCESS_BLOCK_SAMPLE_LIMIT = 10;
10708
10844
  function crawlAccessSummary(requests) {
10709
- const blocked = requests.flatMap((request) => request.outcome === "response" && request.accessBlock ? [{
10710
- url: request.requestedUrl,
10711
- evidence: request.accessBlock
10712
- }] : []);
10713
10845
  const providers = {};
10714
- for (const item of blocked) providers[item.evidence.provider] = (providers[item.evidence.provider] ?? 0) + 1;
10846
+ const samples = [];
10847
+ let blockedRequests = 0;
10848
+ for (const request of requests) {
10849
+ if (request.outcome !== "response" || !request.accessBlock) continue;
10850
+ blockedRequests += 1;
10851
+ providers[request.accessBlock.provider] = (providers[request.accessBlock.provider] ?? 0) + 1;
10852
+ if (samples.length < ACCESS_BLOCK_SAMPLE_LIMIT) samples.push({
10853
+ url: request.requestedUrl,
10854
+ evidence: request.accessBlock
10855
+ });
10856
+ }
10715
10857
  return {
10716
10858
  crawler: SEO_CRAWLER_IDENTITY,
10717
- blockedRequests: blocked.length,
10859
+ blockedRequests,
10718
10860
  providers,
10719
- samples: blocked.slice(0, ACCESS_BLOCK_SAMPLE_LIMIT),
10861
+ samples,
10720
10862
  sampleLimit: ACCESS_BLOCK_SAMPLE_LIMIT,
10721
- truncated: blocked.length > ACCESS_BLOCK_SAMPLE_LIMIT
10863
+ truncated: blockedRequests > ACCESS_BLOCK_SAMPLE_LIMIT
10722
10864
  };
10723
10865
  }
10724
10866
  function crawlRunStats(pages, stats = {}) {
10725
- const discovered = /* @__PURE__ */ new Set();
10867
+ const needsDiscoveredUrls = stats.discoveredUrls === void 0;
10868
+ const needsObservedInternalLinks = stats.observedInternalLinks === void 0;
10869
+ const needsFailedUrls = stats.failedUrls === void 0;
10870
+ const discovered = needsDiscoveredUrls ? /* @__PURE__ */ new Set() : void 0;
10726
10871
  let observedInternalLinks = 0;
10727
- for (const page of pages) {
10728
- discovered.add(page.url);
10729
- for (const url of page.sampleInternalLinks ?? []) discovered.add(url);
10730
- for (const url of page.sampleExternalLinks ?? []) discovered.add(url);
10731
- observedInternalLinks += page.outgoingInternalCount;
10872
+ let failedUrls = 0;
10873
+ if (needsDiscoveredUrls || needsObservedInternalLinks || needsFailedUrls) for (const page of pages) {
10874
+ if (discovered) {
10875
+ discovered.add(page.url);
10876
+ for (const url of page.sampleInternalLinks ?? []) discovered.add(url);
10877
+ for (const url of page.sampleExternalLinks ?? []) discovered.add(url);
10878
+ }
10879
+ if (needsObservedInternalLinks) observedInternalLinks += page.outgoingInternalCount;
10880
+ if (needsFailedUrls && (page.status === 0 || page.status >= 400)) failedUrls += 1;
10732
10881
  }
10733
10882
  const skips = normalizeCrawlSkipReasonCounts({
10734
10883
  skippedUrls: stats.skippedUrls,
10735
10884
  skipReasons: stats.skipReasons ?? crawlSkipReasonCountsFromRecord(stats.skipReasonCounts)
10736
10885
  });
10737
10886
  return {
10738
- discoveredUrls: stats.discoveredUrls ?? discovered.size,
10887
+ discoveredUrls: stats.discoveredUrls ?? discovered?.size ?? 0,
10739
10888
  queuedUrls: stats.queuedUrls ?? pages.length,
10740
10889
  crawledUrls: stats.crawledUrls ?? pages.length,
10741
10890
  skippedUrls: skips.skippedUrls,
10742
10891
  skipReasons: skips.skipReasons,
10743
10892
  skippedUrlsByImpact: skips.skippedUrlsByImpact,
10744
- failedUrls: stats.failedUrls ?? pages.filter((page) => page.status === 0 || page.status >= 400).length,
10893
+ failedUrls: stats.failedUrls ?? failedUrls,
10745
10894
  observedInternalLinks: stats.observedInternalLinks ?? observedInternalLinks,
10746
10895
  pageLimitReached: stats.pageLimitReached ?? false
10747
10896
  };
@@ -10758,11 +10907,14 @@ function normalizeLinkUrl(value) {
10758
10907
  function deriveInternalLinkAuthority(pages, linkGraph) {
10759
10908
  const hasLinkGraph = linkGraph && Object.keys(linkGraph).length > 0;
10760
10909
  const hasStoredScores = pages.some((page) => page.internalInlinkCount !== void 0 || page.internalLinkAuthorityScore !== void 0);
10761
- if (!hasLinkGraph && hasStoredScores) return pages.map((page) => ({
10762
- ...page,
10763
- internalInlinkCount: page.internalInlinkCount ?? 0,
10764
- internalLinkAuthorityScore: page.internalLinkAuthorityScore ?? 0
10765
- }));
10910
+ if (!hasLinkGraph && hasStoredScores) {
10911
+ if (pages.every((page) => page.internalInlinkCount !== void 0 && page.internalLinkAuthorityScore !== void 0)) return pages;
10912
+ return pages.map((page) => page.internalInlinkCount !== void 0 && page.internalLinkAuthorityScore !== void 0 ? page : {
10913
+ ...page,
10914
+ internalInlinkCount: page.internalInlinkCount ?? 0,
10915
+ internalLinkAuthorityScore: page.internalLinkAuthorityScore ?? 0
10916
+ });
10917
+ }
10766
10918
  const pageByUrl = /* @__PURE__ */ new Map();
10767
10919
  for (const page of pages) for (const value of [page.url, page.finalUrl]) {
10768
10920
  const normalized = normalizeLinkUrl(value);
@@ -10781,7 +10933,8 @@ function deriveInternalLinkAuthority(pages, linkGraph) {
10781
10933
  }
10782
10934
  for (const linkedPage of linkedPages) inlinks.set(linkedPage, (inlinks.get(linkedPage) ?? 0) + 1);
10783
10935
  }
10784
- const maxInlinks = Math.max(0, ...inlinks.values());
10936
+ let maxInlinks = 0;
10937
+ for (const count of inlinks.values()) maxInlinks = Math.max(maxInlinks, count);
10785
10938
  return pages.map((page) => {
10786
10939
  const internalInlinkCount = inlinks.get(page.url) ?? 0;
10787
10940
  return {
@@ -11030,6 +11183,7 @@ function crawlCaveats(input) {
11030
11183
  const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
11031
11184
  if (input.queueSafetySkippedUrls > 0) caveats.push(`Left ${input.queueSafetySkippedUrls} eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
11032
11185
  if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
11186
+ if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
11033
11187
  return caveats;
11034
11188
  }
11035
11189
  //#endregion
@@ -11056,6 +11210,26 @@ function cancellationRace(signal) {
11056
11210
  });
11057
11211
  }
11058
11212
  //#endregion
11213
+ //#region packages/core/dist/analyze/crawler/crawl-memory.js
11214
+ const MEBIBYTE$1 = 1024 * 1024;
11215
+ const MIN_CRAWL_MEMORY_LIMIT = 384 * MEBIBYTE$1;
11216
+ const MAX_CRAWL_MEMORY_LIMIT = 640 * MEBIBYTE$1;
11217
+ const MIN_CRAWL_RSS_LIMIT = 512 * MEBIBYTE$1;
11218
+ const MAX_CRAWL_RSS_LIMIT = 896 * MEBIBYTE$1;
11219
+ const SYSTEM_MEMORY_FRACTION = .1;
11220
+ const SYSTEM_RSS_FRACTION = .2;
11221
+ function crawlMemoryLimitBytes(totalMemoryBytes) {
11222
+ if (!Number.isFinite(totalMemoryBytes) || totalMemoryBytes <= 0) return MAX_CRAWL_MEMORY_LIMIT;
11223
+ return Math.min(MAX_CRAWL_MEMORY_LIMIT, Math.max(MIN_CRAWL_MEMORY_LIMIT, Math.floor(totalMemoryBytes * SYSTEM_MEMORY_FRACTION)));
11224
+ }
11225
+ function crawlRssLimitBytes(totalMemoryBytes) {
11226
+ if (!Number.isFinite(totalMemoryBytes) || totalMemoryBytes <= 0) return MAX_CRAWL_RSS_LIMIT;
11227
+ return Math.min(MAX_CRAWL_RSS_LIMIT, Math.max(MIN_CRAWL_RSS_LIMIT, Math.floor(totalMemoryBytes * SYSTEM_RSS_FRACTION)));
11228
+ }
11229
+ function crawlMemoryPressure(input) {
11230
+ return input.memoryUsage.heapUsed + input.memoryUsage.external >= crawlMemoryLimitBytes(input.totalMemoryBytes) || input.memoryUsage.rss >= crawlRssLimitBytes(input.totalMemoryBytes);
11231
+ }
11232
+ //#endregion
11059
11233
  //#region packages/core/dist/analyze/crawler/crawl-url-queue.js
11060
11234
  function before(left, right) {
11061
11235
  return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
@@ -11773,6 +11947,7 @@ function crawlProviderLimits(input) {
11773
11947
  }
11774
11948
  async function crawlDataSources(input) {
11775
11949
  const window = crawlMetricsWindow(input.now());
11950
+ const skippedForMemory = input.skipReason === "memory-pressure";
11776
11951
  const dataSources = {
11777
11952
  searchConsole: {
11778
11953
  status: "skipped",
@@ -11783,7 +11958,7 @@ async function crawlDataSources(input) {
11783
11958
  pageLimit: input.searchMetricsLimit,
11784
11959
  pageLimitReached: false,
11785
11960
  retainedRowLimitReached: false,
11786
- warning: input.cancelled ? "Search Console join skipped because the crawl was cancelled." : "Search Console join skipped because no property was selected."
11961
+ warning: skippedForMemory ? "Search Console join skipped after the local memory safety limit was reached." : input.skipReason === "cancelled" ? "Search Console join skipped because the crawl was cancelled." : "Search Console join skipped because no property was selected."
11787
11962
  },
11788
11963
  analytics: {
11789
11964
  status: "skipped",
@@ -11792,10 +11967,10 @@ async function crawlDataSources(input) {
11792
11967
  joinedPages: 0,
11793
11968
  retainedRowLimit: input.analyticsLimit,
11794
11969
  retainedRowLimitReached: false,
11795
- warning: input.cancelled ? "Google Analytics join skipped because the crawl was cancelled." : "Google Analytics join skipped because no property was selected."
11970
+ warning: skippedForMemory ? "Google Analytics join skipped after the local memory safety limit was reached." : input.skipReason === "cancelled" ? "Google Analytics join skipped because the crawl was cancelled." : "Google Analytics join skipped because no property was selected."
11796
11971
  }
11797
11972
  };
11798
- if (!input.cancelled && input.site) dataSources.searchConsole = await joinSearchMetrics({
11973
+ if (!input.skipReason && input.site) dataSources.searchConsole = await joinSearchMetrics({
11799
11974
  site: input.site,
11800
11975
  pages: input.pages,
11801
11976
  warnings: input.warnings,
@@ -11808,7 +11983,7 @@ async function crawlDataSources(input) {
11808
11983
  queryPagesMetricsBatch: input.queryPagesMetricsBatch,
11809
11984
  queryPagesTopQueriesBatch: input.queryPagesTopQueriesBatch
11810
11985
  });
11811
- if (!input.cancelled && input.googleAnalyticsPropertyId) dataSources.analytics = await joinAnalytics({
11986
+ if (!input.skipReason && input.googleAnalyticsPropertyId) dataSources.analytics = await joinAnalytics({
11812
11987
  propertyId: input.googleAnalyticsPropertyId,
11813
11988
  pages: input.pages,
11814
11989
  warnings: input.warnings,
@@ -11821,17 +11996,6 @@ async function crawlDataSources(input) {
11821
11996
  }
11822
11997
  //#endregion
11823
11998
  //#region packages/core/dist/analyze/monitoring/crawl-page.js
11824
- function sameOriginUrl(href, base) {
11825
- try {
11826
- const url = new URL(href, base);
11827
- url.hash = "";
11828
- if (url.origin !== base.origin) return void 0;
11829
- if (!["http:", "https:"].includes(url.protocol)) return void 0;
11830
- return url.toString();
11831
- } catch {
11832
- return;
11833
- }
11834
- }
11835
11999
  function resolvedUrl(href, base) {
11836
12000
  if (!href) return void 0;
11837
12001
  try {
@@ -11861,49 +12025,6 @@ function safeResponseHeaders(headers) {
11861
12025
  function truncate$1(value, max) {
11862
12026
  return value.length > max ? value.slice(0, max).trimEnd() : value;
11863
12027
  }
11864
- function anchorSamples(links, internal) {
11865
- const seen = /* @__PURE__ */ new Set();
11866
- const samples = [];
11867
- for (const link of links) {
11868
- if (link.internal !== internal) continue;
11869
- const text = truncate$1(link.text.replace(/\s+/g, " ").trim(), 120);
11870
- if (!text) continue;
11871
- const key = `${link.href}\n${text}`;
11872
- if (seen.has(key)) continue;
11873
- seen.add(key);
11874
- samples.push({
11875
- href: link.href,
11876
- text
11877
- });
11878
- if (samples.length >= 25) break;
11879
- }
11880
- return samples;
11881
- }
11882
- function socialProfileLinks(links) {
11883
- const hosts = [
11884
- "facebook.com",
11885
- "instagram.com",
11886
- "linkedin.com",
11887
- "pinterest.com",
11888
- "tiktok.com",
11889
- "twitter.com",
11890
- "x.com",
11891
- "youtube.com"
11892
- ];
11893
- const profiles = /* @__PURE__ */ new Set();
11894
- for (const link of links) {
11895
- if (link.internal) continue;
11896
- try {
11897
- const url = new URL(link.href);
11898
- const host = url.hostname.replace(/^www\./, "").toLowerCase();
11899
- if (hosts.some((value) => host === value || host.endsWith(`.${value}`))) {
11900
- url.hash = "";
11901
- profiles.add(url.toString());
11902
- }
11903
- } catch {}
11904
- }
11905
- return [...profiles].slice(0, 50);
11906
- }
11907
12028
  function sameDocumentUrl(left, right) {
11908
12029
  if (!left || !right) return false;
11909
12030
  try {
@@ -12003,17 +12124,9 @@ function statusOnlyPage(fetched) {
12003
12124
  finalUrl: fetched.finalUrl,
12004
12125
  status: fetched.status,
12005
12126
  contentType,
12006
- responseHeaders: safeResponseHeaders(fetched.headers),
12007
12127
  responseTimeMs: fetched.diagnostics.durationMs,
12008
- usedJs: false,
12009
- fetchSource: "network",
12010
- cacheState: "bypass",
12011
- fetchDiagnostics: fetched.diagnostics,
12012
12128
  auditScope: "status",
12013
- accessBlock: fetched.diagnostics.accessBlock,
12014
- blocked: fetched.diagnostics.blocked,
12015
12129
  contentAuditAllowed: false,
12016
- robotsTxt: fetched.robotsTxt,
12017
12130
  indexable: false,
12018
12131
  indexability: "Not evaluated by status-only health probe",
12019
12132
  declaredIndexability: "unknown",
@@ -12021,8 +12134,7 @@ function statusOnlyPage(fetched) {
12021
12134
  wordCount: 0,
12022
12135
  contentHash: "",
12023
12136
  outgoingInternalCount: 0,
12024
- outgoingExternalCount: 0,
12025
- isHttps: new URL(fetched.finalUrl).protocol === "https:"
12137
+ outgoingExternalCount: 0
12026
12138
  };
12027
12139
  }
12028
12140
  function indexabilityReason(page) {
@@ -12098,12 +12210,11 @@ async function crawlOne(url, opts = {}) {
12098
12210
  };
12099
12211
  try {
12100
12212
  const extracted = await extractPage(fetched, "crawler");
12101
- const base = new URL(extracted.finalUrl);
12102
- const internalLinks = extracted.links.map((link) => sameOriginUrl(link.href, base)).filter((value) => Boolean(value));
12103
- const externalLinks = extracted.links.filter((link) => !link.internal).map((link) => link.href).filter((href) => /^https?:\/\//.test(href));
12104
- const h1 = extracted.headings.find((heading) => heading.level === 1)?.text;
12105
- const uniqueInternalLinks = [...new Set(internalLinks)];
12106
- const uniqueExternalLinks = [...new Set(externalLinks)];
12213
+ const crawlerEvidence = extracted.crawlerEvidence;
12214
+ if (!crawlerEvidence) throw new Error("Crawler extraction did not return crawl evidence.");
12215
+ const h1 = crawlerEvidence.h1;
12216
+ const uniqueInternalLinks = crawlerEvidence.internalLinks;
12217
+ const uniqueExternalLinks = crawlerEvidence.externalLinks;
12107
12218
  const canonical = extracted.canonicalEvidence?.selectedUrl ?? resolvedUrl(extracted.canonical, extracted.finalUrl);
12108
12219
  const richResults = selectGoogleRichResultAssessments(extracted.googleRichResults ?? []);
12109
12220
  const page = {
@@ -12129,9 +12240,9 @@ async function crawlOne(url, opts = {}) {
12129
12240
  metaRobots: extracted.metaRobots,
12130
12241
  xRobotsTag: extracted.xRobotsTag,
12131
12242
  h1,
12132
- h1Count: extracted.headings.filter((heading) => heading.level === 1).length,
12133
- h2Count: extracted.headings.filter((heading) => heading.level === 2).length,
12134
- h3Count: extracted.headings.filter((heading) => heading.level === 3).length,
12243
+ h1Count: crawlerEvidence.h1Count,
12244
+ h2Count: crawlerEvidence.h2Count,
12245
+ h3Count: crawlerEvidence.h3Count,
12135
12246
  indexable: false,
12136
12247
  extractionStatus: "complete",
12137
12248
  wordCount: extracted.wordCount,
@@ -12167,8 +12278,8 @@ async function crawlOne(url, opts = {}) {
12167
12278
  outgoingExternalCount: uniqueExternalLinks.length,
12168
12279
  sampleInternalLinks: uniqueInternalLinks.slice(0, 25),
12169
12280
  sampleExternalLinks: uniqueExternalLinks.slice(0, 25),
12170
- internalAnchorSamples: anchorSamples(extracted.links, true),
12171
- externalAnchorSamples: anchorSamples(extracted.links, false),
12281
+ internalAnchorSamples: crawlerEvidence.internalAnchorSamples,
12282
+ externalAnchorSamples: crawlerEvidence.externalAnchorSamples,
12172
12283
  markdownAlternates: extracted.markdownAlternates,
12173
12284
  schemaTypes: extracted.schemaTypes,
12174
12285
  structuredDataFormats: extracted.structuredDataFormats ?? [],
@@ -12177,7 +12288,7 @@ async function crawlOne(url, opts = {}) {
12177
12288
  schemaSameAs: [...new Set((extracted.schemaSameAsEvidence ?? []).map((evidence) => evidence.url))].slice(0, 50),
12178
12289
  schemaSameAsEvidence: (extracted.schemaSameAsEvidence ?? []).slice(0, 50),
12179
12290
  invalidSchemaSameAs: (extracted.invalidSchemaSameAs ?? []).slice(0, 25),
12180
- socialProfileLinks: socialProfileLinks(extracted.links),
12291
+ socialProfileLinks: crawlerEvidence.socialProfileLinks,
12181
12292
  invalidJsonLdCount: extracted.invalidJsonLdCount,
12182
12293
  invalidJsonLdSamples: extracted.invalidJsonLdSamples,
12183
12294
  unrecognizedJsonLdTypes: extracted.unrecognizedJsonLdTypes ?? [],
@@ -12830,7 +12941,9 @@ function resolveCrawlSiteDependencies(dependencies = {}) {
12830
12941
  fetchLandingPageValues: dependencies.fetchLandingPageValues ?? fetchLandingPageValues,
12831
12942
  landingValueForUrl: dependencies.landingValueForUrl ?? landingValueForUrl,
12832
12943
  collectAgentDiscovery: dependencies.collectAgentDiscovery ?? collectAgentDiscovery,
12833
- now: dependencies.now ?? (() => /* @__PURE__ */ new Date())
12944
+ now: dependencies.now ?? (() => /* @__PURE__ */ new Date()),
12945
+ memoryUsage: dependencies.memoryUsage ?? (() => process.memoryUsage()),
12946
+ totalMemory: dependencies.totalMemory ?? totalmem
12834
12947
  };
12835
12948
  }
12836
12949
  //#endregion
@@ -12986,15 +13099,26 @@ async function crawlSite(input, dependencies) {
12986
13099
  let queuedUrls = 0;
12987
13100
  let skippedUrls = 0;
12988
13101
  const skipReasonCounts = {};
12989
- const recordSkip = (reason) => {
12990
- skippedUrls += 1;
12991
- skipReasonCounts[reason] = (skipReasonCounts[reason] ?? 0) + 1;
13102
+ const recordSkips = (reason, count = 1) => {
13103
+ if (count <= 0) return;
13104
+ skippedUrls += count;
13105
+ skipReasonCounts[reason] = (skipReasonCounts[reason] ?? 0) + count;
12992
13106
  };
13107
+ const recordSkip = (reason) => recordSkips(reason);
12993
13108
  let robotsDeferredUrls = 0;
12994
13109
  let originBackpressureSkippedUrls = 0;
12995
13110
  let queueSafetySkippedUrls = 0;
12996
13111
  let failedUrls = 0;
12997
13112
  let observedInternalLinks = 0;
13113
+ let memoryLimited = false;
13114
+ const memoryDeferredUrls = /* @__PURE__ */ new Set();
13115
+ const checkMemoryPressure = () => {
13116
+ if (memoryLimited) return;
13117
+ memoryLimited = crawlMemoryPressure({
13118
+ memoryUsage: deps.memoryUsage(),
13119
+ totalMemoryBytes: deps.totalMemory()
13120
+ });
13121
+ };
12998
13122
  let sitemapDiscovery;
12999
13123
  let externalLinkVerification;
13000
13124
  let statusEventChain = Promise.resolve();
@@ -13124,6 +13248,7 @@ async function crawlSite(input, dependencies) {
13124
13248
  for (const url of sitemap.urls) enqueue(url, 0);
13125
13249
  sitemapDiscovery = sitemap.discovery;
13126
13250
  }
13251
+ checkMemoryPressure();
13127
13252
  const nextQueueItem = () => {
13128
13253
  while (queue.size) {
13129
13254
  const item = queue.take();
@@ -13149,6 +13274,7 @@ async function crawlSite(input, dependencies) {
13149
13274
  timeoutMs: config.timeoutMs,
13150
13275
  rate: config.fetchRate,
13151
13276
  signal,
13277
+ writeCache: false,
13152
13278
  respectRobots: config.respectRobots,
13153
13279
  robotsResolver,
13154
13280
  renderer
@@ -13176,8 +13302,8 @@ async function crawlSite(input, dependencies) {
13176
13302
  depth: item.depth
13177
13303
  });
13178
13304
  };
13179
- while (!isCancelled() && (queue.size || inFlight.size) && pages.length < config.maxPages) {
13180
- while (!isCancelled() && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
13305
+ while (!isCancelled() && (queue.size || inFlight.size) && (!memoryLimited || inFlight.size > 0) && pages.length < config.maxPages) {
13306
+ while (!isCancelled() && !memoryLimited && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
13181
13307
  const item = nextQueueItem();
13182
13308
  if (!item) break;
13183
13309
  startTask(item);
@@ -13205,6 +13331,7 @@ async function crawlSite(input, dependencies) {
13205
13331
  healthyProbeStreak = 0;
13206
13332
  activeConcurrency = 1;
13207
13333
  }
13334
+ checkMemoryPressure();
13208
13335
  if (request.outcome === "skipped") {
13209
13336
  recordSkip(request.reason === "robots-disallowed" ? "robots-disallowed" : request.reason === "robots-deferred" ? "robots-uncertain" : "origin-backpressure");
13210
13337
  if (request.reason === "robots-deferred") {
@@ -13335,10 +13462,24 @@ async function crawlSite(input, dependencies) {
13335
13462
  depth: task.depth,
13336
13463
  statusCode: page.status
13337
13464
  });
13338
- if (followLinks && task.depth < config.maxDepth) for (const url of result.urls) enqueue(url, task.depth + 1);
13465
+ if (followLinks && task.depth < config.maxDepth) if (memoryLimited) for (const value of result.urls) {
13466
+ const normalized = normalizeUrl$1(value, finalUrl);
13467
+ if (normalized && sameOrigin(normalized, origin) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
13468
+ discovered.add(normalized);
13469
+ memoryDeferredUrls.add(normalized);
13470
+ }
13471
+ }
13472
+ else for (const url of result.urls) enqueue(url, task.depth + 1);
13473
+ }
13474
+ const memoryPressureSkippedUrls = memoryLimited ? queue.size + memoryDeferredUrls.size : 0;
13475
+ const stoppedForMemory = memoryPressureSkippedUrls > 0;
13476
+ if (stoppedForMemory) {
13477
+ recordSkips("memory-pressure", memoryPressureSkippedUrls);
13478
+ warnings.push("Crawl stopped before local memory use could become unsafe. The retained report is partial.");
13339
13479
  }
13340
13480
  const resolvedInlinkCounts = resolveLinkCountAliases(incomingLinkCounts, fetchedAliases);
13341
- const maxInlinks = Math.max(0, ...pages.map((page) => resolvedInlinkCounts.get(page.finalUrl) ?? 0));
13481
+ let maxInlinks = 0;
13482
+ for (const page of pages) maxInlinks = Math.max(maxInlinks, resolvedInlinkCounts.get(page.finalUrl) ?? 0);
13342
13483
  for (const page of pages) {
13343
13484
  const internalInlinkCount = resolvedInlinkCounts.get(page.finalUrl) ?? 0;
13344
13485
  page.internalInlinkCount = internalInlinkCount;
@@ -13358,11 +13499,11 @@ async function crawlSite(input, dependencies) {
13358
13499
  const sitemapEvidencePartial = sitemapDiscovery?.dataStatus === "partial";
13359
13500
  const accessEvidencePartial = requests.some((request) => request.outcome === "response" && request.accessBlock);
13360
13501
  const requiredSitemapUnavailable = config.mode === "sitemap" && sitemapDiscovery?.dataStatus === "unavailable";
13361
- const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
13502
+ const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
13362
13503
  const failed = !cancelled && pages.length === 0 && (requiredSitemapUnavailable || requests.length > 0 && requests.every((request) => request.outcome === "failure"));
13363
13504
  const requestEvidenceStatus = cancelled && inFlight.size > 0 ? "partial" : "available";
13364
13505
  const dataSources = await crawlDataSources({
13365
- cancelled,
13506
+ skipReason: cancelled ? "cancelled" : stoppedForMemory ? "memory-pressure" : void 0,
13366
13507
  site: healthStrategy ? void 0 : site,
13367
13508
  googleAnalyticsPropertyId: healthStrategy ? void 0 : input.googleAnalyticsPropertyId,
13368
13509
  pages,
@@ -13381,7 +13522,7 @@ async function crawlSite(input, dependencies) {
13381
13522
  });
13382
13523
  const sourceEvidencePartial = [dataSources.searchConsole.status, dataSources.analytics.status].some((status) => status === "partial" || status === "unavailable");
13383
13524
  const reportStatus = failed ? "failed" : partial || sourceEvidencePartial ? "partial" : "completed";
13384
- if (!cancelled && config.checkExternal) {
13525
+ if (!cancelled && !stoppedForMemory && config.checkExternal) {
13385
13526
  emitStatus("external_links_started", {
13386
13527
  url: config.url,
13387
13528
  message: "Started external link checks."
@@ -13398,7 +13539,7 @@ async function crawlSite(input, dependencies) {
13398
13539
  message: "Finished external link checks."
13399
13540
  });
13400
13541
  }
13401
- const agentDiscovery = !cancelled && config.checkAgentDiscovery ? await deps.collectAgentDiscovery({
13542
+ const agentDiscovery = !cancelled && !stoppedForMemory && config.checkAgentDiscovery ? await deps.collectAgentDiscovery({
13402
13543
  startUrl: config.url,
13403
13544
  pages,
13404
13545
  timeoutMs: config.timeoutMs,
@@ -13439,7 +13580,8 @@ async function crawlSite(input, dependencies) {
13439
13580
  pageLimitReached,
13440
13581
  maxPages: config.maxPages,
13441
13582
  queueSafetySkippedUrls,
13442
- originBackpressureSkippedUrls
13583
+ originBackpressureSkippedUrls,
13584
+ memoryPressureSkippedUrls
13443
13585
  }),
13444
13586
  stats: {
13445
13587
  discoveredUrls: discovered.size,
@@ -15440,7 +15582,7 @@ function compareText$10(left, right) {
15440
15582
  }
15441
15583
  return leftPoints.length - rightPoints.length;
15442
15584
  }
15443
- function boundedInteger$5(value, fallback, minimum, maximum) {
15585
+ function boundedInteger$6(value, fallback, minimum, maximum) {
15444
15586
  if (value === void 0 || !Number.isFinite(value)) return fallback;
15445
15587
  return Math.min(maximum, Math.max(minimum, Math.floor(value)));
15446
15588
  }
@@ -15517,8 +15659,8 @@ function initialSelection(sourceRows) {
15517
15659
  };
15518
15660
  }
15519
15661
  function analyzeCtrUnderperformersFromRows(input) {
15520
- const minImpressions = boundedInteger$5(input.minImpressions, 200, 1, CTR_MAX_MIN_IMPRESSIONS);
15521
- const limit = boundedInteger$5(input.limit, 25, 1, 100);
15662
+ const minImpressions = boundedInteger$6(input.minImpressions, 200, 1, CTR_MAX_MIN_IMPRESSIONS);
15663
+ const limit = boundedInteger$6(input.limit, 25, 1, 100);
15522
15664
  const selection = initialSelection(input.rows.length);
15523
15665
  const benchmarkRows = aggregateCtrRows(input.rows, selection).filter((row) => {
15524
15666
  const query = row.keys[0] ?? "";
@@ -16017,7 +16159,7 @@ function analyzeSegmentRows(input) {
16017
16159
  }
16018
16160
  //#endregion
16019
16161
  //#region packages/core/dist/analyze/segment-impact-input.js
16020
- const DAY_MS$1 = 864e5;
16162
+ const DAY_MS$2 = 864e5;
16021
16163
  const MAX_WINDOW_DAYS = 240;
16022
16164
  function subtractDays$1(value, days) {
16023
16165
  const date = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
@@ -16025,7 +16167,7 @@ function subtractDays$1(value, days) {
16025
16167
  return date.toISOString().slice(0, 10);
16026
16168
  }
16027
16169
  function segmentRangeDays(range) {
16028
- return Math.floor((Date.parse(`${range.endDate}T00:00:00.000Z`) - Date.parse(`${range.startDate}T00:00:00.000Z`)) / DAY_MS$1) + 1;
16170
+ return Math.floor((Date.parse(`${range.endDate}T00:00:00.000Z`) - Date.parse(`${range.startDate}T00:00:00.000Z`)) / DAY_MS$2) + 1;
16029
16171
  }
16030
16172
  function segmentComparisonRange(after, compareDays) {
16031
16173
  const endDate = subtractDays$1(after.startDate, 1);
@@ -16605,13 +16747,13 @@ const MINIMUM_PAGE_SHARE = .1;
16605
16747
  const MINIMUM_PAGE_IMPRESSIONS = 10;
16606
16748
  const MAXIMUM_DOMINANT_SHARE = .8;
16607
16749
  const MAX_SUPPRESSIONS = 100;
16608
- function boundedInteger$4(value, fallback, minimum, maximum) {
16750
+ function boundedInteger$5(value, fallback, minimum, maximum) {
16609
16751
  if (value === void 0 || !Number.isFinite(value)) return fallback;
16610
16752
  return Math.min(maximum, Math.max(minimum, Math.floor(value)));
16611
16753
  }
16612
16754
  function analyzeCannibalRows(input) {
16613
- const minImpressions = boundedInteger$4(input.minImpressions, 50, 0, 1e9);
16614
- const limit = boundedInteger$4(input.limit, 25, 1, 100);
16755
+ const minImpressions = boundedInteger$5(input.minImpressions, 50, 0, 1e9);
16756
+ const limit = boundedInteger$5(input.limit, 25, 1, 100);
16615
16757
  const selection = {
16616
16758
  sourceRows: input.rows.length,
16617
16759
  invalidRows: 0,
@@ -17391,7 +17533,7 @@ function groupQuickWins(items) {
17391
17533
  }
17392
17534
  //#endregion
17393
17535
  //#region packages/core/dist/analyze/site-diagnostics/quick-wins-analysis-primitives.js
17394
- function boundedInteger$3(value, fallback, minimum, maximum) {
17536
+ function boundedInteger$4(value, fallback, minimum, maximum) {
17395
17537
  if (value === void 0 || !Number.isFinite(value)) return fallback;
17396
17538
  return Math.min(maximum, Math.max(minimum, Math.floor(value)));
17397
17539
  }
@@ -17512,12 +17654,12 @@ function roundedSum(values) {
17512
17654
  //#region packages/core/dist/analyze/site-diagnostics/quick-wins-analysis.js
17513
17655
  const DEFAULT_MIN_IMPRESSIONS$2 = 200;
17514
17656
  const MAX_MIN_IMPRESSIONS$2 = 1e9;
17515
- const DEFAULT_LIMIT$2 = 25;
17516
- const MAX_LIMIT$2 = 100;
17657
+ const DEFAULT_LIMIT$3 = 25;
17658
+ const MAX_LIMIT$3 = 100;
17517
17659
  const FALLBACK_SOURCE = "builtin_position_ctr_curve_v1";
17518
17660
  function analyzeQuickWinsFromRows(input) {
17519
- const minImpressions = boundedInteger$3(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$2, 0, MAX_MIN_IMPRESSIONS$2);
17520
- const limit = boundedInteger$3(input.limit, DEFAULT_LIMIT$2, 1, MAX_LIMIT$2);
17661
+ const minImpressions = boundedInteger$4(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$2, 0, MAX_MIN_IMPRESSIONS$2);
17662
+ const limit = boundedInteger$4(input.limit, DEFAULT_LIMIT$3, 1, MAX_LIMIT$3);
17521
17663
  const selected = selectBenchmarkRows(input);
17522
17664
  const benchmarkContext = createCtrBenchmarkContext(selected.rows, {
17523
17665
  samplePopulation: "all_qualified_url_samples",
@@ -17858,7 +18000,7 @@ function compareText$5(left, right) {
17858
18000
  }
17859
18001
  return leftPoints.length - rightPoints.length;
17860
18002
  }
17861
- function isHttpUrl(value) {
18003
+ function isHttpUrl$1(value) {
17862
18004
  try {
17863
18005
  const parsed = new URL(value);
17864
18006
  return parsed.protocol === "http:" || parsed.protocol === "https:";
@@ -17867,7 +18009,7 @@ function isHttpUrl(value) {
17867
18009
  }
17868
18010
  }
17869
18011
  function isValidStrikingDistanceRow(row, query, url) {
17870
- return Boolean(query) && isHttpUrl(url) && Number.isFinite(row.clicks) && Number.isFinite(row.impressions) && Number.isFinite(row.ctr) && Number.isFinite(row.position) && row.clicks >= 0 && row.impressions > 0 && row.clicks <= row.impressions && row.ctr >= 0 && row.ctr <= 1 && row.position > 0;
18012
+ return Boolean(query) && isHttpUrl$1(url) && Number.isFinite(row.clicks) && Number.isFinite(row.impressions) && Number.isFinite(row.ctr) && Number.isFinite(row.position) && row.clicks >= 0 && row.impressions > 0 && row.clicks <= row.impressions && row.ctr >= 0 && row.ctr <= 1 && row.position > 0;
17871
18013
  }
17872
18014
  function isStrikingDistanceBrandRow(input) {
17873
18015
  return shouldExcludeBrandQuery({
@@ -17957,8 +18099,8 @@ function groupStrikingDistanceItems(items) {
17957
18099
  //#region packages/core/dist/analyze/striking-distance-analysis.js
17958
18100
  const DEFAULT_MIN_IMPRESSIONS$1 = 100;
17959
18101
  const MAX_MIN_IMPRESSIONS$1 = 1e9;
17960
- const DEFAULT_LIMIT$1 = 25;
17961
- const MAX_LIMIT$1 = 100;
18102
+ const DEFAULT_LIMIT$2 = 25;
18103
+ const MAX_LIMIT$2 = 100;
17962
18104
  function integerInRange$1(value, fallback, minimum, maximum) {
17963
18105
  if (value === void 0 || !Number.isFinite(value)) return fallback;
17964
18106
  return Math.min(maximum, Math.max(minimum, Math.floor(value)));
@@ -18000,7 +18142,7 @@ function selectRows(input) {
18000
18142
  }
18001
18143
  function analyzeStrikingDistanceRows(input) {
18002
18144
  const minImpressions = integerInRange$1(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$1, 0, MAX_MIN_IMPRESSIONS$1);
18003
- const limit = integerInRange$1(input.limit, DEFAULT_LIMIT$1, 1, MAX_LIMIT$1);
18145
+ const limit = integerInRange$1(input.limit, DEFAULT_LIMIT$2, 1, MAX_LIMIT$2);
18004
18146
  const selected = selectRows({
18005
18147
  ...input,
18006
18148
  minImpressions
@@ -18074,7 +18216,7 @@ const defaultDependencies$7 = {
18074
18216
  verifyContent: verifyQueryContent,
18075
18217
  now: () => /* @__PURE__ */ new Date()
18076
18218
  };
18077
- function boundedInteger$2(value, fallback, minimum, maximum) {
18219
+ function boundedInteger$3(value, fallback, minimum, maximum) {
18078
18220
  if (value === void 0 || !Number.isFinite(value)) return fallback;
18079
18221
  return Math.min(maximum, Math.max(minimum, Math.floor(value)));
18080
18222
  }
@@ -18138,7 +18280,7 @@ async function strikingDistance(input, dependencies = defaultDependencies$7) {
18138
18280
  brandTerms: input.brandTerms,
18139
18281
  includeBrand: input.includeBrand
18140
18282
  });
18141
- const verifyLimit = boundedInteger$2(input.verifyLimit, DEFAULT_VERIFY_LIMIT, 0, analysis.limit);
18283
+ const verifyLimit = boundedInteger$3(input.verifyLimit, DEFAULT_VERIFY_LIMIT, 0, analysis.limit);
18142
18284
  const verificationRequested = input.verifyContent === true || input.verifyContent !== false && input.verifyLimit !== void 0;
18143
18285
  const attempted = verificationRequested ? Math.min(verifyLimit, analysis.items.length) : 0;
18144
18286
  const items = await Promise.all(analysis.items.map(async (item, index) => {
@@ -18327,7 +18469,7 @@ function standardDeviation(values) {
18327
18469
  function toFixedNumber(value, digits = 3) {
18328
18470
  return Number(value.toFixed(digits));
18329
18471
  }
18330
- function percentChange(baseline, comparison) {
18472
+ function percentChange$1(baseline, comparison) {
18331
18473
  if (!baseline) return comparison ? null : 0;
18332
18474
  return (comparison - baseline) / baseline * 100;
18333
18475
  }
@@ -18513,7 +18655,7 @@ function analyzeTrafficAnomalyRows(input) {
18513
18655
  const outsideFlatBaseline = zScore === null && comparisonMean !== baselineMean;
18514
18656
  const significant = outsideFlatBaseline || zScore !== null && Math.abs(zScore) >= 2;
18515
18657
  const direction = significant ? comparisonMean < baselineMean ? "drop" : "spike" : "normal";
18516
- const relativeChange = percentChange(baselineMean, comparisonMean);
18658
+ const relativeChange = percentChange$1(baselineMean, comparisonMean);
18517
18659
  return {
18518
18660
  site: input.site,
18519
18661
  metric,
@@ -19572,7 +19714,7 @@ function dateShift(date, days) {
19572
19714
  }
19573
19715
  //#endregion
19574
19716
  //#region packages/core/dist/analyze/experiments/measurement/window.js
19575
- const DAY_MS = 864e5;
19717
+ const DAY_MS$1 = 864e5;
19576
19718
  function exactDate(value, label) {
19577
19719
  const pattern = /^\d{4}-\d{2}-\d{2}$/;
19578
19720
  const parsed = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
@@ -19587,7 +19729,7 @@ function boundedDays(value, fallback, label) {
19587
19729
  function calendarDays(startDate, endDate) {
19588
19730
  const start = (/* @__PURE__ */ new Date(`${startDate}T00:00:00.000Z`)).getTime();
19589
19731
  const end = (/* @__PURE__ */ new Date(`${endDate}T00:00:00.000Z`)).getTime();
19590
- return Math.floor((end - start) / DAY_MS) + 1;
19732
+ return Math.floor((end - start) / DAY_MS$1) + 1;
19591
19733
  }
19592
19734
  function subtractCalendarMonths(value, months) {
19593
19735
  const source = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
@@ -20133,7 +20275,7 @@ function internalLinksVerdict(input) {
20133
20275
  function uniqueInternalLinksWarnings(warnings) {
20134
20276
  return [...new Map(warnings.map((warning) => [`${warning.stage}\u0000${warning.url}\u0000${warning.code}\u0000${warning.message}`, warning])).values()];
20135
20277
  }
20136
- function errorMessage(error) {
20278
+ function errorMessage$1(error) {
20137
20279
  return error instanceof Error ? error.message : String(error);
20138
20280
  }
20139
20281
  function acceptedAlias(site, value) {
@@ -20183,7 +20325,7 @@ async function verifyInternalLinkTarget(input) {
20183
20325
  stage: "target-fetch",
20184
20326
  url: input.targetUrl,
20185
20327
  code: "fetch-failed",
20186
- message: errorMessage(error)
20328
+ message: errorMessage$1(error)
20187
20329
  }]
20188
20330
  };
20189
20331
  }
@@ -20204,7 +20346,7 @@ async function verifyInternalLinkTarget(input) {
20204
20346
  stage: "target-extract",
20205
20347
  url: input.targetUrl,
20206
20348
  code: "extract-failed",
20207
- message: errorMessage(error)
20349
+ message: errorMessage$1(error)
20208
20350
  }]
20209
20351
  };
20210
20352
  }
@@ -20267,7 +20409,7 @@ async function verifyInternalLinkCandidate(input) {
20267
20409
  stage: "source-fetch",
20268
20410
  url: input.candidate.sourceUrl,
20269
20411
  code: "fetch-failed",
20270
- message: errorMessage(error)
20412
+ message: errorMessage$1(error)
20271
20413
  }]
20272
20414
  };
20273
20415
  }
@@ -20285,7 +20427,7 @@ async function verifyInternalLinkCandidate(input) {
20285
20427
  stage: "source-fetch",
20286
20428
  url: input.candidate.sourceUrl,
20287
20429
  code: "fetch-failed",
20288
- message: errorMessage(error)
20430
+ message: errorMessage$1(error)
20289
20431
  }]
20290
20432
  };
20291
20433
  }
@@ -20299,7 +20441,7 @@ async function verifyInternalLinkCandidate(input) {
20299
20441
  stage: "source-extract",
20300
20442
  url: input.candidate.sourceUrl,
20301
20443
  code: "extract-failed",
20302
- message: errorMessage(error)
20444
+ message: errorMessage$1(error)
20303
20445
  }]
20304
20446
  };
20305
20447
  }
@@ -21107,7 +21249,7 @@ function compareText$3(left, right) {
21107
21249
  }
21108
21250
  return leftPoints.length - rightPoints.length;
21109
21251
  }
21110
- function boundedInteger$1(value, fallback, maximum) {
21252
+ function boundedInteger$2(value, fallback, maximum) {
21111
21253
  if (value === void 0 || !Number.isFinite(value)) return fallback;
21112
21254
  return Math.min(maximum, Math.max(1, Math.trunc(value)));
21113
21255
  }
@@ -21300,10 +21442,10 @@ function caveats(input) {
21300
21442
  return items;
21301
21443
  }
21302
21444
  function analyzeIndexCoverageSignals(input) {
21303
- const itemLimit = boundedInteger$1(input.limits?.itemsPerSection, DEFAULT_ITEMS_PER_SECTION, MAX_ITEMS_PER_SECTION);
21304
- const clusterLimit = boundedInteger$1(input.limits?.templateClusters, DEFAULT_TEMPLATE_LIMIT, MAX_TEMPLATE_LIMIT);
21305
- const clusterSamples = boundedInteger$1(input.limits?.templateSamples, DEFAULT_TEMPLATE_SAMPLES, MAX_TEMPLATE_SAMPLES);
21306
- const minUrls = boundedInteger$1(input.templateThresholds?.minUrls, DEFAULT_TEMPLATE_MIN_URLS, Number.MAX_SAFE_INTEGER);
21445
+ const itemLimit = boundedInteger$2(input.limits?.itemsPerSection, DEFAULT_ITEMS_PER_SECTION, MAX_ITEMS_PER_SECTION);
21446
+ const clusterLimit = boundedInteger$2(input.limits?.templateClusters, DEFAULT_TEMPLATE_LIMIT, MAX_TEMPLATE_LIMIT);
21447
+ const clusterSamples = boundedInteger$2(input.limits?.templateSamples, DEFAULT_TEMPLATE_SAMPLES, MAX_TEMPLATE_SAMPLES);
21448
+ const minUrls = boundedInteger$2(input.templateThresholds?.minUrls, DEFAULT_TEMPLATE_MIN_URLS, Number.MAX_SAFE_INTEGER);
21307
21449
  const requestedMinShare = input.templateThresholds?.minShare;
21308
21450
  const minShare = Number.isFinite(requestedMinShare) ? Math.min(1, Math.max(0, requestedMinShare ?? DEFAULT_TEMPLATE_MIN_SHARE)) : DEFAULT_TEMPLATE_MIN_SHARE;
21309
21451
  const crawl = crawlInput(input.crawl);
@@ -24990,16 +25132,16 @@ function benchmarkDetails(benchmark) {
24990
25132
  positiveUrlSamples: benchmark.positiveUrlSamples
24991
25133
  };
24992
25134
  }
24993
- function boundedInteger(input) {
25135
+ function boundedInteger$1(input) {
24994
25136
  const value = Number.isFinite(input.value) ? input.value ?? 0 : input.fallback;
24995
25137
  return Math.min(input.maximum ?? Number.POSITIVE_INFINITY, Math.max(1, Math.floor(value)));
24996
25138
  }
24997
25139
  function analyzeQueryClustersFromRows(input) {
24998
- const minImpressions = boundedInteger({
25140
+ const minImpressions = boundedInteger$1({
24999
25141
  value: input.minImpressions,
25000
25142
  fallback: 25
25001
25143
  });
25002
- const limit = boundedInteger({
25144
+ const limit = boundedInteger$1({
25003
25145
  value: input.limit,
25004
25146
  fallback: 25,
25005
25147
  maximum: 100
@@ -25641,8 +25783,8 @@ function reportPresentation(report) {
25641
25783
  //#region packages/core/dist/analyze/second-page-analysis.js
25642
25784
  const DEFAULT_MIN_IMPRESSIONS = 50;
25643
25785
  const MAX_MIN_IMPRESSIONS = 1e9;
25644
- const DEFAULT_LIMIT = 10;
25645
- const MAX_LIMIT = 100;
25786
+ const DEFAULT_LIMIT$1 = 10;
25787
+ const MAX_LIMIT$1 = 100;
25646
25788
  function integerInRange(value, fallback, minimum, maximum) {
25647
25789
  if (value === void 0 || !Number.isFinite(value)) return fallback;
25648
25790
  return Math.min(maximum, Math.max(minimum, Math.floor(value)));
@@ -25729,7 +25871,7 @@ function compareItems(left, right) {
25729
25871
  }
25730
25872
  function analyzeSecondPageRows(input) {
25731
25873
  const minImpressions = integerInRange(input.minImpressions, DEFAULT_MIN_IMPRESSIONS, 0, MAX_MIN_IMPRESSIONS);
25732
- const limit = integerInRange(input.limit, DEFAULT_LIMIT, 1, MAX_LIMIT);
25874
+ const limit = integerInRange(input.limit, DEFAULT_LIMIT$1, 1, MAX_LIMIT$1);
25733
25875
  const selection = {
25734
25876
  sourceRows: input.rows.length,
25735
25877
  invalidRows: 0,
@@ -27127,138 +27269,1489 @@ async function updatePostmortemWorkflow(input, dependencies = {}) {
27127
27269
  });
27128
27270
  }
27129
27271
  //#endregion
27130
- //#region packages/core/dist/doctor.js
27131
- const REQUIRED_SCOPES = ["https://www.googleapis.com/auth/webmasters.readonly", "https://www.googleapis.com/auth/analytics.readonly"];
27132
- async function runDoctor(dependencies = {}) {
27133
- const paths = getSeoCliPaths();
27134
- const config = readConfig();
27135
- const authMode = getAuthModeStatus();
27136
- const serviceAccount = getServiceAccountStatus();
27137
- const tokens = await readTokens();
27138
- const oauthClient = readOauthClient();
27139
- const tokenClientConfigured = tokens ? Boolean(getClientConfig(tokens.client_source)) : void 0;
27140
- const checks = [];
27141
- try {
27142
- const database = (dependencies.checkDatabase ?? checkDatabaseReadiness)();
27143
- checks.push({
27144
- id: "local-database",
27145
- label: "Local database",
27146
- status: "pass",
27147
- detail: database.dbPath
27148
- });
27149
- } catch (error) {
27150
- checks.push({
27151
- id: "local-database",
27152
- label: "Local database",
27153
- status: "fail",
27154
- detail: error instanceof Error ? error.message : "The local database could not be opened.",
27155
- fix: "Upgrade or reinstall `seo`. If this still fails, report it at https://github.com/iannuttall/seo/issues."
27156
- });
27272
+ //#region packages/core/dist/bing/analysis.js
27273
+ const DAY_MS = 864e5;
27274
+ const DEFAULT_COMPARISON_DAYS = 28;
27275
+ const WEEKLY_PERIODS = 4;
27276
+ const OPPORTUNITY_MIN_IMPRESSIONS = 20;
27277
+ const OPPORTUNITY_MIN_POSITION = 4;
27278
+ const OPPORTUNITY_MAX_POSITION = 20;
27279
+ const RESULT_LIMIT = 10;
27280
+ function dateOffset(date, days) {
27281
+ return new Date(Date.parse(`${date}T00:00:00Z`) + days * DAY_MS).toISOString().slice(0, 10);
27282
+ }
27283
+ function percentChange(previous, current) {
27284
+ if (previous === 0) return null;
27285
+ return round((current - previous) / previous * 100);
27286
+ }
27287
+ function round(value, digits = 3) {
27288
+ const factor = 10 ** digits;
27289
+ return Math.round(value * factor) / factor;
27290
+ }
27291
+ function trafficPeriod(byDate, startDate, endDate, expectedDays) {
27292
+ let clicks = 0;
27293
+ let impressions = 0;
27294
+ let observedDays = 0;
27295
+ for (const [date, row] of byDate) {
27296
+ if (date < startDate || date > endDate) continue;
27297
+ observedDays += 1;
27298
+ clicks += row.clicks;
27299
+ impressions += row.impressions;
27157
27300
  }
27158
- checks.push({
27159
- id: "config-dir",
27160
- label: "Config directory",
27161
- status: existsSync(paths.configDir) ? "pass" : "warn",
27162
- detail: paths.configDir,
27163
- fix: existsSync(paths.configDir) ? void 0 : "Run `seo start` or `seo auth setup-client`."
27164
- });
27165
- checks.push({
27166
- id: "oauth-client",
27167
- label: "Google credentials",
27168
- status: serviceAccount.configured ? "pass" : tokenClientConfigured === false ? "fail" : serviceAccount.error ? "fail" : authMode.sharedConfigured || authMode.byoConfigured ? "pass" : "fail",
27169
- detail: serviceAccount.configured ? `Service account configured for ${serviceAccount.identity}.` : tokenClientConfigured === false ? `Stored Google login uses the ${tokens?.client_source === "shared" ? "shared seo app" : "BYO client"}, but that client is not configured.` : serviceAccount.error ? serviceAccount.error : authMode.sharedConfigured ? "Shared client configured." : authMode.byoConfigured ? `BYO client configured at ${paths.oauthClientFile}.` : "No shared or BYO Google OAuth client configured.",
27170
- fix: serviceAccount.configured ? void 0 : tokenClientConfigured === false ? tokens?.client_source === "shared" ? "Reinstall `seo`. If the shared client is still missing, report it at https://github.com/iannuttall/seo/issues." : "Restore the same client with `seo auth setup-client`, or run `seo auth logout` before signing in with a different client." : serviceAccount.error ? "Set one valid service account credential source, then run `seo doctor` again." : authMode.sharedConfigured || authMode.byoConfigured ? void 0 : "Run `seo auth setup-client` or set SEO_GOOGLE_CLIENT_ID and SEO_GOOGLE_CLIENT_SECRET."
27171
- });
27172
- checks.push({
27173
- id: "oauth-client-file",
27174
- label: "BYO OAuth file",
27175
- status: oauthClient ? "pass" : "warn",
27176
- detail: oauthClient ? paths.oauthClientFile : "No BYO OAuth client file. This is fine if using a shared client."
27177
- });
27178
- checks.push({
27179
- id: "google-login",
27180
- label: "Google login",
27181
- status: serviceAccount.configured || tokens ? "pass" : "fail",
27182
- detail: serviceAccount.configured ? `Using service account ${serviceAccount.identity}.` : tokens ? `Signed in as ${tokens.account_email}.` : "No Google token found.",
27183
- fix: serviceAccount.configured || tokens ? void 0 : "Run `seo auth login`, or configure a service account for CI."
27184
- });
27185
- const grantedScopes = new Set(tokens?.scope.split(/\s+/).filter(Boolean) ?? []);
27186
- for (const scope of REQUIRED_SCOPES) checks.push({
27187
- id: `scope:${scope}`,
27188
- label: scope.endsWith("webmasters.readonly") ? "Search Console scope" : "Google Analytics scope",
27189
- status: serviceAccount.configured || tokens && grantedScopes.has(scope) ? "pass" : tokens ? "fail" : "warn",
27190
- detail: serviceAccount.configured ? `${scope} requested by the service account.` : tokens ? grantedScopes.has(scope) ? scope : `Missing ${scope}.` : "Cannot inspect scopes until login is complete.",
27191
- fix: !serviceAccount.configured && tokens && !grantedScopes.has(scope) ? "Run `seo auth logout` then `seo auth login`." : void 0
27192
- });
27193
- checks.push({
27194
- id: "default-site",
27195
- label: "Saved GSC property",
27196
- status: "pass",
27197
- detail: config.defaultSite ?? "No saved default. Human CLI commands will prompt; agents can pass --site."
27198
- });
27199
- checks.push({
27200
- id: "default-google-analytics",
27201
- label: "Saved Google Analytics property",
27202
- status: "pass",
27203
- detail: config.analytics.google.defaultPropertyId ?? "No saved default. Human CLI commands will prompt; agents can pass --property."
27204
- });
27205
27301
  return {
27206
- ok: checks.every((check) => check.status !== "fail"),
27207
- generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
27208
- checks
27302
+ startDate,
27303
+ endDate,
27304
+ expectedDays,
27305
+ observedDays,
27306
+ missingDays: Math.max(0, expectedDays - observedDays),
27307
+ clicks,
27308
+ impressions,
27309
+ ctr: impressions ? round(clicks / impressions * 100) : null
27209
27310
  };
27210
27311
  }
27211
- //#endregion
27212
- //#region packages/core/dist/providers/dataforseo.js
27213
- var DataForSeoProvider = class {
27214
- name = "dataforseo";
27215
- capabilities = { overview: true };
27216
- getAuthHeader() {
27217
- const config = readConfig();
27218
- const login = config.providers.dataForSeoLogin;
27219
- const password = config.providers.dataForSeoPassword;
27220
- if (!login || !password) throw new Error("DataForSEO credentials missing. Add login and password to config.json.");
27221
- return `Basic ${Buffer.from(`${login}:${password}`).toString("base64")}`;
27222
- }
27223
- async keywordOverview(phrase, opts = {}) {
27224
- const response = await fetch$1("https://api.dataforseo.com/v3/dataforseo_labs/google/keyword_overview/live", {
27225
- method: "POST",
27226
- headers: {
27227
- authorization: this.getAuthHeader(),
27228
- "content-type": "application/json"
27229
- },
27230
- body: JSON.stringify([{
27231
- keyword: phrase,
27232
- language_code: "en",
27233
- location_code: 2840
27234
- }])
27235
- });
27236
- if (!response.ok) throw new Error(`DataForSEO request failed with ${response.status}.`);
27237
- const json = await response.json();
27238
- const row = json.tasks?.[0]?.result?.[0];
27239
- return {
27240
- data: {
27241
- phrase: row?.keyword ?? phrase,
27242
- volume: row?.search_volume,
27243
- competition: row?.competition,
27244
- cpc: row?.cpc,
27245
- difficulty: row?.keyword_difficulty
27246
- },
27247
- usage: {
27248
- provider: "DataForSEO",
27249
- units: 1,
27250
- unitLabel: "tasks",
27251
- estimatedUsd: json.cost,
27252
- calls: 1
27253
- },
27254
- warnings: opts.database ? ["DataForSEO overview currently ignores the database option."] : void 0
27312
+ function analyzeBingTraffic(rows, comparisonDays = DEFAULT_COMPARISON_DAYS) {
27313
+ if (!rows.length) return void 0;
27314
+ const byDate = /* @__PURE__ */ new Map();
27315
+ for (const row of rows) {
27316
+ const existing = byDate.get(row.date) ?? {
27317
+ clicks: 0,
27318
+ impressions: 0
27319
+ };
27320
+ existing.clicks += row.clicks;
27321
+ existing.impressions += row.impressions;
27322
+ byDate.set(row.date, existing);
27323
+ }
27324
+ const latestDate = [...byDate.keys()].sort().at(-1);
27325
+ if (!latestDate) return void 0;
27326
+ const currentStart = dateOffset(latestDate, -(comparisonDays - 1));
27327
+ const previousEnd = dateOffset(currentStart, -1);
27328
+ const previousStart = dateOffset(previousEnd, -(comparisonDays - 1));
27329
+ const current = trafficPeriod(byDate, currentStart, latestDate, comparisonDays);
27330
+ const previous = trafficPeriod(byDate, previousStart, previousEnd, comparisonDays);
27331
+ return {
27332
+ comparisonDays,
27333
+ current,
27334
+ previous,
27335
+ changes: {
27336
+ clicks: current.clicks - previous.clicks,
27337
+ clicksPercent: percentChange(previous.clicks, current.clicks),
27338
+ impressions: current.impressions - previous.impressions,
27339
+ impressionsPercent: percentChange(previous.impressions, current.impressions),
27340
+ ctrPercentagePoints: current.ctr === null || previous.ctr === null ? null : round(current.ctr - previous.ctr)
27341
+ }
27342
+ };
27343
+ }
27344
+ const CRAWL_FIELDS = [
27345
+ "crawledPages",
27346
+ "inIndex",
27347
+ "inLinks",
27348
+ "crawlErrors",
27349
+ "code4xx",
27350
+ "code5xx",
27351
+ "blockedByRobotsTxt",
27352
+ "connectionTimeout",
27353
+ "dnsFailures",
27354
+ "containsMalware"
27355
+ ];
27356
+ function analyzeBingCrawl(rows, comparisonDays = DEFAULT_COMPARISON_DAYS) {
27357
+ const ordered = [...rows].sort((a, b) => a.date.localeCompare(b.date, "en"));
27358
+ const current = ordered.at(-1);
27359
+ if (!current) return void 0;
27360
+ const targetDate = dateOffset(current.date, -comparisonDays);
27361
+ const previous = ordered.filter((row) => row.date <= targetDate).at(-1);
27362
+ const changes = {};
27363
+ if (!previous) return {
27364
+ comparisonDays,
27365
+ targetDate,
27366
+ current,
27367
+ previous: void 0,
27368
+ changes
27369
+ };
27370
+ for (const field of CRAWL_FIELDS) {
27371
+ const currentValue = current[field];
27372
+ const previousValue = previous[field];
27373
+ if (typeof currentValue !== "number" || typeof previousValue !== "number") continue;
27374
+ changes[field] = {
27375
+ previous: previousValue,
27376
+ current: currentValue,
27377
+ absolute: currentValue - previousValue,
27378
+ percent: percentChange(previousValue, currentValue)
27255
27379
  };
27256
27380
  }
27257
- };
27258
- //#endregion
27259
- //#region packages/core/dist/providers/semrush/csv.js
27260
- function parseSemicolonCsv(text) {
27261
- return text.trim().split("\n").map((line) => line.split(";").map((cell) => cell.replace(/^"|"$/g, "").trim()));
27381
+ return {
27382
+ comparisonDays,
27383
+ targetDate,
27384
+ current,
27385
+ previous,
27386
+ changes
27387
+ };
27388
+ }
27389
+ function aggregateDimensions(rows, dates) {
27390
+ const values = /* @__PURE__ */ new Map();
27391
+ for (const row of rows) {
27392
+ if (!dates.has(row.date)) continue;
27393
+ const item = values.get(row.value) ?? {
27394
+ clicks: 0,
27395
+ impressions: 0,
27396
+ clickPositionTotal: 0,
27397
+ clickPositionWeight: 0,
27398
+ impressionPositionTotal: 0,
27399
+ impressionPositionWeight: 0,
27400
+ observedDates: /* @__PURE__ */ new Set()
27401
+ };
27402
+ item.clicks += row.clicks;
27403
+ item.impressions += row.impressions;
27404
+ item.observedDates.add(row.date);
27405
+ if (row.avgClickPosition !== void 0 && row.clicks > 0) {
27406
+ const weight = row.clicks;
27407
+ item.clickPositionTotal += row.avgClickPosition * weight;
27408
+ item.clickPositionWeight += weight;
27409
+ }
27410
+ if (row.avgImpressionPosition !== void 0 && row.impressions > 0) {
27411
+ const weight = row.impressions;
27412
+ item.impressionPositionTotal += row.avgImpressionPosition * weight;
27413
+ item.impressionPositionWeight += weight;
27414
+ }
27415
+ values.set(row.value, item);
27416
+ }
27417
+ return new Map([...values].map(([value, item]) => [value, {
27418
+ value,
27419
+ observedWeeks: item.observedDates.size,
27420
+ clicks: item.clicks,
27421
+ impressions: item.impressions,
27422
+ avgClickPosition: item.clickPositionWeight ? round(item.clickPositionTotal / item.clickPositionWeight) : null,
27423
+ avgImpressionPosition: item.impressionPositionWeight ? round(item.impressionPositionTotal / item.impressionPositionWeight) : null
27424
+ }]));
27425
+ }
27426
+ function compareValue(a, b) {
27427
+ return a.value.localeCompare(b.value, "en");
27428
+ }
27429
+ function analyzeBingDimensions(rows, kind) {
27430
+ const dates = [...new Set(rows.map((row) => row.date))].sort().reverse();
27431
+ const currentDates = dates.slice(0, WEEKLY_PERIODS);
27432
+ const previousDates = dates.slice(WEEKLY_PERIODS, WEEKLY_PERIODS * 2);
27433
+ const current = aggregateDimensions(rows, new Set(currentDates));
27434
+ const previous = aggregateDimensions(rows, new Set(previousDates));
27435
+ const matchedValues = [...current.keys()].filter((value) => previous.has(value));
27436
+ const comparableValues = currentDates.length === WEEKLY_PERIODS && previousDates.length === WEEKLY_PERIODS ? matchedValues.filter((value) => {
27437
+ const currentValue = current.get(value);
27438
+ const previousValue = previous.get(value);
27439
+ return currentValue?.observedWeeks === WEEKLY_PERIODS && previousValue?.observedWeeks === WEEKLY_PERIODS;
27440
+ }) : [];
27441
+ const movements = comparableValues.map((value) => {
27442
+ const currentValue = current.get(value);
27443
+ const previousValue = previous.get(value);
27444
+ return {
27445
+ value,
27446
+ current: currentValue,
27447
+ previous: previousValue,
27448
+ changes: {
27449
+ clicks: currentValue.clicks - previousValue.clicks,
27450
+ clicksPercent: percentChange(previousValue.clicks, currentValue.clicks),
27451
+ impressions: currentValue.impressions - previousValue.impressions,
27452
+ impressionsPercent: percentChange(previousValue.impressions, currentValue.impressions)
27453
+ }
27454
+ };
27455
+ }).sort((a, b) => Math.abs(b.changes.clicks) - Math.abs(a.changes.clicks) || Math.abs(b.changes.impressions) - Math.abs(a.changes.impressions) || compareValue(a, b)).slice(0, RESULT_LIMIT);
27456
+ const opportunities = [...current.values()].filter((row) => row.impressions >= OPPORTUNITY_MIN_IMPRESSIONS && row.avgImpressionPosition !== null && row.avgImpressionPosition >= OPPORTUNITY_MIN_POSITION && row.avgImpressionPosition <= OPPORTUNITY_MAX_POSITION).sort((a, b) => b.impressions - a.impressions || b.clicks - a.clicks || compareValue(a, b)).slice(0, RESULT_LIMIT);
27457
+ return {
27458
+ kind,
27459
+ sourceSemantics: "Bing weekly top-list evidence. Missing dimensions are unknown, not zero. Movements require observation in every week of both periods.",
27460
+ periods: {
27461
+ weeksPerPeriod: WEEKLY_PERIODS,
27462
+ currentDates,
27463
+ previousDates
27464
+ },
27465
+ coverage: {
27466
+ sourceRows: rows.length,
27467
+ currentDimensions: current.size,
27468
+ previousDimensions: previous.size,
27469
+ matchedDimensions: matchedValues.length,
27470
+ comparableDimensions: comparableValues.length,
27471
+ incompleteMatchedDimensions: matchedValues.length - comparableValues.length,
27472
+ currentOnlyDimensions: [...current.keys()].filter((value) => !previous.has(value)).length,
27473
+ previousOnlyDimensions: [...previous.keys()].filter((value) => !current.has(value)).length
27474
+ },
27475
+ thresholds: {
27476
+ heuristic: true,
27477
+ opportunityMinImpressions: OPPORTUNITY_MIN_IMPRESSIONS,
27478
+ opportunityPositionRange: [OPPORTUNITY_MIN_POSITION, OPPORTUNITY_MAX_POSITION],
27479
+ resultLimit: RESULT_LIMIT
27480
+ },
27481
+ movements,
27482
+ opportunities
27483
+ };
27484
+ }
27485
+ //#endregion
27486
+ //#region packages/core/dist/bing/client.js
27487
+ const DEFAULT_BASE_URL = "https://ssl.bing.com/webmaster/api.svc/json";
27488
+ const MAX_RESPONSE_BYTES$1 = 2e6;
27489
+ const MAX_STAT_ROWS = 400;
27490
+ const MAX_DIMENSION_ROWS = 8e3;
27491
+ const MAX_LINK_ROWS = 200;
27492
+ function bingDate(value) {
27493
+ if (typeof value !== "string") return void 0;
27494
+ const legacy = /^\/Date\((-?\d+)(?:[+-]\d{4})?\)\/$/.exec(value);
27495
+ const date = legacy ? new Date(Number(legacy[1])) : new Date(value);
27496
+ return Number.isNaN(date.getTime()) ? void 0 : date.toISOString().slice(0, 10);
27497
+ }
27498
+ function nonnegativeNumber(value) {
27499
+ return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
27500
+ }
27501
+ function positionNumber(value) {
27502
+ if (value === void 0 || value === null || value === -1) return { valid: true };
27503
+ const number = nonnegativeNumber(value);
27504
+ return number === void 0 ? { valid: false } : {
27505
+ valid: true,
27506
+ value: number
27507
+ };
27508
+ }
27509
+ function isHttpUrl(value) {
27510
+ if (typeof value !== "string" || value.length > 2e3) return false;
27511
+ try {
27512
+ const url = new URL(value);
27513
+ return url.protocol === "http:" || url.protocol === "https:";
27514
+ } catch {
27515
+ return false;
27516
+ }
27517
+ }
27518
+ function boundedRows(rows, invalidRows, limit = MAX_STAT_ROWS) {
27519
+ const capped = rows.length > limit;
27520
+ return {
27521
+ rows: capped ? rows.slice(-limit) : rows,
27522
+ invalidRows,
27523
+ capped,
27524
+ returnedRows: rows.length
27525
+ };
27526
+ }
27527
+ function dimensionValue(value, kind) {
27528
+ if (typeof value !== "string") return void 0;
27529
+ const normalized = value.trim();
27530
+ if (!normalized || normalized.length > 2e3) return void 0;
27531
+ if (kind === "page" && !isHttpUrl(normalized)) return void 0;
27532
+ return normalized;
27533
+ }
27534
+ function errorMessage(body) {
27535
+ try {
27536
+ const parsed = JSON.parse(body);
27537
+ return typeof parsed.Message === "string" ? parsed.Message : void 0;
27538
+ } catch {
27539
+ return;
27540
+ }
27541
+ }
27542
+ var BingWebmasterClient = class {
27543
+ credentials;
27544
+ authentication;
27545
+ baseUrl;
27546
+ fetchImpl;
27547
+ timeoutMs;
27548
+ constructor(credentials, options = {}) {
27549
+ this.credentials = credentials;
27550
+ if (!(credentials.apiKey?.trim() || credentials.accessToken?.trim())) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster credentials are empty.");
27551
+ this.authentication = credentials.apiKey ? "api-key" : "oauth";
27552
+ this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, "");
27553
+ this.fetchImpl = options.fetchImpl ?? publicHttpFetch;
27554
+ this.timeoutMs = options.timeoutMs ?? 3e4;
27555
+ }
27556
+ async request(method, params = {}) {
27557
+ const url = new URL(`${this.baseUrl}/${method}`);
27558
+ for (const [key, value] of Object.entries(params)) url.searchParams.set(key, value);
27559
+ if (this.credentials.apiKey) url.searchParams.set("apikey", this.credentials.apiKey);
27560
+ let response;
27561
+ try {
27562
+ response = await this.fetchImpl(url.toString(), {
27563
+ headers: {
27564
+ accept: "application/json",
27565
+ ...this.credentials.accessToken ? { authorization: `Bearer ${this.credentials.accessToken}` } : {}
27566
+ },
27567
+ signal: AbortSignal.timeout(this.timeoutMs)
27568
+ });
27569
+ } catch (error) {
27570
+ if (error instanceof SeoError) throw error;
27571
+ throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster did not respond: ${error instanceof Error ? error.message : String(error)}`);
27572
+ }
27573
+ const body = await readBoundedResponseText(response, MAX_RESPONSE_BYTES$1, "Bing Webmaster response");
27574
+ if (!response.ok) {
27575
+ const message = errorMessage(body);
27576
+ if (response.status === 401 || /invalidapikey|unauthorized/i.test(message ?? "")) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster rejected the saved credential. Run `seo providers bing connect` again.");
27577
+ if (response.status === 403) throw new SeoError("ACCESS_DENIED", "Bing Webmaster denied access to this site. Check that it is verified for the connected account.");
27578
+ if (response.status === 429) throw new SeoError("RATE_LIMITED", "Bing Webmaster rate limited this request. Try again later.");
27579
+ throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster request failed with HTTP ${response.status}${message ? ` (${message})` : ""}.`);
27580
+ }
27581
+ try {
27582
+ return JSON.parse(body).d;
27583
+ } catch {
27584
+ throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid JSON.");
27585
+ }
27586
+ }
27587
+ async listSites() {
27588
+ const data = await this.request("GetUserSites");
27589
+ if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned an invalid site list.");
27590
+ const sites = [];
27591
+ let invalidRows = 0;
27592
+ for (const item of data) {
27593
+ if (!item || typeof item !== "object") {
27594
+ invalidRows += 1;
27595
+ continue;
27596
+ }
27597
+ const row = item;
27598
+ if (typeof row.Url !== "string" || typeof row.IsVerified !== "boolean") {
27599
+ invalidRows += 1;
27600
+ continue;
27601
+ }
27602
+ sites.push({
27603
+ url: row.Url,
27604
+ isVerified: row.IsVerified
27605
+ });
27606
+ }
27607
+ sites.sort((a, b) => a.url.localeCompare(b.url, "en"));
27608
+ return {
27609
+ sites,
27610
+ invalidRows
27611
+ };
27612
+ }
27613
+ async getTraffic(siteUrl) {
27614
+ const data = await this.request("GetRankAndTrafficStats", { siteUrl });
27615
+ if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid traffic statistics.");
27616
+ const rows = [];
27617
+ let invalidRows = 0;
27618
+ for (const item of data) {
27619
+ if (!item || typeof item !== "object") {
27620
+ invalidRows += 1;
27621
+ continue;
27622
+ }
27623
+ const row = item;
27624
+ const date = bingDate(row.Date);
27625
+ const clicks = nonnegativeNumber(row.Clicks);
27626
+ const impressions = nonnegativeNumber(row.Impressions);
27627
+ if (!date || clicks === void 0 || impressions === void 0) {
27628
+ invalidRows += 1;
27629
+ continue;
27630
+ }
27631
+ rows.push({
27632
+ date,
27633
+ clicks,
27634
+ impressions
27635
+ });
27636
+ }
27637
+ rows.sort((a, b) => a.date.localeCompare(b.date, "en") || a.clicks - b.clicks || a.impressions - b.impressions);
27638
+ return boundedRows(rows, invalidRows);
27639
+ }
27640
+ async getCrawlStats(siteUrl) {
27641
+ const data = await this.request("GetCrawlStats", { siteUrl });
27642
+ if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid crawl statistics.");
27643
+ const fieldMap = {
27644
+ AllOtherCodes: "allOtherCodes",
27645
+ BlockedByRobotsTxt: "blockedByRobotsTxt",
27646
+ Code2xx: "code2xx",
27647
+ Code301: "code301",
27648
+ Code302: "code302",
27649
+ Code4xx: "code4xx",
27650
+ Code5xx: "code5xx",
27651
+ ConnectionTimeout: "connectionTimeout",
27652
+ ContainsMalware: "containsMalware",
27653
+ CrawledPages: "crawledPages",
27654
+ CrawlErrors: "crawlErrors",
27655
+ DnsFailures: "dnsFailures",
27656
+ InIndex: "inIndex",
27657
+ InLinks: "inLinks"
27658
+ };
27659
+ const rows = [];
27660
+ let invalidRows = 0;
27661
+ for (const item of data) {
27662
+ if (!item || typeof item !== "object") {
27663
+ invalidRows += 1;
27664
+ continue;
27665
+ }
27666
+ const source = item;
27667
+ const date = bingDate(source.Date);
27668
+ if (!date) {
27669
+ invalidRows += 1;
27670
+ continue;
27671
+ }
27672
+ const row = { date };
27673
+ let invalid = false;
27674
+ for (const [providerField, field] of Object.entries(fieldMap)) {
27675
+ const value = source[providerField];
27676
+ if (value === void 0 || value === null) continue;
27677
+ const number = nonnegativeNumber(value);
27678
+ if (number === void 0) {
27679
+ invalid = true;
27680
+ break;
27681
+ }
27682
+ row[field] = number;
27683
+ }
27684
+ if (invalid) {
27685
+ invalidRows += 1;
27686
+ continue;
27687
+ }
27688
+ rows.push(row);
27689
+ }
27690
+ rows.sort((a, b) => a.date.localeCompare(b.date, "en"));
27691
+ return boundedRows(rows, invalidRows);
27692
+ }
27693
+ async getQueryStats(siteUrl) {
27694
+ return this.getDimensionStats("GetQueryStats", siteUrl, "query");
27695
+ }
27696
+ async getPageStats(siteUrl) {
27697
+ return this.getDimensionStats("GetPageStats", siteUrl, "page");
27698
+ }
27699
+ async getDimensionStats(method, siteUrl, kind) {
27700
+ const data = await this.request(method, { siteUrl });
27701
+ if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster returned invalid ${kind} statistics.`);
27702
+ const rows = [];
27703
+ let invalidRows = 0;
27704
+ for (const item of data) {
27705
+ if (!item || typeof item !== "object") {
27706
+ invalidRows += 1;
27707
+ continue;
27708
+ }
27709
+ const source = item;
27710
+ const date = bingDate(source.Date);
27711
+ const value = dimensionValue(source.Query, kind);
27712
+ const clicks = nonnegativeNumber(source.Clicks);
27713
+ const impressions = nonnegativeNumber(source.Impressions);
27714
+ const clickPosition = positionNumber(source.AvgClickPosition);
27715
+ const impressionPosition = positionNumber(source.AvgImpressionPosition);
27716
+ if (!date || !value || clicks === void 0 || impressions === void 0 || !clickPosition.valid || !impressionPosition.valid) {
27717
+ invalidRows += 1;
27718
+ continue;
27719
+ }
27720
+ rows.push({
27721
+ date,
27722
+ value,
27723
+ clicks,
27724
+ impressions,
27725
+ avgClickPosition: clickPosition.value,
27726
+ avgImpressionPosition: impressionPosition.value
27727
+ });
27728
+ }
27729
+ rows.sort((a, b) => a.date.localeCompare(b.date, "en") || a.value.localeCompare(b.value, "en"));
27730
+ return boundedRows(rows, invalidRows, MAX_DIMENSION_ROWS);
27731
+ }
27732
+ async getLinkCounts(siteUrl, page) {
27733
+ const data = await this.request("GetLinkCounts", {
27734
+ siteUrl,
27735
+ page: String(page)
27736
+ });
27737
+ if (!data || typeof data !== "object") throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link counts.");
27738
+ const response = data;
27739
+ if (!Array.isArray(response.Links)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link counts.");
27740
+ const rows = [];
27741
+ let invalidRows = 0;
27742
+ for (const item of response.Links) {
27743
+ if (!item || typeof item !== "object") {
27744
+ invalidRows += 1;
27745
+ continue;
27746
+ }
27747
+ const row = item;
27748
+ const count = nonnegativeNumber(row.Count);
27749
+ if (!isHttpUrl(row.Url) || count === void 0) {
27750
+ invalidRows += 1;
27751
+ continue;
27752
+ }
27753
+ rows.push({
27754
+ url: row.Url,
27755
+ count
27756
+ });
27757
+ }
27758
+ rows.sort((a, b) => b.count - a.count || a.url.localeCompare(b.url, "en"));
27759
+ return linkPage(rows, response.TotalPages, invalidRows);
27760
+ }
27761
+ async getUrlLinks(siteUrl, link, page) {
27762
+ const data = await this.request("GetUrlLinks", {
27763
+ siteUrl,
27764
+ link,
27765
+ page: String(page)
27766
+ });
27767
+ if (!data || typeof data !== "object") throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid referring links.");
27768
+ const response = data;
27769
+ if (!Array.isArray(response.Details)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid referring links.");
27770
+ const rows = [];
27771
+ let invalidRows = 0;
27772
+ for (const item of response.Details) {
27773
+ if (!item || typeof item !== "object") {
27774
+ invalidRows += 1;
27775
+ continue;
27776
+ }
27777
+ const row = item;
27778
+ if (!isHttpUrl(row.Url)) {
27779
+ invalidRows += 1;
27780
+ continue;
27781
+ }
27782
+ rows.push({
27783
+ url: row.Url,
27784
+ anchorText: typeof row.AnchorText === "string" && row.AnchorText.trim() ? row.AnchorText.trim() : void 0
27785
+ });
27786
+ }
27787
+ rows.sort((a, b) => a.url.localeCompare(b.url, "en") || (a.anchorText ?? "").localeCompare(b.anchorText ?? "", "en"));
27788
+ return linkPage(rows, response.TotalPages, invalidRows);
27789
+ }
27790
+ };
27791
+ function linkPage(rows, totalPagesValue, invalidRows) {
27792
+ const totalPages = nonnegativeNumber(totalPagesValue);
27793
+ if (totalPages === void 0 || !Number.isInteger(totalPages)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link pagination.");
27794
+ const capped = rows.length > MAX_LINK_ROWS;
27795
+ return {
27796
+ rows: capped ? rows.slice(0, MAX_LINK_ROWS) : rows,
27797
+ totalPages,
27798
+ invalidRows,
27799
+ capped,
27800
+ returnedRows: rows.length
27801
+ };
27802
+ }
27803
+ //#endregion
27804
+ //#region packages/core/dist/storage/provider-secrets.js
27805
+ const KEYRING_SERVICE = "seo";
27806
+ function account(name) {
27807
+ return `provider:${name}`;
27808
+ }
27809
+ function readFile$1() {
27810
+ return readJsonFile(getSeoCliPaths().providerSecretsFile) ?? {
27811
+ version: 1,
27812
+ secrets: {}
27813
+ };
27814
+ }
27815
+ async function readProviderSecret(input) {
27816
+ const environment = input.env ?? process.env;
27817
+ const environmentValue = input.envVar ? environment[input.envVar]?.trim() : void 0;
27818
+ if (environmentValue) return {
27819
+ value: environmentValue,
27820
+ source: "environment"
27821
+ };
27822
+ if (readConfig().security.useKeychain) try {
27823
+ const value = await getKeyringPassword(KEYRING_SERVICE, account(input.name));
27824
+ if (value) return {
27825
+ value,
27826
+ source: "keychain"
27827
+ };
27828
+ } catch {}
27829
+ const value = readFile$1().secrets[input.name];
27830
+ return value ? {
27831
+ value,
27832
+ source: "file"
27833
+ } : void 0;
27834
+ }
27835
+ //#endregion
27836
+ //#region packages/core/dist/bing/credentials.js
27837
+ const BING_API_KEY_SECRET = "bing-api-key";
27838
+ const BING_API_KEY_ENV = "SEO_BING_API_KEY";
27839
+ async function createBingWebmasterClient(input = {}) {
27840
+ const secret = await readProviderSecret({
27841
+ name: BING_API_KEY_SECRET,
27842
+ envVar: BING_API_KEY_ENV,
27843
+ env: input.env
27844
+ });
27845
+ if (!secret) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster is not connected. Run `seo providers bing connect`, or set SEO_BING_API_KEY for this process.");
27846
+ return {
27847
+ client: new BingWebmasterClient({ apiKey: secret.value }),
27848
+ credentialSource: secret.source
27849
+ };
27850
+ }
27851
+ //#endregion
27852
+ //#region packages/core/dist/bing/overview.js
27853
+ const DAILY_OUTPUT_ROWS = 14;
27854
+ function message(error) {
27855
+ return error instanceof Error ? error.message : String(error);
27856
+ }
27857
+ function range(rows) {
27858
+ return rows.length ? {
27859
+ startDate: rows[0]?.date ?? "",
27860
+ endDate: rows.at(-1)?.date ?? ""
27861
+ } : void 0;
27862
+ }
27863
+ function dailyOutput(data) {
27864
+ const rows = data.rows.slice(-14);
27865
+ return {
27866
+ ...data,
27867
+ rows,
27868
+ retainedRows: data.rows.length,
27869
+ outputSelection: {
27870
+ strategy: "most-recent",
27871
+ availableRows: data.rows.length,
27872
+ returnedRows: rows.length,
27873
+ omittedRows: Math.max(0, data.rows.length - rows.length)
27874
+ }
27875
+ };
27876
+ }
27877
+ async function section(work, options) {
27878
+ try {
27879
+ const data = await work();
27880
+ const reasons = [
27881
+ ...data.invalidRows ? [`${data.invalidRows} invalid provider rows were excluded.`] : [],
27882
+ ...data.capped ? [`The provider response exceeded the ${options.rowLimit}-row limit.`] : [],
27883
+ ...options.sourceWarning ? [options.sourceWarning] : []
27884
+ ];
27885
+ return reasons.length ? {
27886
+ status: "partial",
27887
+ data,
27888
+ warning: reasons.join(" ")
27889
+ } : {
27890
+ status: "complete",
27891
+ data
27892
+ };
27893
+ } catch (error) {
27894
+ if (error instanceof SeoError && (error.code === "AUTH_REQUIRED" || error.code === "ACCESS_DENIED")) throw error;
27895
+ return {
27896
+ status: "unavailable",
27897
+ warning: message(error)
27898
+ };
27899
+ }
27900
+ }
27901
+ function changeFinding(input) {
27902
+ return {
27903
+ code: input.code,
27904
+ severity: "warning",
27905
+ title: input.title,
27906
+ evidence: {
27907
+ current: input.current,
27908
+ previous: input.previous,
27909
+ absoluteChange: input.absolute,
27910
+ percentChange: input.percent,
27911
+ currentDate: input.dates.current,
27912
+ previousDate: input.dates.previous
27913
+ },
27914
+ interpretation: input.interpretation,
27915
+ verification: input.verification
27916
+ };
27917
+ }
27918
+ function buildFindings(input) {
27919
+ const findings = [];
27920
+ const traffic = input.traffic;
27921
+ if (traffic && traffic.current.observedDays === traffic.current.expectedDays && traffic.previous.observedDays === traffic.previous.expectedDays && (traffic.changes.clicksPercent !== null && Math.abs(traffic.changes.clicksPercent) >= 10 || traffic.changes.impressionsPercent !== null && Math.abs(traffic.changes.impressionsPercent) >= 10)) {
27922
+ const metric = traffic.changes.clicksPercent !== null && Math.abs(traffic.changes.clicksPercent) >= 10 ? "clicks" : "impressions";
27923
+ const declined = traffic.changes[metric] < 0;
27924
+ findings.push({
27925
+ code: `bing_${metric}_${declined ? "declined" : "increased"}`,
27926
+ severity: declined ? "warning" : "info",
27927
+ title: `Bing ${metric} ${declined ? "declined" : "increased"} in the latest complete period`,
27928
+ evidence: {
27929
+ currentClicks: traffic.current.clicks,
27930
+ previousClicks: traffic.previous.clicks,
27931
+ clicksChange: traffic.changes.clicks,
27932
+ clicksPercentChange: traffic.changes.clicksPercent,
27933
+ currentImpressions: traffic.current.impressions,
27934
+ previousImpressions: traffic.previous.impressions,
27935
+ impressionsChange: traffic.changes.impressions,
27936
+ impressionsPercentChange: traffic.changes.impressionsPercent,
27937
+ ctrPercentagePointChange: traffic.changes.ctrPercentagePoints,
27938
+ currentRange: {
27939
+ startDate: traffic.current.startDate,
27940
+ endDate: traffic.current.endDate
27941
+ },
27942
+ previousRange: {
27943
+ startDate: traffic.previous.startDate,
27944
+ endDate: traffic.previous.endDate
27945
+ }
27946
+ },
27947
+ interpretation: "This is a Bing-observed traffic change. It does not establish the cause or describe Google traffic.",
27948
+ verification: "Review the query and page evidence for the same dates, then compare current pages and crawl health before changing content."
27949
+ });
27950
+ }
27951
+ const crawl = input.crawl;
27952
+ if (crawl?.previous) {
27953
+ const dates = {
27954
+ current: crawl.current.date,
27955
+ previous: crawl.previous.date
27956
+ };
27957
+ const errors = crawl.changes.crawlErrors;
27958
+ if (errors && errors.absolute >= Math.max(10, Math.ceil(errors.previous * .5))) findings.push(changeFinding({
27959
+ code: "bing_crawl_errors_increased",
27960
+ title: "Bing crawl errors increased",
27961
+ ...errors,
27962
+ dates,
27963
+ interpretation: "Bing reported more crawl errors in its latest daily snapshot. The count may include URLs outside the current sitemap.",
27964
+ verification: "Run the fast sitemap health check, inspect Bing crawl issue samples, then audit any affected URLs directly."
27965
+ }));
27966
+ const code4xx = crawl.changes.code4xx;
27967
+ if (code4xx && code4xx.absolute >= Math.max(10, Math.ceil(code4xx.previous * .5))) findings.push(changeFinding({
27968
+ code: "bing_4xx_increased",
27969
+ title: "Bing-observed 4xx responses increased",
27970
+ ...code4xx,
27971
+ dates,
27972
+ interpretation: "Bing encountered more 4xx responses. Intentional removals and obsolete discovered URLs can contribute to this count.",
27973
+ verification: "Inspect Bing crawl issue URLs and recover or redirect only URLs that should still exist or retain search value."
27974
+ }));
27975
+ const code5xx = crawl.changes.code5xx;
27976
+ if (code5xx && code5xx.current > 0 && code5xx.absolute > 0) findings.push(changeFinding({
27977
+ code: "bing_5xx_increased",
27978
+ title: "Bing-observed server errors increased",
27979
+ ...code5xx,
27980
+ dates,
27981
+ interpretation: "Bing encountered more server errors in the latest snapshot. Even a small increase deserves a live check.",
27982
+ verification: "Check current server logs and run targeted URL audits before treating the provider count as an active outage."
27983
+ }));
27984
+ }
27985
+ for (const [kind, analysis] of [["query", input.queries], ["page", input.pages]]) {
27986
+ if (!analysis?.opportunities.length) continue;
27987
+ findings.push({
27988
+ code: `bing_${kind}_opportunities`,
27989
+ severity: "review",
27990
+ title: `${analysis.opportunities.length} Bing ${kind} opportunities need review`,
27991
+ evidence: {
27992
+ retainedOpportunities: analysis.opportunities.length,
27993
+ currentDimensions: analysis.coverage.currentDimensions,
27994
+ matchedDimensions: analysis.coverage.matchedDimensions,
27995
+ comparableDimensions: analysis.coverage.comparableDimensions,
27996
+ thresholds: analysis.thresholds
27997
+ },
27998
+ interpretation: "These are high-impression top-list entries currently observed around positions 4 to 20. The threshold is a prioritization heuristic, not a search-engine rule.",
27999
+ verification: kind === "query" ? "Check intent and the ranking page before changing copy or creating content." : "Audit the selected page and its matching queries before changing internal links, titles, or content."
28000
+ });
28001
+ }
28002
+ return findings;
28003
+ }
28004
+ async function bingWebmasterOverview(input) {
28005
+ if (!input.site.trim()) throw new SeoError("INVALID_INPUT", "Pass a Bing Webmaster site URL.");
28006
+ const resolved = input.client ? {
28007
+ client: input.client,
28008
+ credentialSource: input.credentialSource ?? "environment"
28009
+ } : await createBingWebmasterClient();
28010
+ const observedAt = (/* @__PURE__ */ new Date()).toISOString();
28011
+ const topListWarning = "Bing returns weekly top-list rows, so dimensions missing from a period remain unknown.";
28012
+ const [traffic, crawl, queries, pages] = await Promise.all([
28013
+ section(() => resolved.client.getTraffic(input.site), { rowLimit: 400 }),
28014
+ section(() => resolved.client.getCrawlStats(input.site), { rowLimit: 400 }),
28015
+ section(() => resolved.client.getQueryStats(input.site), {
28016
+ rowLimit: 8e3,
28017
+ sourceWarning: topListWarning
28018
+ }),
28019
+ section(() => resolved.client.getPageStats(input.site), {
28020
+ rowLimit: 8e3,
28021
+ sourceWarning: topListWarning
28022
+ })
28023
+ ]);
28024
+ const statuses = [
28025
+ traffic.status,
28026
+ crawl.status,
28027
+ queries.status,
28028
+ pages.status
28029
+ ];
28030
+ const dataStatus = statuses.every((status) => status === "complete") ? "complete" : statuses.every((status) => status === "unavailable") ? "unavailable" : "partial";
28031
+ const trafficAnalysis = traffic.status === "unavailable" ? void 0 : analyzeBingTraffic(traffic.data.rows);
28032
+ const crawlAnalysis = crawl.status === "unavailable" ? void 0 : analyzeBingCrawl(crawl.data.rows);
28033
+ const queryAnalysis = queries.status === "unavailable" ? void 0 : analyzeBingDimensions(queries.data.rows, "query");
28034
+ const pageAnalysis = pages.status === "unavailable" ? void 0 : analyzeBingDimensions(pages.data.rows, "page");
28035
+ const findings = buildFindings({
28036
+ traffic: trafficAnalysis,
28037
+ crawl: crawlAnalysis,
28038
+ queries: queryAnalysis,
28039
+ pages: pageAnalysis
28040
+ });
28041
+ return {
28042
+ schemaVersion: 2,
28043
+ site: input.site,
28044
+ generatedAt: observedAt,
28045
+ dataStatus,
28046
+ summary: {
28047
+ findings: findings.length,
28048
+ warnings: findings.filter((finding) => finding.severity === "warning").length,
28049
+ reviewItems: findings.filter((finding) => finding.severity === "review").length
28050
+ },
28051
+ findings,
28052
+ outputBudget: {
28053
+ maxFindings: 6,
28054
+ maxDailyRowsPerSection: DAILY_OUTPUT_ROWS,
28055
+ maxDailyRowsTotal: DAILY_OUTPUT_ROWS * 2,
28056
+ maxDimensionItemsPerList: 10,
28057
+ maxDimensionItemsTotal: 40
28058
+ },
28059
+ provenance: {
28060
+ provider: "bing-webmaster",
28061
+ authentication: resolved.client.authentication,
28062
+ credentialSource: resolved.credentialSource,
28063
+ observedAt,
28064
+ cached: false,
28065
+ rowLimits: {
28066
+ traffic: 400,
28067
+ crawl: 400,
28068
+ queries: 8e3,
28069
+ pages: 8e3
28070
+ },
28071
+ methods: [
28072
+ "GetRankAndTrafficStats",
28073
+ "GetCrawlStats",
28074
+ "GetQueryStats",
28075
+ "GetPageStats"
28076
+ ]
28077
+ },
28078
+ traffic: traffic.status === "unavailable" ? traffic : {
28079
+ ...traffic,
28080
+ data: {
28081
+ ...dailyOutput(traffic.data),
28082
+ range: range(traffic.data.rows),
28083
+ clicks: traffic.data.rows.reduce((total, row) => total + row.clicks, 0),
28084
+ impressions: traffic.data.rows.reduce((total, row) => total + row.impressions, 0),
28085
+ analysis: trafficAnalysis
28086
+ }
28087
+ },
28088
+ crawl: crawl.status === "unavailable" ? crawl : {
28089
+ ...crawl,
28090
+ data: {
28091
+ ...dailyOutput(crawl.data),
28092
+ range: range(crawl.data.rows),
28093
+ latest: crawl.data.rows.at(-1),
28094
+ analysis: crawlAnalysis
28095
+ }
28096
+ },
28097
+ queries: queries.status === "unavailable" ? queries : {
28098
+ ...queries,
28099
+ data: {
28100
+ returnedRows: queries.data.returnedRows,
28101
+ retainedRows: queries.data.rows.length,
28102
+ invalidRows: queries.data.invalidRows,
28103
+ capped: queries.data.capped,
28104
+ range: range(queries.data.rows),
28105
+ analysis: queryAnalysis
28106
+ }
28107
+ },
28108
+ pages: pages.status === "unavailable" ? pages : {
28109
+ ...pages,
28110
+ data: {
28111
+ returnedRows: pages.data.returnedRows,
28112
+ retainedRows: pages.data.rows.length,
28113
+ invalidRows: pages.data.invalidRows,
28114
+ capped: pages.data.capped,
28115
+ range: range(pages.data.rows),
28116
+ analysis: pageAnalysis
28117
+ }
28118
+ },
28119
+ caveats: [
28120
+ "Bing reports its own observed search and crawl evidence. It is not a complete view of every search engine.",
28121
+ "The inIndex field is provider-reported crawl statistics, not independent proof that a URL is indexed.",
28122
+ "Query and page movements include only dimensions observed in every weekly top list in both periods. Missing entries are not treated as zero.",
28123
+ "Opportunity thresholds prioritize review. They are heuristics, not search-engine requirements."
28124
+ ]
28125
+ };
28126
+ }
28127
+ //#endregion
28128
+ //#region packages/core/dist/doctor.js
28129
+ const REQUIRED_SCOPES = ["https://www.googleapis.com/auth/webmasters.readonly", "https://www.googleapis.com/auth/analytics.readonly"];
28130
+ async function runDoctor(dependencies = {}) {
28131
+ const paths = getSeoCliPaths();
28132
+ const config = readConfig();
28133
+ const authMode = getAuthModeStatus();
28134
+ const serviceAccount = getServiceAccountStatus();
28135
+ const tokens = await readTokens();
28136
+ const oauthClient = readOauthClient();
28137
+ const tokenClientConfigured = tokens ? Boolean(getClientConfig(tokens.client_source)) : void 0;
28138
+ const checks = [];
28139
+ try {
28140
+ const database = (dependencies.checkDatabase ?? checkDatabaseReadiness)();
28141
+ checks.push({
28142
+ id: "local-database",
28143
+ label: "Local database",
28144
+ status: "pass",
28145
+ detail: database.dbPath
28146
+ });
28147
+ } catch (error) {
28148
+ checks.push({
28149
+ id: "local-database",
28150
+ label: "Local database",
28151
+ status: "fail",
28152
+ detail: error instanceof Error ? error.message : "The local database could not be opened.",
28153
+ fix: "Upgrade or reinstall `seo`. If this still fails, report it at https://github.com/iannuttall/seo/issues."
28154
+ });
28155
+ }
28156
+ checks.push({
28157
+ id: "config-dir",
28158
+ label: "Config directory",
28159
+ status: existsSync(paths.configDir) ? "pass" : "warn",
28160
+ detail: paths.configDir,
28161
+ fix: existsSync(paths.configDir) ? void 0 : "Run `seo start` or `seo auth setup-client`."
28162
+ });
28163
+ checks.push({
28164
+ id: "oauth-client",
28165
+ label: "Google credentials",
28166
+ status: serviceAccount.configured ? "pass" : tokenClientConfigured === false ? "fail" : serviceAccount.error ? "fail" : authMode.sharedConfigured || authMode.byoConfigured ? "pass" : "fail",
28167
+ detail: serviceAccount.configured ? `Service account configured for ${serviceAccount.identity}.` : tokenClientConfigured === false ? `Stored Google login uses the ${tokens?.client_source === "shared" ? "shared seo app" : "BYO client"}, but that client is not configured.` : serviceAccount.error ? serviceAccount.error : authMode.sharedConfigured ? "Shared client configured." : authMode.byoConfigured ? `BYO client configured at ${paths.oauthClientFile}.` : "No shared or BYO Google OAuth client configured.",
28168
+ fix: serviceAccount.configured ? void 0 : tokenClientConfigured === false ? tokens?.client_source === "shared" ? "Reinstall `seo`. If the shared client is still missing, report it at https://github.com/iannuttall/seo/issues." : "Restore the same client with `seo auth setup-client`, or run `seo auth logout` before signing in with a different client." : serviceAccount.error ? "Set one valid service account credential source, then run `seo doctor` again." : authMode.sharedConfigured || authMode.byoConfigured ? void 0 : "Run `seo auth setup-client` or set SEO_GOOGLE_CLIENT_ID and SEO_GOOGLE_CLIENT_SECRET."
28169
+ });
28170
+ checks.push({
28171
+ id: "oauth-client-file",
28172
+ label: "BYO OAuth file",
28173
+ status: oauthClient ? "pass" : "warn",
28174
+ detail: oauthClient ? paths.oauthClientFile : "No BYO OAuth client file. This is fine if using a shared client."
28175
+ });
28176
+ checks.push({
28177
+ id: "google-login",
28178
+ label: "Google login",
28179
+ status: serviceAccount.configured || tokens ? "pass" : "fail",
28180
+ detail: serviceAccount.configured ? `Using service account ${serviceAccount.identity}.` : tokens ? `Signed in as ${tokens.account_email}.` : "No Google token found.",
28181
+ fix: serviceAccount.configured || tokens ? void 0 : "Run `seo auth login`, or configure a service account for CI."
28182
+ });
28183
+ const grantedScopes = new Set(tokens?.scope.split(/\s+/).filter(Boolean) ?? []);
28184
+ for (const scope of REQUIRED_SCOPES) checks.push({
28185
+ id: `scope:${scope}`,
28186
+ label: scope.endsWith("webmasters.readonly") ? "Search Console scope" : "Google Analytics scope",
28187
+ status: serviceAccount.configured || tokens && grantedScopes.has(scope) ? "pass" : tokens ? "fail" : "warn",
28188
+ detail: serviceAccount.configured ? `${scope} requested by the service account.` : tokens ? grantedScopes.has(scope) ? scope : `Missing ${scope}.` : "Cannot inspect scopes until login is complete.",
28189
+ fix: !serviceAccount.configured && tokens && !grantedScopes.has(scope) ? "Run `seo auth logout` then `seo auth login`." : void 0
28190
+ });
28191
+ checks.push({
28192
+ id: "default-site",
28193
+ label: "Saved GSC property",
28194
+ status: "pass",
28195
+ detail: config.defaultSite ?? "No saved default. Human CLI commands will prompt; agents can pass --site."
28196
+ });
28197
+ checks.push({
28198
+ id: "default-google-analytics",
28199
+ label: "Saved Google Analytics property",
28200
+ status: "pass",
28201
+ detail: config.analytics.google.defaultPropertyId ?? "No saved default. Human CLI commands will prompt; agents can pass --property."
28202
+ });
28203
+ return {
28204
+ ok: checks.every((check) => check.status !== "fail"),
28205
+ generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
28206
+ checks
28207
+ };
28208
+ }
28209
+ //#endregion
28210
+ //#region packages/core/dist/links/normalize.js
28211
+ const FIELD_ALIASES = {
28212
+ sourceUrl: [
28213
+ "sourceurl",
28214
+ "source",
28215
+ "referringurl",
28216
+ "referringpage",
28217
+ "fromurl"
28218
+ ],
28219
+ targetUrl: [
28220
+ "targeturl",
28221
+ "target",
28222
+ "destinationurl",
28223
+ "destination",
28224
+ "tourl"
28225
+ ],
28226
+ anchorText: ["anchortext", "anchor"],
28227
+ firstSeenAt: ["firstseenat", "firstseen"],
28228
+ lastSeenAt: ["lastseenat", "lastseen"],
28229
+ nofollow: ["nofollow", "isnofollow"]
28230
+ };
28231
+ function key(value) {
28232
+ return value.toLowerCase().replaceAll(/[^a-z0-9]/g, "");
28233
+ }
28234
+ function valueFor(row, field) {
28235
+ const aliases = new Set(FIELD_ALIASES[field]);
28236
+ for (const [name, value] of Object.entries(row)) if (aliases.has(key(name))) return value;
28237
+ }
28238
+ function httpUrl(value) {
28239
+ if (typeof value !== "string" || !value.trim() || value.trim().length > 2e3) return;
28240
+ try {
28241
+ const url = new URL(value.trim());
28242
+ return url.protocol === "http:" || url.protocol === "https:" ? url : void 0;
28243
+ } catch {
28244
+ return;
28245
+ }
28246
+ }
28247
+ function optionalString(value) {
28248
+ return typeof value === "string" && value.trim() ? value.trim().slice(0, 1e3) : void 0;
28249
+ }
28250
+ function optionalBoolean(value) {
28251
+ if (typeof value === "boolean") return value;
28252
+ if (typeof value !== "string") return void 0;
28253
+ if (/^(true|yes|1)$/i.test(value.trim())) return true;
28254
+ if (/^(false|no|0)$/i.test(value.trim())) return false;
28255
+ }
28256
+ function normalizeLinkEvidenceRow(raw) {
28257
+ const source = httpUrl(valueFor(raw, "sourceUrl"));
28258
+ const target = httpUrl(valueFor(raw, "targetUrl"));
28259
+ if (!source || !target) return void 0;
28260
+ return {
28261
+ sourceUrl: source.toString(),
28262
+ sourceDomain: source.hostname.toLowerCase(),
28263
+ targetUrl: target.toString(),
28264
+ anchorText: optionalString(valueFor(raw, "anchorText")),
28265
+ firstSeenAt: optionalString(valueFor(raw, "firstSeenAt")),
28266
+ lastSeenAt: optionalString(valueFor(raw, "lastSeenAt")),
28267
+ nofollow: optionalBoolean(valueFor(raw, "nofollow"))
28268
+ };
28269
+ }
28270
+ function linkEvidenceKey(row) {
28271
+ return `${row.sourceUrl}\u0000${row.targetUrl}\u0000${row.anchorText ?? ""}`;
28272
+ }
28273
+ function compareLinkEvidence(a, b) {
28274
+ return a.targetUrl.localeCompare(b.targetUrl, "en") || a.sourceUrl.localeCompare(b.sourceUrl, "en") || (a.anchorText ?? "").localeCompare(b.anchorText ?? "", "en");
28275
+ }
28276
+ //#endregion
28277
+ //#region packages/core/dist/links/bing.js
28278
+ const MAX_COUNT_PAGES = 5;
28279
+ const DEFAULT_TARGET_LIMIT = 20;
28280
+ const MAX_TARGET_LIMIT = 50;
28281
+ const DEFAULT_DETAIL_PAGES = 1;
28282
+ const MAX_DETAIL_PAGES = 3;
28283
+ const DEFAULT_ROW_LIMIT$2 = 500;
28284
+ const MAX_ROW_LIMIT$2 = 1e3;
28285
+ const MAX_CONCURRENCY = 3;
28286
+ function integer(value, fallback, max, name) {
28287
+ const resolved = value ?? fallback;
28288
+ if (!Number.isInteger(resolved) || resolved < 1 || resolved > max) throw new SeoError("INVALID_INPUT", `${name} must be between 1 and ${max}.`);
28289
+ return resolved;
28290
+ }
28291
+ async function mapConcurrent(values, concurrency, work) {
28292
+ const output = new Array(values.length);
28293
+ let next = 0;
28294
+ async function worker() {
28295
+ while (next < values.length) {
28296
+ const index = next;
28297
+ next += 1;
28298
+ const value = values[index];
28299
+ if (value !== void 0) output[index] = await work(value);
28300
+ }
28301
+ }
28302
+ await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, () => worker()));
28303
+ return output;
28304
+ }
28305
+ async function targetDetails(input) {
28306
+ const rows = [];
28307
+ let requestedPages = 0;
28308
+ let invalidRows = 0;
28309
+ let returnedRows = 0;
28310
+ let partial = false;
28311
+ let totalPages = 0;
28312
+ for (let page = 0; page < input.pages; page += 1) {
28313
+ requestedPages += 1;
28314
+ let response;
28315
+ try {
28316
+ response = await input.client.getUrlLinks(input.site, input.target.url, page);
28317
+ } catch (error) {
28318
+ if (error instanceof SeoError && (error.code === "AUTH_REQUIRED" || error.code === "ACCESS_DENIED")) throw error;
28319
+ return {
28320
+ rows,
28321
+ requestedPages,
28322
+ invalidRows,
28323
+ returnedRows,
28324
+ partial: true,
28325
+ failed: true
28326
+ };
28327
+ }
28328
+ invalidRows += response.invalidRows;
28329
+ returnedRows += response.returnedRows;
28330
+ rows.push(...response.rows);
28331
+ totalPages = response.totalPages;
28332
+ partial ||= response.capped;
28333
+ if (page + 1 >= response.totalPages || response.rows.length === 0) break;
28334
+ }
28335
+ partial ||= requestedPages < totalPages;
28336
+ return {
28337
+ rows,
28338
+ requestedPages,
28339
+ invalidRows,
28340
+ returnedRows,
28341
+ partial,
28342
+ failed: false
28343
+ };
28344
+ }
28345
+ async function collectBingLinkEvidence(input) {
28346
+ if (!input.site.trim()) throw new SeoError("INVALID_INPUT", "Pass a Bing Webmaster site URL.");
28347
+ const targetLimit = integer(input.targetLimit, DEFAULT_TARGET_LIMIT, MAX_TARGET_LIMIT, "Bing link target limit");
28348
+ const detailPages = integer(input.detailPagesPerTarget, DEFAULT_DETAIL_PAGES, MAX_DETAIL_PAGES, "Bing detail page limit");
28349
+ const rowLimit = integer(input.rowLimit, DEFAULT_ROW_LIMIT$2, MAX_ROW_LIMIT$2, "Bing link row limit");
28350
+ const client = input.client ?? (await createBingWebmasterClient()).client;
28351
+ const countRows = [];
28352
+ let countPagesRequested = 0;
28353
+ let invalidRows = 0;
28354
+ let countInvalidRows = 0;
28355
+ let countRowsReturned = 0;
28356
+ let partial = false;
28357
+ let totalCountPages = 0;
28358
+ for (let page = 0; page < MAX_COUNT_PAGES; page += 1) {
28359
+ const response = await client.getLinkCounts(input.site, page);
28360
+ countPagesRequested += 1;
28361
+ invalidRows += response.invalidRows;
28362
+ countInvalidRows += response.invalidRows;
28363
+ countRowsReturned += response.returnedRows;
28364
+ countRows.push(...response.rows);
28365
+ totalCountPages = response.totalPages;
28366
+ partial ||= response.capped;
28367
+ if (countRows.length >= targetLimit || page + 1 >= response.totalPages || response.rows.length === 0) break;
28368
+ }
28369
+ partial ||= countPagesRequested < totalCountPages;
28370
+ const countsByUrl = /* @__PURE__ */ new Map();
28371
+ for (const row of countRows) countsByUrl.set(row.url, Math.max(row.count, countsByUrl.get(row.url) ?? 0));
28372
+ const counts = [...countsByUrl].map(([url, count]) => ({
28373
+ url,
28374
+ count
28375
+ }));
28376
+ counts.sort((a, b) => b.count - a.count || a.url.localeCompare(b.url, "en"));
28377
+ const selectedTargets = counts.slice(0, targetLimit);
28378
+ partial ||= counts.length > targetLimit;
28379
+ const details = await mapConcurrent(selectedTargets, MAX_CONCURRENCY, async (target) => targetDetails({
28380
+ client,
28381
+ site: input.site,
28382
+ target,
28383
+ pages: detailPages
28384
+ }));
28385
+ const rows = [];
28386
+ const keys = /* @__PURE__ */ new Set();
28387
+ let duplicateRows = 0;
28388
+ let detailPagesRequested = 0;
28389
+ let detailInvalidRows = 0;
28390
+ let detailRowsReturned = 0;
28391
+ let failedTargets = 0;
28392
+ for (const [index, detail] of details.entries()) {
28393
+ const target = selectedTargets[index];
28394
+ if (!target) continue;
28395
+ detailPagesRequested += detail.requestedPages;
28396
+ if (detail.failed) failedTargets += 1;
28397
+ invalidRows += detail.invalidRows;
28398
+ detailInvalidRows += detail.invalidRows;
28399
+ detailRowsReturned += detail.returnedRows;
28400
+ partial ||= detail.partial;
28401
+ for (const source of detail.rows) {
28402
+ if (rows.length >= rowLimit) {
28403
+ partial = true;
28404
+ break;
28405
+ }
28406
+ const row = normalizeLinkEvidenceRow({
28407
+ sourceUrl: source.url,
28408
+ targetUrl: target.url,
28409
+ anchorText: source.anchorText
28410
+ });
28411
+ if (!row) {
28412
+ invalidRows += 1;
28413
+ continue;
28414
+ }
28415
+ const key = linkEvidenceKey(row);
28416
+ if (keys.has(key)) {
28417
+ duplicateRows += 1;
28418
+ continue;
28419
+ }
28420
+ keys.add(key);
28421
+ rows.push(row);
28422
+ }
28423
+ }
28424
+ rows.sort(compareLinkEvidence);
28425
+ return {
28426
+ rows,
28427
+ targetCounts: selectedTargets.map((target) => ({
28428
+ targetUrl: target.url,
28429
+ providerReportedLinks: target.count,
28430
+ observedLinks: rows.filter((row) => row.targetUrl === target.url).length
28431
+ })),
28432
+ provenance: {
28433
+ provider: "bing-webmaster",
28434
+ observedAt: (/* @__PURE__ */ new Date()).toISOString(),
28435
+ cached: false,
28436
+ suppliedRows: detailRowsReturned + detailInvalidRows,
28437
+ validRows: rows.length,
28438
+ invalidRows,
28439
+ duplicateRows,
28440
+ capped: partial,
28441
+ rowLimit,
28442
+ completeness: partial ? "partial" : "unknown",
28443
+ providerRequests: {
28444
+ methods: ["GetLinkCounts", "GetUrlLinks"],
28445
+ targetPagesRequested: countPagesRequested,
28446
+ detailPagesRequested,
28447
+ maxConcurrentRequests: MAX_CONCURRENCY
28448
+ },
28449
+ providerCoverage: {
28450
+ targetCountRows: {
28451
+ returnedRows: countRowsReturned,
28452
+ retainedRows: counts.length,
28453
+ invalidRows: countInvalidRows
28454
+ },
28455
+ detailRows: {
28456
+ returnedRows: detailRowsReturned,
28457
+ retainedRows: rows.length,
28458
+ invalidRows: detailInvalidRows
28459
+ }
28460
+ }
28461
+ },
28462
+ warnings: [
28463
+ ...countRowsReturned === 0 ? ["Bing returned no target-link rows. This does not prove that the site has no links."] : [],
28464
+ ...partial ? ["Bing link evidence was bounded by provider pagination, failed requests, or report limits."] : [],
28465
+ ...failedTargets ? [`${failedTargets} target detail request${failedTargets === 1 ? "" : "s"} failed while other evidence was retained.`] : []
28466
+ ]
28467
+ };
28468
+ }
28469
+ //#endregion
28470
+ //#region packages/core/dist/links/import.js
28471
+ const DEFAULT_ROW_LIMIT$1 = 1e4;
28472
+ const MAX_ROW_LIMIT$1 = 1e5;
28473
+ const MAX_STREAM_BYTES = 5e7;
28474
+ const MAX_JSON_BYTES = 1e7;
28475
+ const MAX_CELL_BYTES = 1e6;
28476
+ function formatFor$1(path, explicit) {
28477
+ if (explicit) return explicit;
28478
+ const extension = extname(path).toLowerCase();
28479
+ if (extension === ".csv") return "csv";
28480
+ if (extension === ".jsonl" || extension === ".ndjson") return "jsonl";
28481
+ if (extension === ".json") return "json";
28482
+ throw new SeoError("INVALID_INPUT", "Link imports must be CSV, JSON, JSONL, or NDJSON.");
28483
+ }
28484
+ function boundedRowLimit(value) {
28485
+ const limit = value ?? DEFAULT_ROW_LIMIT$1;
28486
+ if (!Number.isInteger(limit) || limit < 1 || limit > MAX_ROW_LIMIT$1) throw new SeoError("INVALID_INPUT", `Link import row limit must be between 1 and ${MAX_ROW_LIMIT$1}.`);
28487
+ return limit;
28488
+ }
28489
+ async function* csvRecords(path, byteLimit, progress) {
28490
+ const stream = createReadStream(path, { encoding: "utf8" });
28491
+ let record = [];
28492
+ let field = "";
28493
+ let quoted = false;
28494
+ let pendingQuote = false;
28495
+ for await (const chunk of stream) {
28496
+ progress.bytesRead += Buffer.byteLength(chunk);
28497
+ if (progress.bytesRead > byteLimit) {
28498
+ stream.destroy();
28499
+ throw new SeoError("INVALID_INPUT", `Link import exceeds the ${byteLimit}-byte streaming limit.`);
28500
+ }
28501
+ for (const character of chunk) {
28502
+ if (pendingQuote) {
28503
+ if (character === "\"") {
28504
+ field += "\"";
28505
+ pendingQuote = false;
28506
+ continue;
28507
+ }
28508
+ quoted = false;
28509
+ pendingQuote = false;
28510
+ }
28511
+ if (quoted) if (character === "\"") pendingQuote = true;
28512
+ else field += character;
28513
+ else if (character === "\"" && field.length === 0) quoted = true;
28514
+ else if (character === ",") {
28515
+ record.push(field);
28516
+ field = "";
28517
+ } else if (character === "\n") {
28518
+ record.push(field.replace(/\r$/, ""));
28519
+ field = "";
28520
+ if (record.some((value) => value.length > 0)) yield record;
28521
+ record = [];
28522
+ } else field += character;
28523
+ if (Buffer.byteLength(field) > MAX_CELL_BYTES) {
28524
+ stream.destroy();
28525
+ throw new SeoError("INVALID_INPUT", `Link import contains a cell larger than ${MAX_CELL_BYTES} bytes.`);
28526
+ }
28527
+ }
28528
+ }
28529
+ if (quoted && !pendingQuote) throw new SeoError("INVALID_INPUT", "Link CSV contains an unclosed quote.");
28530
+ if (field || record.length) {
28531
+ record.push(field.replace(/\r$/, ""));
28532
+ if (record.some((value) => value.length > 0)) yield record;
28533
+ }
28534
+ }
28535
+ async function* jsonlRecords(path, byteLimit, progress) {
28536
+ const stream = createReadStream(path, { encoding: "utf8" });
28537
+ let remainder = "";
28538
+ for await (const chunk of stream) {
28539
+ progress.bytesRead += Buffer.byteLength(chunk);
28540
+ if (progress.bytesRead > byteLimit) {
28541
+ stream.destroy();
28542
+ throw new SeoError("INVALID_INPUT", `Link import exceeds the ${byteLimit}-byte streaming limit.`);
28543
+ }
28544
+ remainder += chunk;
28545
+ let newline = remainder.indexOf("\n");
28546
+ while (newline >= 0) {
28547
+ const line = remainder.slice(0, newline).trim();
28548
+ remainder = remainder.slice(newline + 1);
28549
+ if (line) yield parseObject(line);
28550
+ newline = remainder.indexOf("\n");
28551
+ }
28552
+ if (Buffer.byteLength(remainder) > MAX_CELL_BYTES) {
28553
+ stream.destroy();
28554
+ throw new SeoError("INVALID_INPUT", `Link import contains a line larger than ${MAX_CELL_BYTES} bytes.`);
28555
+ }
28556
+ }
28557
+ if (remainder.trim()) yield parseObject(remainder);
28558
+ }
28559
+ function parseObject(value) {
28560
+ try {
28561
+ const parsed = JSON.parse(value);
28562
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("not an object");
28563
+ return parsed;
28564
+ } catch {
28565
+ throw new SeoError("INVALID_INPUT", "Link JSONL must contain one JSON object per line.");
28566
+ }
28567
+ }
28568
+ function rowFromCsv(headers, values) {
28569
+ return Object.fromEntries(headers.map((header, index) => [header, values[index]]));
28570
+ }
28571
+ async function importLinkEvidence(input) {
28572
+ const path = resolve(input.file);
28573
+ const file = await stat(path).catch(() => void 0);
28574
+ if (!file?.isFile()) throw new SeoError("INVALID_INPUT", `Link import file was not found: ${path}`);
28575
+ const format = formatFor$1(path, input.format);
28576
+ const rowLimit = boundedRowLimit(input.rowLimit);
28577
+ if (format === "json" && file.size > MAX_JSON_BYTES) throw new SeoError("INVALID_INPUT", `JSON link imports are limited to ${MAX_JSON_BYTES} bytes. Use CSV or JSONL for larger files.`);
28578
+ const progress = { bytesRead: 0 };
28579
+ let records;
28580
+ if (format === "csv") {
28581
+ const csv = csvRecords(path, MAX_STREAM_BYTES, progress);
28582
+ const first = await csv.next();
28583
+ const headers = (Array.isArray(first.value) ? first.value : []).map((value, index) => index === 0 ? value.replace(/^\uFEFF/, "") : value);
28584
+ if (!headers.length) throw new SeoError("INVALID_INPUT", "Link CSV has no header row.");
28585
+ records = (async function* () {
28586
+ for await (const values of csv) yield rowFromCsv(headers, values);
28587
+ })();
28588
+ } else if (format === "jsonl") records = jsonlRecords(path, MAX_STREAM_BYTES, progress);
28589
+ else {
28590
+ const body = await readFile(path, "utf8");
28591
+ progress.bytesRead = Buffer.byteLength(body);
28592
+ let parsed;
28593
+ try {
28594
+ parsed = JSON.parse(body);
28595
+ } catch {
28596
+ throw new SeoError("INVALID_INPUT", "Link JSON is not valid JSON.");
28597
+ }
28598
+ if (!Array.isArray(parsed)) throw new SeoError("INVALID_INPUT", "Link JSON must be an array of objects.");
28599
+ records = (async function* () {
28600
+ for (const row of parsed) yield row && typeof row === "object" && !Array.isArray(row) ? row : {};
28601
+ })();
28602
+ }
28603
+ const rows = [];
28604
+ const keys = /* @__PURE__ */ new Set();
28605
+ let suppliedRows = 0;
28606
+ let invalidRows = 0;
28607
+ let duplicateRows = 0;
28608
+ let capped = false;
28609
+ for await (const raw of records) {
28610
+ if (suppliedRows >= rowLimit) {
28611
+ capped = true;
28612
+ break;
28613
+ }
28614
+ suppliedRows += 1;
28615
+ const row = normalizeLinkEvidenceRow(raw);
28616
+ if (!row) {
28617
+ invalidRows += 1;
28618
+ continue;
28619
+ }
28620
+ const key = linkEvidenceKey(row);
28621
+ if (keys.has(key)) {
28622
+ duplicateRows += 1;
28623
+ continue;
28624
+ }
28625
+ keys.add(key);
28626
+ rows.push(row);
28627
+ }
28628
+ rows.sort(compareLinkEvidence);
28629
+ const observedAt = (/* @__PURE__ */ new Date()).toISOString();
28630
+ return {
28631
+ rows,
28632
+ targetCounts: [],
28633
+ provenance: {
28634
+ provider: `${format}-import`,
28635
+ observedAt,
28636
+ cached: false,
28637
+ suppliedRows,
28638
+ validRows: rows.length,
28639
+ invalidRows,
28640
+ duplicateRows,
28641
+ capped,
28642
+ rowLimit,
28643
+ completeness: capped ? "partial" : "unknown",
28644
+ file: {
28645
+ path,
28646
+ format,
28647
+ bytesRead: progress.bytesRead,
28648
+ fileBytes: file.size
28649
+ }
28650
+ },
28651
+ warnings: capped ? [`Import stopped after ${rowLimit} rows. The file contains more data.`] : []
28652
+ };
28653
+ }
28654
+ //#endregion
28655
+ //#region packages/core/dist/links/report.js
28656
+ const DEFAULT_OUTPUT_LIMIT = 100;
28657
+ const MAX_OUTPUT_LIMIT = 500;
28658
+ function boundedLimit(value) {
28659
+ const limit = value ?? DEFAULT_OUTPUT_LIMIT;
28660
+ if (!Number.isInteger(limit) || limit < 1 || limit > MAX_OUTPUT_LIMIT) throw new SeoError("INVALID_INPUT", `Link report limit must be between 1 and ${MAX_OUTPUT_LIMIT}.`);
28661
+ return limit;
28662
+ }
28663
+ function targetCounts(rows) {
28664
+ const counts = /* @__PURE__ */ new Map();
28665
+ for (const row of rows) counts.set(row.targetUrl, (counts.get(row.targetUrl) ?? 0) + 1);
28666
+ return [...counts].map(([targetUrl, observedLinks]) => ({
28667
+ targetUrl,
28668
+ observedLinks
28669
+ })).sort((a, b) => b.observedLinks - a.observedLinks || a.targetUrl.localeCompare(b.targetUrl, "en"));
28670
+ }
28671
+ function linkEvidenceReport(input) {
28672
+ const limit = boundedLimit(input.limit);
28673
+ const rows = input.evidence.rows.slice(0, limit);
28674
+ const domains = new Set(input.evidence.rows.map((row) => row.sourceDomain));
28675
+ const observedTargets = targetCounts(input.evidence.rows);
28676
+ const counts = input.evidence.targetCounts.length ? input.evidence.targetCounts : observedTargets;
28677
+ return {
28678
+ schemaVersion: 1,
28679
+ generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
28680
+ dataStatus: input.evidence.provenance.completeness === "complete" ? "complete" : "partial",
28681
+ summary: {
28682
+ observedLinks: input.evidence.rows.length,
28683
+ referringDomains: domains.size,
28684
+ targetPages: new Set(input.evidence.rows.map((row) => row.targetUrl)).size,
28685
+ providerTargetPages: counts.length
28686
+ },
28687
+ provenance: input.evidence.provenance,
28688
+ selection: {
28689
+ availableRows: input.evidence.rows.length,
28690
+ returnedRows: rows.length,
28691
+ omittedRows: Math.max(0, input.evidence.rows.length - rows.length),
28692
+ limit
28693
+ },
28694
+ targetCounts: counts.slice(0, 100),
28695
+ links: rows,
28696
+ warnings: input.evidence.warnings,
28697
+ caveats: [
28698
+ "This is bounded provider or imported evidence, not a complete backlink index.",
28699
+ "A missing link is not proof that no link exists, and an observed link is not a quality or ranking verdict.",
28700
+ "Referring-domain counts describe the retained rows only."
28701
+ ]
28702
+ };
28703
+ }
28704
+ //#endregion
28705
+ //#region packages/core/dist/providers/dataforseo.js
28706
+ var DataForSeoProvider = class {
28707
+ name = "dataforseo";
28708
+ capabilities = { overview: true };
28709
+ getAuthHeader() {
28710
+ const config = readConfig();
28711
+ const login = config.providers.dataForSeoLogin;
28712
+ const password = config.providers.dataForSeoPassword;
28713
+ if (!login || !password) throw new Error("DataForSEO credentials missing. Add login and password to config.json.");
28714
+ return `Basic ${Buffer.from(`${login}:${password}`).toString("base64")}`;
28715
+ }
28716
+ async keywordOverview(phrase, opts = {}) {
28717
+ const response = await fetch$1("https://api.dataforseo.com/v3/dataforseo_labs/google/keyword_overview/live", {
28718
+ method: "POST",
28719
+ headers: {
28720
+ authorization: this.getAuthHeader(),
28721
+ "content-type": "application/json"
28722
+ },
28723
+ body: JSON.stringify([{
28724
+ keyword: phrase,
28725
+ language_code: "en",
28726
+ location_code: 2840
28727
+ }])
28728
+ });
28729
+ if (!response.ok) throw new Error(`DataForSEO request failed with ${response.status}.`);
28730
+ const json = await response.json();
28731
+ const row = json.tasks?.[0]?.result?.[0];
28732
+ return {
28733
+ data: {
28734
+ phrase: row?.keyword ?? phrase,
28735
+ volume: row?.search_volume,
28736
+ competition: row?.competition,
28737
+ cpc: row?.cpc,
28738
+ difficulty: row?.keyword_difficulty
28739
+ },
28740
+ usage: {
28741
+ provider: "DataForSEO",
28742
+ units: 1,
28743
+ unitLabel: "tasks",
28744
+ estimatedUsd: json.cost,
28745
+ calls: 1
28746
+ },
28747
+ warnings: opts.database ? ["DataForSEO overview currently ignores the database option."] : void 0
28748
+ };
28749
+ }
28750
+ };
28751
+ //#endregion
28752
+ //#region packages/core/dist/providers/semrush/csv.js
28753
+ function parseSemicolonCsv(text) {
28754
+ return text.trim().split("\n").map((line) => line.split(";").map((cell) => cell.replace(/^"|"$/g, "").trim()));
27262
28755
  }
27263
28756
  //#endregion
27264
28757
  //#region packages/core/dist/providers/semrush/cache.js
@@ -27463,6 +28956,472 @@ var LocalCredentialsProvider = class {
27463
28956
  }
27464
28957
  };
27465
28958
  new LocalCredentialsProvider();
28959
+ //#endregion
28960
+ //#region packages/core/dist/server-logs/parse.js
28961
+ const MONTHS = {
28962
+ Jan: 0,
28963
+ Feb: 1,
28964
+ Mar: 2,
28965
+ Apr: 3,
28966
+ May: 4,
28967
+ Jun: 5,
28968
+ Jul: 6,
28969
+ Aug: 7,
28970
+ Sep: 8,
28971
+ Oct: 9,
28972
+ Nov: 10,
28973
+ Dec: 11
28974
+ };
28975
+ const CRAWLERS = [
28976
+ {
28977
+ family: "Googlebot",
28978
+ category: "search",
28979
+ patterns: ["googlebot", "google-inspectiontool"]
28980
+ },
28981
+ {
28982
+ family: "Bingbot",
28983
+ category: "search",
28984
+ patterns: ["bingbot", "bingpreview"]
28985
+ },
28986
+ {
28987
+ family: "Applebot",
28988
+ category: "search",
28989
+ patterns: ["applebot"]
28990
+ },
28991
+ {
28992
+ family: "DuckDuckBot",
28993
+ category: "search",
28994
+ patterns: ["duckduckbot"]
28995
+ },
28996
+ {
28997
+ family: "YandexBot",
28998
+ category: "search",
28999
+ patterns: ["yandexbot"]
29000
+ },
29001
+ {
29002
+ family: "Baiduspider",
29003
+ category: "search",
29004
+ patterns: ["baiduspider"]
29005
+ },
29006
+ {
29007
+ family: "PetalBot",
29008
+ category: "search",
29009
+ patterns: ["petalbot"]
29010
+ },
29011
+ {
29012
+ family: "OpenAI",
29013
+ category: "ai",
29014
+ patterns: [
29015
+ "gptbot",
29016
+ "chatgpt-user",
29017
+ "oai-searchbot"
29018
+ ]
29019
+ },
29020
+ {
29021
+ family: "Anthropic",
29022
+ category: "ai",
29023
+ patterns: [
29024
+ "claudebot",
29025
+ "claude-searchbot",
29026
+ "claude-user"
29027
+ ]
29028
+ },
29029
+ {
29030
+ family: "Perplexity",
29031
+ category: "ai",
29032
+ patterns: ["perplexitybot", "perplexity-user"]
29033
+ },
29034
+ {
29035
+ family: "Amazonbot",
29036
+ category: "ai",
29037
+ patterns: ["amazonbot"]
29038
+ },
29039
+ {
29040
+ family: "Bytespider",
29041
+ category: "ai",
29042
+ patterns: ["bytespider"]
29043
+ },
29044
+ {
29045
+ family: "CCBot",
29046
+ category: "ai",
29047
+ patterns: ["ccbot"]
29048
+ },
29049
+ {
29050
+ family: "Cohere",
29051
+ category: "ai",
29052
+ patterns: ["cohere-ai"]
29053
+ }
29054
+ ];
29055
+ const COMBINED_LOG = /^\S+\s+\S+\s+\S+\s+\[([^\]]+)]\s+"([^"\s]+)\s+([^"\s]+)(?:\s+[^"\s]+)?"\s+(\d{3})\s+(\d+|-)\s*(?:"[^"]*"\s+"([^"]*)")?\s*$/;
29056
+ function classifyCrawler(userAgent) {
29057
+ if (!userAgent) return void 0;
29058
+ const value = userAgent.toLowerCase();
29059
+ const match = CRAWLERS.find((crawler) => crawler.patterns.some((pattern) => value.includes(pattern)));
29060
+ return match ? {
29061
+ family: match.family,
29062
+ category: match.category
29063
+ } : void 0;
29064
+ }
29065
+ function logTimestamp(value) {
29066
+ const match = /^(\d{2})\/([A-Z][a-z]{2})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})$/.exec(value);
29067
+ if (!match) return void 0;
29068
+ const month = MONTHS[match[2] ?? ""];
29069
+ if (month === void 0) return void 0;
29070
+ const utc = Date.UTC(Number(match[3]), month, Number(match[1]), Number(match[4]), Number(match[5]), Number(match[6]));
29071
+ const offsetMinutes = Number(match[8]) * 60 + Number(match[9]);
29072
+ const adjusted = utc + (match[7] === "+" ? -1 : 1) * offsetMinutes * 6e4;
29073
+ const date = new Date(adjusted);
29074
+ return Number.isNaN(date.getTime()) ? void 0 : date.toISOString();
29075
+ }
29076
+ function normalizedPath(value) {
29077
+ if (typeof value !== "string" || value.length < 1 || value.length > 2e3) return;
29078
+ try {
29079
+ if (value.startsWith("/")) return new URL(value, "https://server-log.invalid").pathname;
29080
+ const url = new URL(value);
29081
+ return ["http:", "https:"].includes(url.protocol) ? url.pathname : void 0;
29082
+ } catch {
29083
+ return;
29084
+ }
29085
+ }
29086
+ function normalizedMethod(value) {
29087
+ if (typeof value !== "string" || !/^[A-Za-z]{1,16}$/.test(value)) return;
29088
+ return value.toUpperCase();
29089
+ }
29090
+ function normalizedStatus(value) {
29091
+ const status = typeof value === "number" ? value : Number(value);
29092
+ return Number.isInteger(status) && status >= 100 && status <= 599 ? status : void 0;
29093
+ }
29094
+ function normalizedBytes(value) {
29095
+ if (value === void 0 || value === null || value === "" || value === "-") return;
29096
+ const bytes = typeof value === "number" ? value : Number(value);
29097
+ return Number.isSafeInteger(bytes) && bytes >= 0 ? bytes : void 0;
29098
+ }
29099
+ function record(input) {
29100
+ let timestamp;
29101
+ if (typeof input.timestamp === "string") {
29102
+ const date = new Date(input.timestamp);
29103
+ if (!Number.isNaN(date.getTime())) timestamp = date.toISOString();
29104
+ }
29105
+ const method = normalizedMethod(input.method);
29106
+ const path = normalizedPath(input.path);
29107
+ const status = normalizedStatus(input.status);
29108
+ if (!timestamp || !method || !path || status === void 0) return void 0;
29109
+ const userAgent = typeof input.userAgent === "string" && input.userAgent.length <= 2e3 ? input.userAgent : void 0;
29110
+ return {
29111
+ timestamp,
29112
+ method,
29113
+ path,
29114
+ status,
29115
+ bytes: normalizedBytes(input.bytes),
29116
+ userAgent,
29117
+ crawler: classifyCrawler(userAgent)
29118
+ };
29119
+ }
29120
+ function parseCombinedLogLine(line) {
29121
+ const match = COMBINED_LOG.exec(line);
29122
+ if (!match) return void 0;
29123
+ const timestamp = logTimestamp(match[1] ?? "");
29124
+ if (!timestamp) return void 0;
29125
+ return record({
29126
+ timestamp,
29127
+ method: match[2],
29128
+ path: match[3],
29129
+ status: match[4],
29130
+ bytes: match[5],
29131
+ userAgent: match[6]
29132
+ });
29133
+ }
29134
+ function firstValue(value, names) {
29135
+ for (const name of names) if (value[name] !== void 0) return value[name];
29136
+ }
29137
+ function parseJsonLogLine(line) {
29138
+ let value;
29139
+ try {
29140
+ value = JSON.parse(line);
29141
+ } catch {
29142
+ return;
29143
+ }
29144
+ if (!value || typeof value !== "object" || Array.isArray(value)) return;
29145
+ const row = value;
29146
+ return record({
29147
+ timestamp: firstValue(row, [
29148
+ "timestamp",
29149
+ "time",
29150
+ "datetime",
29151
+ "@timestamp"
29152
+ ]),
29153
+ method: firstValue(row, [
29154
+ "method",
29155
+ "requestMethod",
29156
+ "http_method"
29157
+ ]),
29158
+ path: firstValue(row, [
29159
+ "path",
29160
+ "url",
29161
+ "requestUri",
29162
+ "request_uri"
29163
+ ]),
29164
+ status: firstValue(row, [
29165
+ "status",
29166
+ "statusCode",
29167
+ "status_code"
29168
+ ]),
29169
+ bytes: firstValue(row, [
29170
+ "bytes",
29171
+ "bodyBytesSent",
29172
+ "body_bytes_sent"
29173
+ ]),
29174
+ userAgent: firstValue(row, [
29175
+ "userAgent",
29176
+ "user_agent",
29177
+ "httpUserAgent",
29178
+ "http_user_agent"
29179
+ ])
29180
+ });
29181
+ }
29182
+ function emptyStatusBreakdown() {
29183
+ return {
29184
+ success: 0,
29185
+ redirect: 0,
29186
+ clientError: 0,
29187
+ serverError: 0,
29188
+ other: 0
29189
+ };
29190
+ }
29191
+ function addStatus(breakdown, status) {
29192
+ if (status >= 200 && status < 300) breakdown.success += 1;
29193
+ else if (status >= 300 && status < 400) breakdown.redirect += 1;
29194
+ else if (status >= 400 && status < 500) breakdown.clientError += 1;
29195
+ else if (status >= 500) breakdown.serverError += 1;
29196
+ else breakdown.other += 1;
29197
+ }
29198
+ //#endregion
29199
+ //#region packages/core/dist/server-logs/import.js
29200
+ const DEFAULT_ROW_LIMIT = 1e6;
29201
+ const MAX_ROW_LIMIT = 1e7;
29202
+ const DEFAULT_PATH_LIMIT = 25e3;
29203
+ const MAX_PATH_LIMIT = 1e5;
29204
+ const BYTE_LIMIT = 1e9;
29205
+ const MAX_LINE_BYTES = 65536;
29206
+ function boundedInteger(value, fallback, maximum, label) {
29207
+ const result = value ?? fallback;
29208
+ if (!Number.isInteger(result) || result < 1 || result > maximum) throw new SeoError("INVALID_INPUT", `${label} must be between 1 and ${maximum}.`);
29209
+ return result;
29210
+ }
29211
+ function formatFor(path, explicit) {
29212
+ if (explicit) return explicit;
29213
+ const extension = extname(path).toLowerCase();
29214
+ return [".jsonl", ".ndjson"].includes(extension) ? "jsonl" : "combined";
29215
+ }
29216
+ async function* lines(path, progress) {
29217
+ const stream = createReadStream(path, {
29218
+ encoding: "utf8",
29219
+ highWaterMark: 64 * 1024
29220
+ });
29221
+ let remainder = "";
29222
+ let discarding = false;
29223
+ for await (const chunk of stream) {
29224
+ const remainingBytes = BYTE_LIMIT - progress.bytesRead;
29225
+ if (remainingBytes <= 0) {
29226
+ progress.bytesCapped = true;
29227
+ break;
29228
+ }
29229
+ const chunkBytes = Buffer.byteLength(chunk);
29230
+ const accepted = chunkBytes > remainingBytes ? Buffer.from(chunk).subarray(0, remainingBytes).toString("utf8") : chunk;
29231
+ progress.bytesRead += Buffer.byteLength(accepted);
29232
+ if (chunkBytes > remainingBytes) progress.bytesCapped = true;
29233
+ remainder += accepted;
29234
+ let newline = remainder.indexOf("\n");
29235
+ while (newline >= 0) {
29236
+ const line = remainder.slice(0, newline).replace(/\r$/, "");
29237
+ remainder = remainder.slice(newline + 1);
29238
+ if (discarding || Buffer.byteLength(line) > MAX_LINE_BYTES) {
29239
+ yield { tooLong: true };
29240
+ discarding = false;
29241
+ } else if (line.trim()) yield { line };
29242
+ newline = remainder.indexOf("\n");
29243
+ }
29244
+ if (Buffer.byteLength(remainder) > MAX_LINE_BYTES) {
29245
+ remainder = "";
29246
+ discarding = true;
29247
+ }
29248
+ if (progress.bytesCapped) break;
29249
+ }
29250
+ if (discarding || Buffer.byteLength(remainder) > MAX_LINE_BYTES) yield { tooLong: true };
29251
+ else if (remainder.trim()) yield { line: remainder.replace(/\r$/, "") };
29252
+ }
29253
+ function compareCrawler(a, b) {
29254
+ return b.requests - a.requests || a.family.localeCompare(b.family, "en");
29255
+ }
29256
+ function compareCrawlerPath(a, b) {
29257
+ return b.requests - a.requests || a.family.localeCompare(b.family, "en") || a.path.localeCompare(b.path, "en");
29258
+ }
29259
+ function updatePeriod(timestamp, period) {
29260
+ if (!period.firstSeenAt || timestamp < period.firstSeenAt) period.firstSeenAt = timestamp;
29261
+ if (!period.lastSeenAt || timestamp > period.lastSeenAt) period.lastSeenAt = timestamp;
29262
+ }
29263
+ function newCrawler(record) {
29264
+ const crawler = record.crawler;
29265
+ if (!crawler) throw new Error("Crawler evidence is required.");
29266
+ return {
29267
+ family: crawler.family,
29268
+ category: crawler.category,
29269
+ requests: 0,
29270
+ lastSeenAt: record.timestamp,
29271
+ ...emptyStatusBreakdown()
29272
+ };
29273
+ }
29274
+ function newCrawlerPath(record) {
29275
+ return {
29276
+ ...newCrawler(record),
29277
+ path: record.path
29278
+ };
29279
+ }
29280
+ async function importServerLog(input) {
29281
+ const path = resolve(input.file);
29282
+ const file = await stat(path).catch(() => void 0);
29283
+ if (!file?.isFile()) throw new SeoError("INVALID_INPUT", `Server log was not found: ${path}`);
29284
+ const format = formatFor(path, input.format);
29285
+ const rowLimit = boundedInteger(input.rowLimit, DEFAULT_ROW_LIMIT, MAX_ROW_LIMIT, "Server log row limit");
29286
+ const pathLimit = boundedInteger(input.pathLimit, DEFAULT_PATH_LIMIT, MAX_PATH_LIMIT, "Server log path limit");
29287
+ const parse = format === "jsonl" ? parseJsonLogLine : parseCombinedLogLine;
29288
+ const progress = {
29289
+ bytesRead: 0,
29290
+ bytesCapped: false
29291
+ };
29292
+ const period = {};
29293
+ const statusCodes = /* @__PURE__ */ new Map();
29294
+ const crawlers = /* @__PURE__ */ new Map();
29295
+ const crawlerPaths = /* @__PURE__ */ new Map();
29296
+ let suppliedRows = 0;
29297
+ let parsedRows = 0;
29298
+ let invalidRows = 0;
29299
+ let crawlerRows = 0;
29300
+ let responseBytes = 0;
29301
+ let rowsCapped = false;
29302
+ let pathsCapped = false;
29303
+ let untrackedCrawlerPathRows = 0;
29304
+ for await (const item of lines(path, progress)) {
29305
+ if (suppliedRows >= rowLimit) {
29306
+ rowsCapped = true;
29307
+ break;
29308
+ }
29309
+ suppliedRows += 1;
29310
+ const parsed = item.line ? parse(item.line) : void 0;
29311
+ if (!parsed) {
29312
+ invalidRows += 1;
29313
+ continue;
29314
+ }
29315
+ parsedRows += 1;
29316
+ responseBytes += parsed.bytes ?? 0;
29317
+ updatePeriod(parsed.timestamp, period);
29318
+ statusCodes.set(parsed.status, (statusCodes.get(parsed.status) ?? 0) + 1);
29319
+ if (!parsed.crawler) continue;
29320
+ crawlerRows += 1;
29321
+ const crawler = crawlers.get(parsed.crawler.family) ?? newCrawler(parsed);
29322
+ crawler.requests += 1;
29323
+ if (parsed.timestamp > crawler.lastSeenAt) crawler.lastSeenAt = parsed.timestamp;
29324
+ addStatus(crawler, parsed.status);
29325
+ crawlers.set(parsed.crawler.family, crawler);
29326
+ const key = `${parsed.crawler.family}\u0000${parsed.path}`;
29327
+ let crawlerPath = crawlerPaths.get(key);
29328
+ if (!crawlerPath && crawlerPaths.size >= pathLimit) {
29329
+ pathsCapped = true;
29330
+ untrackedCrawlerPathRows += 1;
29331
+ continue;
29332
+ }
29333
+ crawlerPath ??= newCrawlerPath(parsed);
29334
+ crawlerPath.requests += 1;
29335
+ if (parsed.timestamp > crawlerPath.lastSeenAt) crawlerPath.lastSeenAt = parsed.timestamp;
29336
+ addStatus(crawlerPath, parsed.status);
29337
+ crawlerPaths.set(key, crawlerPath);
29338
+ }
29339
+ const fileReadCompletely = !rowsCapped && !progress.bytesCapped && progress.bytesRead >= file.size;
29340
+ const completeness = fileReadCompletely && invalidRows === 0 && !pathsCapped ? "complete" : "partial";
29341
+ const warnings = [];
29342
+ if (rowsCapped) warnings.push(`Analysis stopped after ${rowLimit} rows.`);
29343
+ if (progress.bytesCapped) warnings.push(`Analysis stopped after ${BYTE_LIMIT} input bytes.`);
29344
+ if (invalidRows) warnings.push(`${invalidRows} malformed or unsupported rows were skipped.`);
29345
+ if (pathsCapped) warnings.push(`Crawler path tracking stopped at ${pathLimit} unique crawler and path pairs. Overall crawler and status totals still include later valid rows.`);
29346
+ return {
29347
+ summary: {
29348
+ suppliedRows,
29349
+ parsedRows,
29350
+ invalidRows,
29351
+ crawlerRows,
29352
+ nonCrawlerRows: parsedRows - crawlerRows,
29353
+ responseBytes,
29354
+ ...period
29355
+ },
29356
+ statusCodes: [...statusCodes].map(([status, requests]) => ({
29357
+ status,
29358
+ requests
29359
+ })).sort((a, b) => a.status - b.status),
29360
+ crawlers: [...crawlers.values()].sort(compareCrawler),
29361
+ crawlerPaths: [...crawlerPaths.values()].sort(compareCrawlerPath),
29362
+ provenance: {
29363
+ source: "local-server-log",
29364
+ observedAt: (/* @__PURE__ */ new Date()).toISOString(),
29365
+ cached: false,
29366
+ file: {
29367
+ path,
29368
+ format,
29369
+ bytesRead: progress.bytesRead,
29370
+ fileBytes: file.size
29371
+ },
29372
+ limits: {
29373
+ rowLimit,
29374
+ pathLimit,
29375
+ byteLimit: BYTE_LIMIT,
29376
+ maxLineBytes: MAX_LINE_BYTES
29377
+ },
29378
+ coverage: {
29379
+ fileReadCompletely,
29380
+ rowsCapped,
29381
+ bytesCapped: progress.bytesCapped,
29382
+ pathsCapped,
29383
+ untrackedCrawlerPathRows
29384
+ },
29385
+ completeness
29386
+ },
29387
+ warnings
29388
+ };
29389
+ }
29390
+ //#endregion
29391
+ //#region packages/core/dist/server-logs/report.js
29392
+ const DEFAULT_LIMIT = 100;
29393
+ const MAX_LIMIT = 500;
29394
+ function outputLimit(value) {
29395
+ const limit = value ?? DEFAULT_LIMIT;
29396
+ if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) throw new SeoError("INVALID_INPUT", `Server log output limit must be between 1 and ${MAX_LIMIT}.`);
29397
+ return limit;
29398
+ }
29399
+ function serverLogReport(input) {
29400
+ const limit = outputLimit(input.limit);
29401
+ const crawlerPaths = input.evidence.crawlerPaths.slice(0, limit);
29402
+ return {
29403
+ schemaVersion: 1,
29404
+ generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
29405
+ dataStatus: input.evidence.provenance.completeness,
29406
+ summary: input.evidence.summary,
29407
+ statusCodes: input.evidence.statusCodes,
29408
+ crawlers: input.evidence.crawlers,
29409
+ crawlerPaths,
29410
+ selection: {
29411
+ availableCrawlerPaths: input.evidence.crawlerPaths.length,
29412
+ returnedCrawlerPaths: crawlerPaths.length,
29413
+ omittedCrawlerPaths: Math.max(0, input.evidence.crawlerPaths.length - crawlerPaths.length),
29414
+ limit
29415
+ },
29416
+ provenance: input.evidence.provenance,
29417
+ warnings: input.evidence.warnings,
29418
+ caveats: [
29419
+ "Crawler families are identified from user-agent strings, which can be missing or spoofed.",
29420
+ "This report describes only the supplied log file and cannot prove that every server or time period is represented.",
29421
+ "Crawler path rankings describe retained path aggregates when the path limit is reached."
29422
+ ]
29423
+ };
29424
+ }
27466
29425
  const TELEMETRY_TIMEOUT_MS = 2e3;
27467
29426
  const TELEMETRY_REPORTS = [
27468
29427
  "affected-urls",
@@ -27472,6 +29431,7 @@ const TELEMETRY_REPORTS = [
27472
29431
  "ai-search-scorecard",
27473
29432
  "audit-page",
27474
29433
  "audit-urls",
29434
+ "bing-webmaster-overview",
27475
29435
  "cannibalisation",
27476
29436
  "community-intent",
27477
29437
  "compare-crawls",
@@ -27490,6 +29450,8 @@ const TELEMETRY_REPORTS = [
27490
29450
  "index-monitor",
27491
29451
  "index-watch",
27492
29452
  "internal-links",
29453
+ "link-evidence",
29454
+ "server-log-analysis",
27493
29455
  "link-recovery",
27494
29456
  "crawl-history",
27495
29457
  "crawler-rules",
@@ -29462,6 +31424,87 @@ function registerOpportunityTools(server, dependencies = {}) {
29462
31424
  });
29463
31425
  }
29464
31426
  //#endregion
31427
+ //#region packages/mcp/src/provider-tools.ts
31428
+ const MAX_AGENT_ROWS_PER_SECTION = 14;
31429
+ function compactRows(data) {
31430
+ if (data.rows.length <= MAX_AGENT_ROWS_PER_SECTION && data.outputSelection) return data;
31431
+ const rows = data.rows.slice(-14);
31432
+ return {
31433
+ ...data,
31434
+ rows,
31435
+ outputSelection: {
31436
+ strategy: "most-recent",
31437
+ availableRows: data.rows.length,
31438
+ returnedRows: rows.length,
31439
+ omittedRows: Math.max(0, data.rows.length - rows.length)
31440
+ }
31441
+ };
31442
+ }
31443
+ function compactBingWebmasterOverview(result) {
31444
+ return {
31445
+ ...result,
31446
+ traffic: result.traffic.status === "unavailable" ? result.traffic : {
31447
+ ...result.traffic,
31448
+ data: compactRows(result.traffic.data)
31449
+ },
31450
+ crawl: result.crawl.status === "unavailable" ? result.crawl : {
31451
+ ...result.crawl,
31452
+ data: compactRows(result.crawl.data)
31453
+ },
31454
+ outputBudget: result.outputBudget
31455
+ };
31456
+ }
31457
+ function registerProviderTools(server) {
31458
+ server.registerTool("seo_bing_webmaster_overview", {
31459
+ description: "Find bounded Bing traffic, crawl, query, and page insights for one verified site",
31460
+ inputSchema: { site: z$1.string().url().max(2e3) }
31461
+ }, async ({ site }) => {
31462
+ try {
31463
+ const result = await bingWebmasterOverview({ site });
31464
+ const firstFinding = result.findings[0]?.title;
31465
+ return toolSuccess(`Bing evidence is ${result.dataStatus} with ${result.summary.findings} findings.${firstFinding ? ` Review first: ${firstFinding}.` : ""}`, compactBingWebmasterOverview(result));
31466
+ } catch (error) {
31467
+ return toolError(error);
31468
+ }
31469
+ });
31470
+ server.registerTool("seo_link_evidence", {
31471
+ description: "Review bounded referring-link evidence from Bing Webmaster or a local export",
31472
+ inputSchema: {
31473
+ site: z$1.string().url().max(2e3).optional(),
31474
+ file: z$1.string().min(1).max(4096).optional(),
31475
+ format: z$1.enum([
31476
+ "csv",
31477
+ "json",
31478
+ "jsonl"
31479
+ ]).optional(),
31480
+ rowLimit: z$1.number().int().min(1).max(1e5).optional(),
31481
+ targetLimit: z$1.number().int().min(1).max(50).optional(),
31482
+ detailPagesPerTarget: z$1.number().int().min(1).max(3).optional(),
31483
+ limit: z$1.number().int().min(1).max(500).optional()
31484
+ }
31485
+ }, async ({ site, file, format, rowLimit, targetLimit, detailPagesPerTarget, limit }) => {
31486
+ try {
31487
+ if (Boolean(site) === Boolean(file)) throw new SeoError("INVALID_INPUT", "Pass exactly one of site for Bing or file for a local import.");
31488
+ const report = linkEvidenceReport({
31489
+ evidence: file ? await importLinkEvidence({
31490
+ file,
31491
+ format,
31492
+ rowLimit
31493
+ }) : await collectBingLinkEvidence({
31494
+ site: site ?? "",
31495
+ rowLimit,
31496
+ targetLimit,
31497
+ detailPagesPerTarget
31498
+ }),
31499
+ limit
31500
+ });
31501
+ return toolSuccess(`${report.summary.observedLinks} referring links were retained from ${report.provenance.provider}.`, report);
31502
+ } catch (error) {
31503
+ return toolError(error);
31504
+ }
31505
+ });
31506
+ }
31507
+ //#endregion
29465
31508
  //#region packages/mcp/src/pseo-tools.ts
29466
31509
  function compactPseoReport(result) {
29467
31510
  return {
@@ -29754,6 +31797,91 @@ const REPORT_DEPTH = {
29754
31797
  }
29755
31798
  ]
29756
31799
  },
31800
+ "bing-webmaster-overview": {
31801
+ readOrder: [
31802
+ "dataStatus and provenance",
31803
+ "summary and findings, including verification steps",
31804
+ "traffic analysis current and previous periods, then changes",
31805
+ "crawl analysis current and previous snapshots, then changes",
31806
+ "query and page coverage before movements or opportunities",
31807
+ "outputBudget, per-section outputSelection, and caveats"
31808
+ ],
31809
+ doNotClaim: [
31810
+ "Bing evidence does not describe Google or the whole search market, and inIndex is not URL-level proof of indexing.",
31811
+ "A missing weekly top-list entry is not zero traffic, and movements require observation in every week of both periods.",
31812
+ "Opportunity thresholds prioritize review and are not Bing requirements.",
31813
+ "Partial, capped, invalid, or unavailable rows cannot support an all-clear."
31814
+ ],
31815
+ verify: "Check the same verified site in Bing Webmaster Tools and investigate the exact dates behind any crawl or traffic change.",
31816
+ related: [
31817
+ {
31818
+ id: "search-performance-overview",
31819
+ reason: "Adds Search Console evidence."
31820
+ },
31821
+ {
31822
+ id: "site-crawl",
31823
+ reason: "Checks the current site directly."
31824
+ },
31825
+ {
31826
+ id: "link-recovery",
31827
+ reason: "Finds broken search-value URLs."
31828
+ }
31829
+ ]
31830
+ },
31831
+ "link-evidence": {
31832
+ readOrder: [
31833
+ "dataStatus, provenance, warnings, and caveats",
31834
+ "selection and summary coverage counts",
31835
+ "targetCounts, then the bounded links list"
31836
+ ],
31837
+ doNotClaim: [
31838
+ "The retained rows are not a complete backlink index.",
31839
+ "Missing rows do not prove a link does not exist.",
31840
+ "Observed links do not establish quality, authority, value, or ranking impact."
31841
+ ],
31842
+ verify: "Open a selected referring URL and confirm the link is still present before acting on it.",
31843
+ related: [
31844
+ {
31845
+ id: "link-recovery",
31846
+ reason: "Finds broken search-value URLs."
31847
+ },
31848
+ {
31849
+ id: "redirect-trace",
31850
+ reason: "Checks a target redirect chain."
31851
+ },
31852
+ {
31853
+ id: "site-crawl",
31854
+ reason: "Checks internal links and live pages."
31855
+ }
31856
+ ]
31857
+ },
31858
+ "server-log-analysis": {
31859
+ readOrder: [
31860
+ "dataStatus, provenance coverage, warnings, and caveats",
31861
+ "summary and statusCodes",
31862
+ "crawlers, then the bounded crawlerPaths list and selection"
31863
+ ],
31864
+ doNotClaim: [
31865
+ "A user-agent match does not verify the crawler identity or source IP.",
31866
+ "The supplied file does not prove that every server or time period is represented.",
31867
+ "Path rankings are partial when path tracking reached its limit."
31868
+ ],
31869
+ verify: "Check important crawler errors against the original log line and deployment or server configuration before changing the site.",
31870
+ related: [
31871
+ {
31872
+ id: "site-crawl",
31873
+ reason: "Checks the current site directly."
31874
+ },
31875
+ {
31876
+ id: "crawler-rules",
31877
+ reason: "Explains current crawl checks."
31878
+ },
31879
+ {
31880
+ id: "redirect-trace",
31881
+ reason: "Traces one failing or redirecting path."
31882
+ }
31883
+ ]
31884
+ },
29757
31885
  cannibalisation: {
29758
31886
  readOrder: [
29759
31887
  "dataStatus and both source.pageExposure and source.propertyDemand with validation counts and completeness",
@@ -31037,6 +33165,27 @@ const REPORT_GUIDANCE = {
31037
33165
  avoidWhen: ["You need the crawler to discover pages for you.", "You only need a deep review of one page."],
31038
33166
  outcome: "Consistent technical checks for each supplied URL."
31039
33167
  },
33168
+ "bing-webmaster-overview": {
33169
+ name: "Bing Webmaster overview",
33170
+ description: "Review bounded Bing traffic trends, crawl changes, and query and page opportunities for one verified site.",
33171
+ useWhen: ["You want processed search and crawl insights observed by Bing.", "You need a second search-engine view beside Search Console."],
33172
+ avoidWhen: ["Bing Webmaster is not connected for this installation.", "You need URL-level backlink rows or proof that a page is indexed."],
33173
+ outcome: "Prioritized Bing findings with traffic comparisons, crawl changes, query and page opportunities, provider coverage, and caveats."
33174
+ },
33175
+ "link-evidence": {
33176
+ name: "Referring link evidence",
33177
+ description: "Review a bounded set of pages that link to the site from Bing Webmaster or a local export.",
33178
+ useWhen: ["You want concrete referring URLs and anchor text where available.", "You need one normalized view of Bing or an exported CSV, JSON, or JSONL file."],
33179
+ avoidWhen: ["You need a complete backlink index or a domain authority score.", "You want to find broken internal links during a crawl."],
33180
+ outcome: "A bounded, deduplicated link list with target-page counts and explicit source coverage."
33181
+ },
33182
+ "server-log-analysis": {
33183
+ name: "Server log crawler analysis",
33184
+ description: "Stream a local access log to see observed search and AI crawler requests, statuses, and paths.",
33185
+ useWhen: ["You need evidence of crawler requests that reached the web server.", "You want to find crawler-facing 4xx or 5xx responses in a local access log."],
33186
+ avoidWhen: ["You do not have a combined or JSONL access log.", "You need to prove crawler identity from a user-agent string alone."],
33187
+ outcome: "Bounded crawler, status, and path aggregates with file coverage and parser limits kept visible."
33188
+ },
31040
33189
  cannibalisation: {
31041
33190
  name: "Query and URL overlap",
31042
33191
  description: "Find search queries that appear across several URLs and decide whether the overlap needs attention.",
@@ -31543,6 +33692,35 @@ function registerReportTools(server) {
31543
33692
  registerSecondPageTool(server);
31544
33693
  }
31545
33694
  //#endregion
33695
+ //#region packages/mcp/src/server-log-tools.ts
33696
+ function registerServerLogTools(server) {
33697
+ server.registerTool("seo_server_log_analysis", {
33698
+ description: "Stream a local access log into bounded search and AI crawler evidence",
33699
+ inputSchema: {
33700
+ file: z$1.string().min(1).max(4096),
33701
+ format: z$1.enum(["combined", "jsonl"]).optional(),
33702
+ rowLimit: z$1.number().int().min(1).max(1e7).optional(),
33703
+ pathLimit: z$1.number().int().min(1).max(1e5).optional(),
33704
+ limit: z$1.number().int().min(1).max(500).optional()
33705
+ }
33706
+ }, async ({ file, format, rowLimit, pathLimit, limit }) => {
33707
+ try {
33708
+ const report = serverLogReport({
33709
+ evidence: await importServerLog({
33710
+ file,
33711
+ format,
33712
+ rowLimit,
33713
+ pathLimit
33714
+ }),
33715
+ limit
33716
+ });
33717
+ return toolSuccess(`${report.summary.crawlerRows} crawler requests were observed in ${report.summary.parsedRows} parsed log rows.`, report);
33718
+ } catch (error) {
33719
+ return toolError(error);
33720
+ }
33721
+ });
33722
+ }
33723
+ //#endregion
31546
33724
  //#region packages/mcp/src/agent-output-budget.ts
31547
33725
  const AGENT_STRUCTURED_OUTPUT_MAX_BYTES = 96 * 1024;
31548
33726
  function jsonBytes(value) {
@@ -31939,6 +34117,16 @@ const reportGroups = [
31939
34117
  "seo_redirect_trace"
31940
34118
  ]
31941
34119
  },
34120
+ {
34121
+ category: "monitoring",
34122
+ register: registerProviderTools,
34123
+ names: ["seo_bing_webmaster_overview", "seo_link_evidence"]
34124
+ },
34125
+ {
34126
+ category: "monitoring",
34127
+ register: registerServerLogTools,
34128
+ names: ["seo_server_log_analysis"]
34129
+ },
31942
34130
  {
31943
34131
  category: "reporting",
31944
34132
  register: registerReportTools,