seo 0.2.9 → 0.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -1
- package/dist/cli.js +219 -109
- package/dist/cli.js.map +1 -1
- package/dist/{dist-BrBR9umh.js → dist-D0EI_L8C.js} +1 -1
- package/dist/{dist-C6zXDARm.js → dist-DaHrxVkW.js} +85 -80
- package/dist/dist-DaHrxVkW.js.map +1 -0
- package/dist/index.d.ts +1223 -64
- package/dist/index.js +2801 -410
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +2574 -386
- package/dist/mcp.js.map +1 -1
- package/dist/{package-C51oI-6o.js → package-B-M-4PZ_.js} +2 -2
- package/dist/{package-C51oI-6o.js.map → package-B-M-4PZ_.js.map} +1 -1
- package/evals/bing-webmaster-overview.json +31 -0
- package/evals/link-evidence.json +29 -0
- package/evals/seo.json +16 -0
- package/evals/server-log-analysis.json +29 -0
- package/package.json +3 -3
- package/skills/seo/SKILL.md +56 -67
- package/dist/dist-C6zXDARm.js.map +0 -1
package/dist/mcp.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { n as load, t as version } from "./package-
|
|
2
|
+
import { n as load, t as version } from "./package-B-M-4PZ_.js";
|
|
3
3
|
import { Headers, fetch as fetch$1 } from "undici";
|
|
4
|
-
import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
5
|
-
import { arch, homedir, platform } from "node:os";
|
|
6
|
-
import { dirname, join } from "node:path";
|
|
4
|
+
import { chmodSync, createReadStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
5
|
+
import { arch, homedir, platform, totalmem } from "node:os";
|
|
6
|
+
import { dirname, extname, join, resolve } from "node:path";
|
|
7
7
|
import { z } from "zod";
|
|
8
8
|
import lockfile from "proper-lockfile";
|
|
9
9
|
import { createHash, randomUUID, sign } from "node:crypto";
|
|
@@ -21,9 +21,9 @@ import { gunzipSync } from "node:zlib";
|
|
|
21
21
|
import { execFile } from "node:child_process";
|
|
22
22
|
import { fileURLToPath } from "node:url";
|
|
23
23
|
import { promisify } from "node:util";
|
|
24
|
+
import { readFile, stat } from "node:fs/promises";
|
|
24
25
|
import "pino";
|
|
25
26
|
import "pino-pretty";
|
|
26
|
-
import "node:fs/promises";
|
|
27
27
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
28
28
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
29
29
|
import * as z$1 from "zod/v4";
|
|
@@ -86,12 +86,12 @@ var SeoError = class extends Error {
|
|
|
86
86
|
this.retryable = errorMetadata[code].retryable;
|
|
87
87
|
}
|
|
88
88
|
};
|
|
89
|
-
function errorMessage$
|
|
89
|
+
function errorMessage$3(error) {
|
|
90
90
|
return error instanceof Error ? error.message : String(error);
|
|
91
91
|
}
|
|
92
92
|
function toSeoError(error) {
|
|
93
93
|
if (error instanceof SeoError) return error;
|
|
94
|
-
const message = errorMessage$
|
|
94
|
+
const message = errorMessage$3(error);
|
|
95
95
|
if (/not logged in/i.test(message)) return new SeoError("AUTH_REQUIRED", message);
|
|
96
96
|
if (/oauth client config.*missing|shared seo google app/i.test(message)) return new SeoError("AUTH_CONFIG_REQUIRED", message);
|
|
97
97
|
if (/invalid_grant|refresh token.*no longer valid/i.test(message)) return new SeoError("AUTH_EXPIRED", message);
|
|
@@ -148,6 +148,7 @@ function resolveSeoCliPaths(input = {}) {
|
|
|
148
148
|
configFile: join(configDir, "config.json"),
|
|
149
149
|
tokensFile: join(configDir, "tokens.json"),
|
|
150
150
|
oauthClientFile: join(configDir, "oauth-client.json"),
|
|
151
|
+
providerSecretsFile: join(configDir, "provider-secrets.json"),
|
|
151
152
|
telemetryStateFile: join(configDir, "telemetry.json"),
|
|
152
153
|
cacheDbFile: join(cacheDir, "cache.db")
|
|
153
154
|
};
|
|
@@ -227,6 +228,7 @@ const clientProfileSchema = z.object({
|
|
|
227
228
|
watchUrls: z.array(z.string()).default([]),
|
|
228
229
|
brandTerms: z.array(z.string()).default([]),
|
|
229
230
|
analytics: analyticsConnectionsSchema,
|
|
231
|
+
searchEngines: z.object({ bing: z.object({ siteUrl: z.string() }).optional() }).optional(),
|
|
230
232
|
reportDay: z.number().int().min(1).max(31).optional(),
|
|
231
233
|
technicalWeekday: z.number().int().min(0).max(7).optional(),
|
|
232
234
|
isDefault: z.boolean().optional(),
|
|
@@ -305,7 +307,7 @@ async function deleteKeyringPassword(service, account) {
|
|
|
305
307
|
}
|
|
306
308
|
//#endregion
|
|
307
309
|
//#region packages/core/dist/storage/config.js
|
|
308
|
-
const KEYRING_SERVICE = "seo";
|
|
310
|
+
const KEYRING_SERVICE$1 = "seo";
|
|
309
311
|
const PRIVATE_FILE_MODE = 384;
|
|
310
312
|
function tightenConfigPermissions(path) {
|
|
311
313
|
try {
|
|
@@ -336,7 +338,7 @@ function tokenMetadata(tokens) {
|
|
|
336
338
|
}
|
|
337
339
|
async function readKeyringTokens(tokens) {
|
|
338
340
|
const account = getKeyringAccount(tokens);
|
|
339
|
-
const [accessToken, refreshToken] = await Promise.all([getKeyringPassword(KEYRING_SERVICE, `${account}:access`), getKeyringPassword(KEYRING_SERVICE, `${account}:refresh`)]);
|
|
341
|
+
const [accessToken, refreshToken] = await Promise.all([getKeyringPassword(KEYRING_SERVICE$1, `${account}:access`), getKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`)]);
|
|
340
342
|
return {
|
|
341
343
|
accessToken: accessToken ?? void 0,
|
|
342
344
|
refreshToken: refreshToken ?? void 0
|
|
@@ -344,14 +346,14 @@ async function readKeyringTokens(tokens) {
|
|
|
344
346
|
}
|
|
345
347
|
async function writeKeyringTokens(tokens) {
|
|
346
348
|
const account = getKeyringAccount(tokens);
|
|
347
|
-
if (tokens.access_token) await setKeyringPassword(KEYRING_SERVICE, `${account}:access`, tokens.access_token);
|
|
348
|
-
else await deleteKeyringPassword(KEYRING_SERVICE, `${account}:access`);
|
|
349
|
-
if (tokens.refresh_token) await setKeyringPassword(KEYRING_SERVICE, `${account}:refresh`, tokens.refresh_token);
|
|
350
|
-
else await deleteKeyringPassword(KEYRING_SERVICE, `${account}:refresh`);
|
|
349
|
+
if (tokens.access_token) await setKeyringPassword(KEYRING_SERVICE$1, `${account}:access`, tokens.access_token);
|
|
350
|
+
else await deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:access`);
|
|
351
|
+
if (tokens.refresh_token) await setKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`, tokens.refresh_token);
|
|
352
|
+
else await deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`);
|
|
351
353
|
}
|
|
352
354
|
async function deleteKeyringTokens(tokens) {
|
|
353
355
|
const account = getKeyringAccount(tokens);
|
|
354
|
-
await Promise.all([deleteKeyringPassword(KEYRING_SERVICE, `${account}:access`), deleteKeyringPassword(KEYRING_SERVICE, `${account}:refresh`)]);
|
|
356
|
+
await Promise.all([deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:access`), deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`)]);
|
|
355
357
|
}
|
|
356
358
|
async function readTokens() {
|
|
357
359
|
ensureSeoCliDirs();
|
|
@@ -769,14 +771,14 @@ async function createGoogleAccessTokenClient() {
|
|
|
769
771
|
}
|
|
770
772
|
//#endregion
|
|
771
773
|
//#region packages/core/dist/storage/cache-maintenance.js
|
|
772
|
-
const MEBIBYTE$
|
|
774
|
+
const MEBIBYTE$2 = 1024 * 1024;
|
|
773
775
|
const CACHE_MAX_AGE_MS = 720 * 60 * 60 * 1e3;
|
|
774
776
|
const CACHE_LIMITS = {
|
|
775
|
-
gsc_cache: 48 * MEBIBYTE$
|
|
776
|
-
ga4_cache: 24 * MEBIBYTE$
|
|
777
|
-
semrush_cache: 16 * MEBIBYTE$
|
|
778
|
-
http_cache: 160 * MEBIBYTE$
|
|
779
|
-
performance_reports: 8 * MEBIBYTE$
|
|
777
|
+
gsc_cache: 48 * MEBIBYTE$2,
|
|
778
|
+
ga4_cache: 24 * MEBIBYTE$2,
|
|
779
|
+
semrush_cache: 16 * MEBIBYTE$2,
|
|
780
|
+
http_cache: 160 * MEBIBYTE$2,
|
|
781
|
+
performance_reports: 8 * MEBIBYTE$2
|
|
780
782
|
};
|
|
781
783
|
const CACHE_POLICIES = [
|
|
782
784
|
{
|
|
@@ -2571,7 +2573,7 @@ function isValidStructuredDate(value) {
|
|
|
2571
2573
|
const parsed = new Date(trimmed);
|
|
2572
2574
|
return Number.isFinite(parsed.getTime()) && parsed.toISOString().slice(0, 10) === datePart;
|
|
2573
2575
|
}
|
|
2574
|
-
function parseJsonLd(blocks, baseUrl) {
|
|
2576
|
+
function parseJsonLd(blocks, baseUrl, retainJsonLd) {
|
|
2575
2577
|
const jsonLd = [];
|
|
2576
2578
|
const invalidJsonLdSamples = [];
|
|
2577
2579
|
const unrecognizedJsonLdTypes = [];
|
|
@@ -2643,7 +2645,7 @@ function parseJsonLd(blocks, baseUrl) {
|
|
|
2643
2645
|
blocks.forEach((block, index) => {
|
|
2644
2646
|
try {
|
|
2645
2647
|
const parsed = JSON.parse(block);
|
|
2646
|
-
jsonLd.push(parsed);
|
|
2648
|
+
if (retainJsonLd) jsonLd.push(parsed);
|
|
2647
2649
|
visit(parsed, { prefixes: /* @__PURE__ */ new Map() }, index, "$");
|
|
2648
2650
|
} catch (error) {
|
|
2649
2651
|
invalidJsonLdSamples.push({
|
|
@@ -2704,8 +2706,8 @@ function usableElementValue($, selector) {
|
|
|
2704
2706
|
return ($(element).attr("content") ?? $(element).attr("datetime") ?? $(element).text()).trim().length > 0;
|
|
2705
2707
|
});
|
|
2706
2708
|
}
|
|
2707
|
-
function extractStructuredData($, baseUrl) {
|
|
2708
|
-
const parsed = parseJsonLd($("script[type=\"application/ld+json\"]").toArray().map((element) => $(element).html() ?? ""), baseUrl);
|
|
2709
|
+
function extractStructuredData($, baseUrl, options = {}) {
|
|
2710
|
+
const parsed = parseJsonLd($("script[type=\"application/ld+json\"]").toArray().map((element) => $(element).html() ?? ""), baseUrl, options.retainJsonLd ?? true);
|
|
2709
2711
|
const microdata = microdataTypes($);
|
|
2710
2712
|
const rdfa = rdfaTypes($);
|
|
2711
2713
|
const schemaTypes = [.../* @__PURE__ */ new Set([
|
|
@@ -2739,6 +2741,16 @@ function extractStructuredData($, baseUrl) {
|
|
|
2739
2741
|
}
|
|
2740
2742
|
//#endregion
|
|
2741
2743
|
//#region packages/core/dist/extract/page-extractor.js
|
|
2744
|
+
const SOCIAL_PROFILE_HOSTS = [
|
|
2745
|
+
"facebook.com",
|
|
2746
|
+
"instagram.com",
|
|
2747
|
+
"linkedin.com",
|
|
2748
|
+
"pinterest.com",
|
|
2749
|
+
"tiktok.com",
|
|
2750
|
+
"twitter.com",
|
|
2751
|
+
"x.com",
|
|
2752
|
+
"youtube.com"
|
|
2753
|
+
];
|
|
2742
2754
|
function safeText$1(value) {
|
|
2743
2755
|
const trimmed = value?.replace(/\s+/g, " ").trim();
|
|
2744
2756
|
return trimmed ? trimmed : void 0;
|
|
@@ -2755,7 +2767,7 @@ function absoluteUrl$2(value, base) {
|
|
|
2755
2767
|
return;
|
|
2756
2768
|
}
|
|
2757
2769
|
}
|
|
2758
|
-
function httpUrl(value, base) {
|
|
2770
|
+
function httpUrl$1(value, base) {
|
|
2759
2771
|
const resolved = absoluteUrl$2(value, base);
|
|
2760
2772
|
if (!resolved) return void 0;
|
|
2761
2773
|
const protocol = new URL(resolved).protocol;
|
|
@@ -2811,6 +2823,28 @@ function imageDimensionsFromUrl(src) {
|
|
|
2811
2823
|
height
|
|
2812
2824
|
};
|
|
2813
2825
|
}
|
|
2826
|
+
function addAnchorSample(samples, seen, href, text) {
|
|
2827
|
+
if (samples.length >= 25 || !text) return;
|
|
2828
|
+
const sampleText = text.length > 120 ? text.slice(0, 120).trimEnd() : text;
|
|
2829
|
+
const key = `${href}\n${sampleText}`;
|
|
2830
|
+
if (seen.has(key)) return;
|
|
2831
|
+
seen.add(key);
|
|
2832
|
+
samples.push({
|
|
2833
|
+
href,
|
|
2834
|
+
text: sampleText
|
|
2835
|
+
});
|
|
2836
|
+
}
|
|
2837
|
+
function addSocialProfile(profiles, href) {
|
|
2838
|
+
if (profiles.size >= 50) return;
|
|
2839
|
+
try {
|
|
2840
|
+
const url = new URL(href);
|
|
2841
|
+
const host = url.hostname.replace(/^www\./, "").toLowerCase();
|
|
2842
|
+
if (SOCIAL_PROFILE_HOSTS.some((value) => host === value || host.endsWith(`.${value}`))) {
|
|
2843
|
+
url.hash = "";
|
|
2844
|
+
profiles.add(url.toString());
|
|
2845
|
+
}
|
|
2846
|
+
} catch {}
|
|
2847
|
+
}
|
|
2814
2848
|
async function extractPage(fetchResult, extractor = "defuddle", dependencies = {}) {
|
|
2815
2849
|
const $ = extractor === "crawler" && /<html(?:\s|>)/i.test(fetchResult.html) ? load(fetchResult.html, { xml: {
|
|
2816
2850
|
xmlMode: false,
|
|
@@ -2818,65 +2852,106 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
2818
2852
|
} }) : load(fetchResult.html);
|
|
2819
2853
|
const url = new URL(fetchResult.finalUrl);
|
|
2820
2854
|
const canonicalEvidence = extractCanonicalEvidence($, fetchResult.headers, fetchResult.finalUrl);
|
|
2821
|
-
const headings =
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
2855
|
+
const headings = [];
|
|
2856
|
+
let crawlerH1;
|
|
2857
|
+
let crawlerH1Count = 0;
|
|
2858
|
+
let crawlerH2Count = 0;
|
|
2859
|
+
let crawlerH3Count = 0;
|
|
2860
|
+
let questionHeadings = 0;
|
|
2861
|
+
$("h1, h2, h3, h4, h5, h6").each((_index, element) => {
|
|
2862
|
+
const level = Number.parseInt(element.tagName.slice(1), 10);
|
|
2863
|
+
const text = $(element).text().replace(/\s+/g, " ").trim();
|
|
2864
|
+
if (!text) return;
|
|
2865
|
+
if (text.trimEnd().endsWith("?")) questionHeadings += 1;
|
|
2866
|
+
if (level === 1) {
|
|
2867
|
+
crawlerH1 ??= text;
|
|
2868
|
+
crawlerH1Count += 1;
|
|
2869
|
+
} else if (level === 2) crawlerH2Count += 1;
|
|
2870
|
+
else if (level === 3) crawlerH3Count += 1;
|
|
2871
|
+
if (extractor !== "crawler") headings.push({
|
|
2872
|
+
level,
|
|
2873
|
+
text
|
|
2874
|
+
});
|
|
2875
|
+
});
|
|
2825
2876
|
const links = [];
|
|
2877
|
+
const crawlerInternalLinks = /* @__PURE__ */ new Set();
|
|
2878
|
+
const crawlerExternalLinks = /* @__PURE__ */ new Set();
|
|
2879
|
+
const internalAnchorSamples = [];
|
|
2880
|
+
const externalAnchorSamples = [];
|
|
2881
|
+
const internalAnchorKeys = /* @__PURE__ */ new Set();
|
|
2882
|
+
const externalAnchorKeys = /* @__PURE__ */ new Set();
|
|
2883
|
+
const socialProfileLinks = /* @__PURE__ */ new Set();
|
|
2826
2884
|
let invalidLinkCount = 0;
|
|
2827
2885
|
let unsupportedLinkCount = 0;
|
|
2828
|
-
|
|
2886
|
+
$("a[href]").each((_index, element) => {
|
|
2829
2887
|
const href = $(element).attr("href") ?? "";
|
|
2830
2888
|
try {
|
|
2831
2889
|
const absolute = new URL(href, fetchResult.finalUrl);
|
|
2832
2890
|
if (!["http:", "https:"].includes(absolute.protocol)) {
|
|
2833
2891
|
unsupportedLinkCount += 1;
|
|
2834
|
-
|
|
2892
|
+
return;
|
|
2835
2893
|
}
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2894
|
+
const absoluteHref = absolute.toString();
|
|
2895
|
+
const text = $(element).text().replace(/\s+/g, " ").trim();
|
|
2896
|
+
const internal = absolute.host === url.host;
|
|
2897
|
+
if (extractor === "crawler") {
|
|
2898
|
+
if (absolute.origin === url.origin) {
|
|
2899
|
+
absolute.hash = "";
|
|
2900
|
+
crawlerInternalLinks.add(absolute.toString());
|
|
2901
|
+
}
|
|
2902
|
+
if (!internal) {
|
|
2903
|
+
crawlerExternalLinks.add(absoluteHref);
|
|
2904
|
+
addSocialProfile(socialProfileLinks, absoluteHref);
|
|
2905
|
+
}
|
|
2906
|
+
addAnchorSample(internal ? internalAnchorSamples : externalAnchorSamples, internal ? internalAnchorKeys : externalAnchorKeys, absoluteHref, text);
|
|
2907
|
+
} else links.push({
|
|
2908
|
+
href: absoluteHref,
|
|
2909
|
+
text,
|
|
2839
2910
|
rel: ($(element).attr("rel") ?? "").split(/\s+/).filter(Boolean),
|
|
2840
|
-
internal
|
|
2911
|
+
internal,
|
|
2841
2912
|
location: $(element).closest("nav, header").length ? "navigation" : $(element).closest("footer").length ? "footer" : $(element).closest("main, article").length ? "main-content" : "other"
|
|
2842
2913
|
});
|
|
2843
2914
|
} catch {
|
|
2844
2915
|
invalidLinkCount += 1;
|
|
2845
2916
|
}
|
|
2846
|
-
}
|
|
2917
|
+
});
|
|
2847
2918
|
const hreflang = [];
|
|
2848
2919
|
let invalidHreflangCount = 0;
|
|
2849
2920
|
for (const element of $("link[rel~=\"alternate\"][hreflang][href]").toArray()) {
|
|
2850
2921
|
const language = $(element).attr("hreflang") ?? "";
|
|
2851
|
-
const href = httpUrl($(element).attr("href"), fetchResult.finalUrl);
|
|
2922
|
+
const href = httpUrl$1($(element).attr("href"), fetchResult.finalUrl);
|
|
2852
2923
|
if (language && href) hreflang.push({
|
|
2853
2924
|
hreflang: language,
|
|
2854
2925
|
href
|
|
2855
2926
|
});
|
|
2856
2927
|
else invalidHreflangCount += 1;
|
|
2857
2928
|
}
|
|
2858
|
-
const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
|
|
2929
|
+
const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl$1($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
|
|
2859
2930
|
const openGraph = Object.fromEntries($("meta[property^=\"og:\"]").toArray().map((element) => [$(element).attr("property") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
2860
2931
|
const twitter = Object.fromEntries($("meta[name^=\"twitter:\"]").toArray().map((element) => [$(element).attr("name") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
2861
|
-
const structuredData = extractStructuredData($, fetchResult.finalUrl);
|
|
2932
|
+
const structuredData = extractStructuredData($, fetchResult.finalUrl, { retainJsonLd: extractor !== "crawler" });
|
|
2862
2933
|
const hasDate = structuredData.hasDate || isValidStructuredDate($("meta[property=\"article:published_time\"]").attr("content")) || isValidStructuredDate($("meta[property=\"article:modified_time\"]").attr("content")) || $("article time[datetime], main time[datetime]").toArray().some((element) => isValidStructuredDate($(element).attr("datetime")));
|
|
2863
2934
|
const author = safeText$1($("meta[name=\"author\"]").attr("content"));
|
|
2864
2935
|
const hasAuthor = Boolean(author) || structuredData.hasAuthor || $("[rel~=author], .author, .byline").toArray().some((element) => Boolean(safeText$1($(element).text())));
|
|
2865
2936
|
const semanticHtml = $("main, article").length > 0;
|
|
2866
|
-
const questionHeadings = headings.filter((heading) => heading.text.trimEnd().endsWith("?")).length;
|
|
2867
2937
|
const listCount = $("ul, ol").length;
|
|
2868
2938
|
const tableCount = $("table").length;
|
|
2869
2939
|
const structuredBlocks = listCount + tableCount;
|
|
2870
2940
|
const answerable = $("main p, article p, p").toArray().slice(0, 3).some((element) => $(element).text().replace(/\s+/g, " ").trim().split(/\s+/).filter(Boolean).length >= 25);
|
|
2871
|
-
|
|
2872
|
-
|
|
2941
|
+
let imagesTotal = 0;
|
|
2942
|
+
let imagesMissingAlt = 0;
|
|
2943
|
+
const oversizedImageCandidates = [];
|
|
2944
|
+
$("img").each((_index, element) => {
|
|
2945
|
+
imagesTotal += 1;
|
|
2946
|
+
if ($(element).attr("alt") === void 0) imagesMissingAlt += 1;
|
|
2947
|
+
if (oversizedImageCandidates.length >= 25) return;
|
|
2873
2948
|
const src = absoluteUrl$2($(element).attr("src"), fetchResult.finalUrl);
|
|
2874
|
-
if (!src) return
|
|
2949
|
+
if (!src) return;
|
|
2875
2950
|
const width = numericAttribute($(element).attr("width"));
|
|
2876
2951
|
const height = numericAttribute($(element).attr("height"));
|
|
2877
2952
|
const srcsetWidth = largestSrcsetWidth($(element).attr("srcset"));
|
|
2878
2953
|
const filenameDimensions = imageDimensionsFromUrl(src);
|
|
2879
|
-
if (Math.max(width ?? 0, height ?? 0, srcsetWidth ?? 0, filenameDimensions?.width ?? 0, filenameDimensions?.height ?? 0) < 2e3) return
|
|
2954
|
+
if (Math.max(width ?? 0, height ?? 0, srcsetWidth ?? 0, filenameDimensions?.width ?? 0, filenameDimensions?.height ?? 0) < 2e3) return;
|
|
2880
2955
|
const detectedFrom = [
|
|
2881
2956
|
width && width >= 2e3 ? "width" : void 0,
|
|
2882
2957
|
height && height >= 2e3 ? "height" : void 0,
|
|
@@ -2885,13 +2960,13 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
2885
2960
|
].filter(Boolean).join(",");
|
|
2886
2961
|
const candidateWidth = Math.max(width ?? 0, srcsetWidth ?? 0, filenameDimensions?.width ?? 0);
|
|
2887
2962
|
const candidateHeight = Math.max(height ?? 0, filenameDimensions?.height ?? 0);
|
|
2888
|
-
|
|
2963
|
+
oversizedImageCandidates.push({
|
|
2889
2964
|
src,
|
|
2890
2965
|
detectedFrom,
|
|
2891
2966
|
...candidateWidth > 0 ? { width: candidateWidth } : {},
|
|
2892
2967
|
...candidateHeight > 0 ? { height: candidateHeight } : {}
|
|
2893
|
-
};
|
|
2894
|
-
})
|
|
2968
|
+
});
|
|
2969
|
+
});
|
|
2895
2970
|
const mixedContentUrls = url.protocol === "https:" ? [...new Set($("img[src], script[src], iframe[src], source[src], video[src], audio[src], link[href]").toArray().map((element) => absoluteUrl$2($(element).attr("src") ?? $(element).attr("href"), fetchResult.finalUrl)).filter((value) => typeof value === "string" && new URL(value).protocol === "http:"))] : [];
|
|
2896
2971
|
const metaRobots = combineRobotsValues($("meta[name]").toArray().filter((element) => {
|
|
2897
2972
|
const name = $(element).attr("name")?.trim().toLowerCase();
|
|
@@ -2915,6 +2990,18 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
2915
2990
|
hasViewport: Boolean($("meta[name=\"viewport\"]").attr("content")),
|
|
2916
2991
|
headings,
|
|
2917
2992
|
links,
|
|
2993
|
+
...extractor === "crawler" ? { crawlerEvidence: {
|
|
2994
|
+
h1: crawlerH1,
|
|
2995
|
+
h1Count: crawlerH1Count,
|
|
2996
|
+
h2Count: crawlerH2Count,
|
|
2997
|
+
h3Count: crawlerH3Count,
|
|
2998
|
+
questionHeadings,
|
|
2999
|
+
internalLinks: [...crawlerInternalLinks],
|
|
3000
|
+
externalLinks: [...crawlerExternalLinks],
|
|
3001
|
+
internalAnchorSamples,
|
|
3002
|
+
externalAnchorSamples,
|
|
3003
|
+
socialProfileLinks: [...socialProfileLinks]
|
|
3004
|
+
} } : {},
|
|
2918
3005
|
markdownAlternates,
|
|
2919
3006
|
hreflang,
|
|
2920
3007
|
jsonLd: structuredData.jsonLd,
|
|
@@ -2923,19 +3010,17 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
2923
3010
|
unrecognizedJsonLdTypes: structuredData.unrecognizedJsonLdTypes.slice(0, 25),
|
|
2924
3011
|
structuredDataFormats: structuredData.formats,
|
|
2925
3012
|
googleRichResults: structuredData.googleRichResults,
|
|
2926
|
-
schemaSameAsEvidence: structuredData.sameAs,
|
|
2927
|
-
invalidSchemaSameAs: structuredData.invalidSameAs,
|
|
3013
|
+
schemaSameAsEvidence: extractor === "crawler" ? structuredData.sameAs.slice(0, 50) : structuredData.sameAs,
|
|
3014
|
+
invalidSchemaSameAs: extractor === "crawler" ? structuredData.invalidSameAs.slice(0, 25) : structuredData.invalidSameAs,
|
|
2928
3015
|
schemaTypes: structuredData.schemaTypes,
|
|
2929
3016
|
openGraph,
|
|
2930
3017
|
twitter,
|
|
2931
3018
|
author,
|
|
2932
3019
|
hasAuthor,
|
|
2933
3020
|
hasDate,
|
|
2934
|
-
imagesTotal
|
|
2935
|
-
imagesMissingAlt
|
|
2936
|
-
|
|
2937
|
-
}).length,
|
|
2938
|
-
oversizedImageCandidates: oversizedImageCandidates.slice(0, 25),
|
|
3021
|
+
imagesTotal,
|
|
3022
|
+
imagesMissingAlt,
|
|
3023
|
+
oversizedImageCandidates,
|
|
2939
3024
|
mixedContentUrls,
|
|
2940
3025
|
semanticHtml,
|
|
2941
3026
|
questionHeadings,
|
|
@@ -2943,7 +3028,7 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
2943
3028
|
tableCount,
|
|
2944
3029
|
structuredBlocks,
|
|
2945
3030
|
answerable,
|
|
2946
|
-
contentText: text.replace(/\s+/g, " ").trim(),
|
|
3031
|
+
contentText: extractor === "crawler" ? text : text.replace(/\s+/g, " ").trim(),
|
|
2947
3032
|
excerpt,
|
|
2948
3033
|
wordCount: content.wordCount,
|
|
2949
3034
|
contentExtraction: content.diagnostics,
|
|
@@ -3473,7 +3558,7 @@ function decodePageFetchCacheEvidence(value) {
|
|
|
3473
3558
|
return;
|
|
3474
3559
|
}
|
|
3475
3560
|
}
|
|
3476
|
-
async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, respectRobots = false) {
|
|
3561
|
+
async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, respectRobots = false, writeCache = true) {
|
|
3477
3562
|
const startedAt = Date.now();
|
|
3478
3563
|
const db = getDb();
|
|
3479
3564
|
const key = hashKey(["page", url]);
|
|
@@ -3585,7 +3670,7 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
|
|
|
3585
3670
|
warnings: [...robots.allowed === null ? [`robots.txt availability is unknown${robots.status ? ` after HTTP ${robots.status}` : ""}; this fetch does not prove crawler access is allowed.`] : [], ...accessBlock ? [`${accessBlock.guidance.summary} Identify the crawler as ${accessBlock.crawler.userAgent}.`] : []],
|
|
3586
3671
|
robotsTxt: pageRobotsEvidence(robots)
|
|
3587
3672
|
};
|
|
3588
|
-
if (!accessBlock) {
|
|
3673
|
+
if (!accessBlock && writeCache) {
|
|
3589
3674
|
db.prepare(`INSERT OR REPLACE INTO http_cache
|
|
3590
3675
|
(url_hash, url, status, headers_json, body_blob, metadata_json, etag, fetched_at, expires_at)
|
|
3591
3676
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(key, url, response.response.status, JSON.stringify(headerMap), Buffer.from(html), encodePageFetchCacheEvidence(result), response.response.headers.get("etag"), Date.now(), Date.now() + 36e5);
|
|
@@ -4153,7 +4238,7 @@ async function fetchPage(url, opts = {}, dependencies = {}) {
|
|
|
4153
4238
|
const makeRenderer = dependencies.createPageRenderer ?? createPageRenderer;
|
|
4154
4239
|
const rate = normalizeRateControls(opts.rate);
|
|
4155
4240
|
return await queueForHost(new URL(url).host, rate).add(async () => {
|
|
4156
|
-
const first = await plainFetch(url, opts.refresh, opts.timeoutMs, rate, opts.signal, opts.respectRobots);
|
|
4241
|
+
const first = await plainFetch(url, opts.refresh, opts.timeoutMs, rate, opts.signal, opts.respectRobots, opts.writeCache);
|
|
4157
4242
|
if (!(!first.diagnostics.accessBlock && (renderingMode === "on" || renderingMode === "auto" && looksLikeSpa(first.html))) || opts.signal?.aborted) return withRenderingState(first, renderingMode, opts.signal?.aborted ? "skipped" : renderingMode === "off" ? "not-requested" : "not-needed");
|
|
4158
4243
|
const ownsRenderer = !opts.renderer;
|
|
4159
4244
|
const renderer = opts.renderer ?? makeRenderer();
|
|
@@ -4258,7 +4343,7 @@ function urlInspectionQuotaDate(now) {
|
|
|
4258
4343
|
function urlInspectionQuotaResetAt(now) {
|
|
4259
4344
|
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1));
|
|
4260
4345
|
}
|
|
4261
|
-
function boundedLimit(value) {
|
|
4346
|
+
function boundedLimit$1(value) {
|
|
4262
4347
|
const limit = value ?? 2e3;
|
|
4263
4348
|
if (!Number.isInteger(limit) || limit < 1 || limit > 2e3) throw new SeoError("INVALID_INPUT", "URL Inspection daily limit must be an integer from 1 to 2000.");
|
|
4264
4349
|
return limit;
|
|
@@ -4291,7 +4376,7 @@ function reserveUrlInspectionQuota(input) {
|
|
|
4291
4376
|
const database = input.database ?? getDb();
|
|
4292
4377
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
4293
4378
|
const nowMs = now.getTime();
|
|
4294
|
-
const requestedLimit = boundedLimit(input.limit);
|
|
4379
|
+
const requestedLimit = boundedLimit$1(input.limit);
|
|
4295
4380
|
const quotaDate = urlInspectionQuotaDate(now);
|
|
4296
4381
|
const resetAt = urlInspectionQuotaResetAt(now);
|
|
4297
4382
|
const result = database.transaction(() => {
|
|
@@ -6074,13 +6159,13 @@ function isLowActionabilityQuery(query) {
|
|
|
6074
6159
|
//#endregion
|
|
6075
6160
|
//#region packages/core/dist/analyze/page-opportunities-selection.js
|
|
6076
6161
|
const DEFAULT_MIN_IMPRESSIONS$3 = 10;
|
|
6077
|
-
const DEFAULT_LIMIT$
|
|
6078
|
-
const MAX_LIMIT$
|
|
6162
|
+
const DEFAULT_LIMIT$4 = 25;
|
|
6163
|
+
const MAX_LIMIT$4 = 100;
|
|
6079
6164
|
function integerInRange$2(value, fallback, minimum, maximum) {
|
|
6080
6165
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
6081
6166
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
6082
6167
|
}
|
|
6083
|
-
function isHttpUrl$
|
|
6168
|
+
function isHttpUrl$2(value) {
|
|
6084
6169
|
try {
|
|
6085
6170
|
const parsed = new URL(value);
|
|
6086
6171
|
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
@@ -6102,7 +6187,7 @@ function isBrandRow(input) {
|
|
|
6102
6187
|
function normalizePageOpportunityOptions(input) {
|
|
6103
6188
|
return {
|
|
6104
6189
|
minImpressions: integerInRange$2(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$3, 1, Number.MAX_SAFE_INTEGER),
|
|
6105
|
-
limit: integerInRange$2(input.limit, DEFAULT_LIMIT$
|
|
6190
|
+
limit: integerInRange$2(input.limit, DEFAULT_LIMIT$4, 1, MAX_LIMIT$4)
|
|
6106
6191
|
};
|
|
6107
6192
|
}
|
|
6108
6193
|
function selectTargetRows(input) {
|
|
@@ -6121,7 +6206,7 @@ function selectTargetRows(input) {
|
|
|
6121
6206
|
for (const row of input.rows) {
|
|
6122
6207
|
const query = row.keys[0]?.trim() ?? "";
|
|
6123
6208
|
const page = row.keys[1] ?? "";
|
|
6124
|
-
if (row.keys.length < 2 || !query || !isHttpUrl$
|
|
6209
|
+
if (row.keys.length < 2 || !query || !isHttpUrl$2(page) || !hasValidMetrics(row)) {
|
|
6125
6210
|
selection.invalidRows++;
|
|
6126
6211
|
continue;
|
|
6127
6212
|
}
|
|
@@ -6157,7 +6242,7 @@ function selectTargetRows(input) {
|
|
|
6157
6242
|
function selectBenchmarkRows$1(input) {
|
|
6158
6243
|
return input.rows.filter((row) => {
|
|
6159
6244
|
const query = row.keys[0]?.trim() ?? "";
|
|
6160
|
-
return row.keys.length >= 2 && isHttpUrl$
|
|
6245
|
+
return row.keys.length >= 2 && isHttpUrl$2(row.keys[1] ?? "") && hasValidMetrics(row) && row.position >= 1 && row.position <= 10 && Boolean(query) && !isLowActionabilityQuery(query) && !isBrandRow({
|
|
6161
6246
|
query,
|
|
6162
6247
|
site: input.site,
|
|
6163
6248
|
brandTerms: input.brandTerms,
|
|
@@ -6339,7 +6424,7 @@ const defaultDependencies$14 = {
|
|
|
6339
6424
|
extract: extractPage,
|
|
6340
6425
|
now: () => /* @__PURE__ */ new Date()
|
|
6341
6426
|
};
|
|
6342
|
-
function errorMessage$
|
|
6427
|
+
function errorMessage$2(error) {
|
|
6343
6428
|
return error instanceof Error ? error.message : String(error);
|
|
6344
6429
|
}
|
|
6345
6430
|
function unique$4(values) {
|
|
@@ -6360,7 +6445,7 @@ async function verifyPage(input) {
|
|
|
6360
6445
|
});
|
|
6361
6446
|
warnings.push(...fetched.warnings);
|
|
6362
6447
|
} catch (error) {
|
|
6363
|
-
const reason = `Page fetch failed: ${errorMessage$
|
|
6448
|
+
const reason = `Page fetch failed: ${errorMessage$2(error)}`;
|
|
6364
6449
|
return {
|
|
6365
6450
|
verification: {
|
|
6366
6451
|
status: "failed",
|
|
@@ -6382,7 +6467,7 @@ async function verifyPage(input) {
|
|
|
6382
6467
|
warnings: unique$4(warnings)
|
|
6383
6468
|
};
|
|
6384
6469
|
} catch (error) {
|
|
6385
|
-
const reason = `Page extraction failed: ${errorMessage$
|
|
6470
|
+
const reason = `Page extraction failed: ${errorMessage$2(error)}`;
|
|
6386
6471
|
return {
|
|
6387
6472
|
fetched,
|
|
6388
6473
|
verification: {
|
|
@@ -7574,7 +7659,7 @@ function check$1(section, input) {
|
|
|
7574
7659
|
...input
|
|
7575
7660
|
};
|
|
7576
7661
|
}
|
|
7577
|
-
function section$
|
|
7662
|
+
function section$2(id, title, checks) {
|
|
7578
7663
|
return {
|
|
7579
7664
|
id,
|
|
7580
7665
|
title,
|
|
@@ -8012,19 +8097,19 @@ function agentReadiness(report) {
|
|
|
8012
8097
|
headline: "No agent-discovery evidence is stored in this crawl. Run a fresh focused check.",
|
|
8013
8098
|
profileApplicability: profiles,
|
|
8014
8099
|
summary: summariseChecks(checks),
|
|
8015
|
-
sections: [section$
|
|
8100
|
+
sections: [section$2("collection", "Evidence collection", [missing]), section$2("profile", "Profile scope", scopeChecks)],
|
|
8016
8101
|
checks,
|
|
8017
8102
|
topActions: [missing],
|
|
8018
8103
|
caveats: ["Not evaluated does not mean absent or broken.", "This report does not measure rankings, AI mentions, citations, or selection."]
|
|
8019
8104
|
};
|
|
8020
8105
|
}
|
|
8021
8106
|
const sections = [
|
|
8022
|
-
section$
|
|
8023
|
-
section$
|
|
8024
|
-
section$
|
|
8025
|
-
section$
|
|
8026
|
-
section$
|
|
8027
|
-
section$
|
|
8107
|
+
section$2("profile", "Profile scope", profileChecks(discovery)),
|
|
8108
|
+
section$2("access", "Crawler and protocol access", accessChecks(report, discovery)),
|
|
8109
|
+
section$2("representations", "HTML and Markdown representations", markdownChecks(discovery)),
|
|
8110
|
+
section$2("discovery", "Agent discovery files", discoveryChecks(discovery, report.ai?.robotsTxt?.contentSignals)),
|
|
8111
|
+
section$2("endpoints", "Agent endpoints and auth discovery", endpointChecks(discovery)),
|
|
8112
|
+
section$2("identity", "Identity evidence", identityChecks(report))
|
|
8028
8113
|
];
|
|
8029
8114
|
const checks = sections.flatMap((item) => item.checks);
|
|
8030
8115
|
const actionable = checks.filter((item) => [
|
|
@@ -8211,7 +8296,7 @@ function status(score, maxScore) {
|
|
|
8211
8296
|
if (ratio >= .5) return "warning";
|
|
8212
8297
|
return "fail";
|
|
8213
8298
|
}
|
|
8214
|
-
function section(id, title, checks) {
|
|
8299
|
+
function section$1(id, title, checks) {
|
|
8215
8300
|
return {
|
|
8216
8301
|
id,
|
|
8217
8302
|
title,
|
|
@@ -8269,7 +8354,7 @@ function aiReadiness(report) {
|
|
|
8269
8354
|
const blockedSnippets = restrictedSnippets.filter(({ control }) => control.status === "blocked");
|
|
8270
8355
|
const limitedSnippets = restrictedSnippets.filter(({ control }) => control.status === "limited");
|
|
8271
8356
|
const sections = [
|
|
8272
|
-
section("agent-access", "Agent access", [
|
|
8357
|
+
section$1("agent-access", "Agent access", [
|
|
8273
8358
|
check({
|
|
8274
8359
|
id: "robots-ai-bots",
|
|
8275
8360
|
section: "agent-access",
|
|
@@ -8312,7 +8397,7 @@ function aiReadiness(report) {
|
|
|
8312
8397
|
evidence: { resources: report.ai?.agentResources ?? [] }
|
|
8313
8398
|
})
|
|
8314
8399
|
]),
|
|
8315
|
-
section("machine-readable", "Machine-readable signals", [check({
|
|
8400
|
+
section$1("machine-readable", "Machine-readable signals", [check({
|
|
8316
8401
|
id: "llms-txt",
|
|
8317
8402
|
section: "machine-readable",
|
|
8318
8403
|
maxScore: 0,
|
|
@@ -8337,7 +8422,7 @@ function aiReadiness(report) {
|
|
|
8337
8422
|
action: invalidJsonLdPages.length ? "Fix invalid JSON-LD before relying on its structured-data claims." : "Use structured data only where it accurately represents the visible page and a supported use case.",
|
|
8338
8423
|
urls: invalidJsonLdPages.slice(0, 10).map((page) => page.finalUrl)
|
|
8339
8424
|
})]),
|
|
8340
|
-
section("content-clarity", "Content clarity", [
|
|
8425
|
+
section$1("content-clarity", "Content clarity", [
|
|
8341
8426
|
check({
|
|
8342
8427
|
id: "answerable-content",
|
|
8343
8428
|
section: "content-clarity",
|
|
@@ -8387,7 +8472,7 @@ function aiReadiness(report) {
|
|
|
8387
8472
|
action: "Set the html lang attribute so crawlers and assistive tech understand the page language."
|
|
8388
8473
|
})
|
|
8389
8474
|
]),
|
|
8390
|
-
section("entity-signals", "Entity signals", [check({
|
|
8475
|
+
section$1("entity-signals", "Entity signals", [check({
|
|
8391
8476
|
id: "entity-schema",
|
|
8392
8477
|
section: "entity-signals",
|
|
8393
8478
|
maxScore: 0,
|
|
@@ -8406,7 +8491,7 @@ function aiReadiness(report) {
|
|
|
8406
8491
|
plainEnglish: "Author and date signals help agents judge freshness and accountability.",
|
|
8407
8492
|
action: "Add visible authors, reviewed dates, and updated dates to content where trust and freshness matter."
|
|
8408
8493
|
})]),
|
|
8409
|
-
section("technical-ux", "Technical UX", [
|
|
8494
|
+
section$1("technical-ux", "Technical UX", [
|
|
8410
8495
|
check({
|
|
8411
8496
|
id: "snippet-controls",
|
|
8412
8497
|
section: "technical-ux",
|
|
@@ -8444,7 +8529,7 @@ function aiReadiness(report) {
|
|
|
8444
8529
|
action: "Fix 4xx/5xx pages, accidental noindex, and blocked internal URLs before expanding AI discovery work."
|
|
8445
8530
|
})
|
|
8446
8531
|
]),
|
|
8447
|
-
section("agent-content", "Agent content observations", [
|
|
8532
|
+
section$1("agent-content", "Agent content observations", [
|
|
8448
8533
|
check({
|
|
8449
8534
|
id: "markdown-alternates",
|
|
8450
8535
|
section: "agent-content",
|
|
@@ -8484,7 +8569,7 @@ function aiReadiness(report) {
|
|
|
8484
8569
|
evidence: agentDiscovery?.profileApplicability
|
|
8485
8570
|
})
|
|
8486
8571
|
]),
|
|
8487
|
-
section("crawl-completeness", "Crawl completeness", [check({
|
|
8572
|
+
section$1("crawl-completeness", "Crawl completeness", [check({
|
|
8488
8573
|
id: "crawl-depth",
|
|
8489
8574
|
section: "crawl-completeness",
|
|
8490
8575
|
maxScore: 8,
|
|
@@ -10431,6 +10516,7 @@ const CRAWL_SKIP_REASONS = [
|
|
|
10431
10516
|
"configured-exclusion",
|
|
10432
10517
|
"duplicate-normalization",
|
|
10433
10518
|
"invalid-url",
|
|
10519
|
+
"memory-pressure",
|
|
10434
10520
|
"off-origin",
|
|
10435
10521
|
"origin-backpressure",
|
|
10436
10522
|
"queue-safety-limit",
|
|
@@ -10443,6 +10529,7 @@ const SKIP_IMPACT = {
|
|
|
10443
10529
|
"configured-exclusion": "non-impacting",
|
|
10444
10530
|
"duplicate-normalization": "non-impacting",
|
|
10445
10531
|
"invalid-url": "non-impacting",
|
|
10532
|
+
"memory-pressure": "coverage-affecting",
|
|
10446
10533
|
"off-origin": "non-impacting",
|
|
10447
10534
|
"origin-backpressure": "coverage-affecting",
|
|
10448
10535
|
"queue-safety-limit": "coverage-affecting",
|
|
@@ -10620,17 +10707,44 @@ function sanitizeTenantString(value) {
|
|
|
10620
10707
|
return value.replace(/https?:\/\/[^\s"'<>`]+/g, (match) => sanitizeUrlString(match)).replace(/\b(access_?token|api_?key|auth|client_?secret|jwt|password|refresh_?token|secret|session|signature|token)(\s*[=:]\s*)[^\s,;&]+/gi, (_match, key, separator) => `${key}${separator}[redacted]`).replace(/\b[A-Za-z]:\\[^\s"'<>`]+/g, "[local-path]").replace(/\/(?:Users|home|tmp|var\/folders)\/[^\s"'<>`]+/g, "[local-path]");
|
|
10621
10708
|
}
|
|
10622
10709
|
function sanitizeTenantValue(value, key) {
|
|
10623
|
-
if (key && isSensitiveFieldName(key)) return "[redacted]";
|
|
10710
|
+
if (key && isSensitiveFieldName(key)) return value === "[redacted]" ? value : "[redacted]";
|
|
10624
10711
|
if (typeof value === "string") return sanitizeTenantString(value);
|
|
10625
|
-
if (Array.isArray(value))
|
|
10712
|
+
if (Array.isArray(value)) {
|
|
10713
|
+
let sanitized;
|
|
10714
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
10715
|
+
const item = sanitizeTenantValue(value[index]);
|
|
10716
|
+
if (sanitized) sanitized.push(item);
|
|
10717
|
+
else if (item !== value[index]) sanitized = [...value.slice(0, index), item];
|
|
10718
|
+
}
|
|
10719
|
+
return sanitized ?? value;
|
|
10720
|
+
}
|
|
10626
10721
|
if (!value || typeof value !== "object") return value;
|
|
10627
|
-
|
|
10722
|
+
const record = value;
|
|
10723
|
+
let sanitized;
|
|
10724
|
+
for (const [entryKey, entry] of Object.entries(record)) {
|
|
10725
|
+
const safeEntry = sanitizeTenantValue(entry, entryKey);
|
|
10726
|
+
if (safeEntry === entry) continue;
|
|
10727
|
+
sanitized ??= { ...record };
|
|
10728
|
+
sanitized[entryKey] = safeEntry;
|
|
10729
|
+
}
|
|
10730
|
+
return sanitized ?? value;
|
|
10628
10731
|
}
|
|
10629
10732
|
function sanitizeCrawlConfig(config) {
|
|
10630
10733
|
return sanitizeTenantValue(config);
|
|
10631
10734
|
}
|
|
10632
10735
|
function sanitizePages(pages) {
|
|
10633
|
-
|
|
10736
|
+
const safePages = sanitizeTenantValue(pages);
|
|
10737
|
+
let normalized;
|
|
10738
|
+
for (let index = 0; index < safePages.length; index += 1) {
|
|
10739
|
+
const page = safePages[index];
|
|
10740
|
+
if (!page) continue;
|
|
10741
|
+
if ("seoScore" in page || "geoScore" in page) {
|
|
10742
|
+
normalized ??= safePages.slice(0, index);
|
|
10743
|
+
const { seoScore: _seoScore, geoScore: _geoScore, ...current } = page;
|
|
10744
|
+
normalized.push(current);
|
|
10745
|
+
} else if (normalized) normalized.push(page);
|
|
10746
|
+
}
|
|
10747
|
+
return normalized ?? safePages;
|
|
10634
10748
|
}
|
|
10635
10749
|
function sanitizeIssues(issues) {
|
|
10636
10750
|
return sanitizeTenantValue(issues);
|
|
@@ -10659,18 +10773,28 @@ function summarizeCrawlReport(input) {
|
|
|
10659
10773
|
const byCategory = {};
|
|
10660
10774
|
let responseMs = 0;
|
|
10661
10775
|
let responseCount = 0;
|
|
10776
|
+
let statusOnlyPages = 0;
|
|
10777
|
+
let indexablePages = 0;
|
|
10778
|
+
let nonIndexablePages = 0;
|
|
10779
|
+
let statusErrors = 0;
|
|
10662
10780
|
for (const page of input.pages) {
|
|
10663
10781
|
byStatus[String(page.status)] = (byStatus[String(page.status)] ?? 0) + 1;
|
|
10664
10782
|
if (typeof page.responseTimeMs === "number") {
|
|
10665
10783
|
responseMs += page.responseTimeMs;
|
|
10666
10784
|
responseCount += 1;
|
|
10667
10785
|
}
|
|
10786
|
+
if (page.auditScope === "status") statusOnlyPages += 1;
|
|
10787
|
+
else if (page.indexable) indexablePages += 1;
|
|
10788
|
+
else nonIndexablePages += 1;
|
|
10789
|
+
if (page.status === 0 || page.status >= 400) statusErrors += 1;
|
|
10668
10790
|
}
|
|
10669
10791
|
const requests = input.requests ?? [];
|
|
10670
|
-
const failedRequests = requests.filter((request) => request.outcome === "failure" && request.failureKind !== "aborted");
|
|
10671
|
-
const abortedRequests = requests.filter((request) => request.outcome === "failure" && request.failureKind === "aborted");
|
|
10672
10792
|
let requestMs = 0;
|
|
10673
10793
|
let requestMsCount = 0;
|
|
10794
|
+
let responseRequests = 0;
|
|
10795
|
+
let failedRequests = 0;
|
|
10796
|
+
let abortedRequests = 0;
|
|
10797
|
+
let extractionFailures = 0;
|
|
10674
10798
|
for (const request of requests) {
|
|
10675
10799
|
const status = request.outcome === "response" ? String(request.status) : request.outcome === "skipped" ? request.reason : request.failureKind === "aborted" ? "aborted" : "no-response";
|
|
10676
10800
|
requestByStatus[status] = (requestByStatus[status] ?? 0) + 1;
|
|
@@ -10678,27 +10802,39 @@ function summarizeCrawlReport(input) {
|
|
|
10678
10802
|
requestMs += request.durationMs;
|
|
10679
10803
|
requestMsCount += 1;
|
|
10680
10804
|
}
|
|
10805
|
+
if (request.outcome === "response") {
|
|
10806
|
+
responseRequests += 1;
|
|
10807
|
+
if (request.extraction === "failed") extractionFailures += 1;
|
|
10808
|
+
} else if (request.outcome === "failure") if (request.failureKind === "aborted") abortedRequests += 1;
|
|
10809
|
+
else failedRequests += 1;
|
|
10810
|
+
}
|
|
10811
|
+
let highIssues = 0;
|
|
10812
|
+
let mediumIssues = 0;
|
|
10813
|
+
let lowIssues = 0;
|
|
10814
|
+
for (const issue of input.issues) {
|
|
10815
|
+
byCategory[issue.category] = (byCategory[issue.category] ?? 0) + 1;
|
|
10816
|
+
if (issue.severity === "high") highIssues += 1;
|
|
10817
|
+
else if (issue.severity === "medium") mediumIssues += 1;
|
|
10818
|
+
else lowIssues += 1;
|
|
10681
10819
|
}
|
|
10682
|
-
for (const issue of input.issues) byCategory[issue.category] = (byCategory[issue.category] ?? 0) + 1;
|
|
10683
10820
|
const runStats = crawlRunStats(input.pages, input.stats);
|
|
10684
|
-
const indexabilityPages = input.pages.filter((page) => page.auditScope !== "status");
|
|
10685
10821
|
return {
|
|
10686
10822
|
totalPages: input.pages.length,
|
|
10687
|
-
statusOnlyPages
|
|
10688
|
-
indexablePages
|
|
10689
|
-
nonIndexablePages
|
|
10690
|
-
statusErrors
|
|
10823
|
+
statusOnlyPages,
|
|
10824
|
+
indexablePages,
|
|
10825
|
+
nonIndexablePages,
|
|
10826
|
+
statusErrors,
|
|
10691
10827
|
...runStats,
|
|
10692
10828
|
attemptedRequests: input.requestEvidenceStatus === "unavailable" ? 0 : requests.length,
|
|
10693
|
-
responseRequests
|
|
10694
|
-
failedRequests
|
|
10695
|
-
abortedRequests
|
|
10696
|
-
extractionFailures
|
|
10829
|
+
responseRequests,
|
|
10830
|
+
failedRequests,
|
|
10831
|
+
abortedRequests,
|
|
10832
|
+
extractionFailures,
|
|
10697
10833
|
requestByStatus,
|
|
10698
10834
|
avgRequestMs: requestMsCount ? Math.round(requestMs / requestMsCount) : void 0,
|
|
10699
|
-
highIssues
|
|
10700
|
-
mediumIssues
|
|
10701
|
-
lowIssues
|
|
10835
|
+
highIssues,
|
|
10836
|
+
mediumIssues,
|
|
10837
|
+
lowIssues,
|
|
10702
10838
|
avgResponseMs: responseCount ? Math.round(responseMs / responseCount) : void 0,
|
|
10703
10839
|
byStatus,
|
|
10704
10840
|
byCategory
|
|
@@ -10706,42 +10842,55 @@ function summarizeCrawlReport(input) {
|
|
|
10706
10842
|
}
|
|
10707
10843
|
const ACCESS_BLOCK_SAMPLE_LIMIT = 10;
|
|
10708
10844
|
function crawlAccessSummary(requests) {
|
|
10709
|
-
const blocked = requests.flatMap((request) => request.outcome === "response" && request.accessBlock ? [{
|
|
10710
|
-
url: request.requestedUrl,
|
|
10711
|
-
evidence: request.accessBlock
|
|
10712
|
-
}] : []);
|
|
10713
10845
|
const providers = {};
|
|
10714
|
-
|
|
10846
|
+
const samples = [];
|
|
10847
|
+
let blockedRequests = 0;
|
|
10848
|
+
for (const request of requests) {
|
|
10849
|
+
if (request.outcome !== "response" || !request.accessBlock) continue;
|
|
10850
|
+
blockedRequests += 1;
|
|
10851
|
+
providers[request.accessBlock.provider] = (providers[request.accessBlock.provider] ?? 0) + 1;
|
|
10852
|
+
if (samples.length < ACCESS_BLOCK_SAMPLE_LIMIT) samples.push({
|
|
10853
|
+
url: request.requestedUrl,
|
|
10854
|
+
evidence: request.accessBlock
|
|
10855
|
+
});
|
|
10856
|
+
}
|
|
10715
10857
|
return {
|
|
10716
10858
|
crawler: SEO_CRAWLER_IDENTITY,
|
|
10717
|
-
blockedRequests
|
|
10859
|
+
blockedRequests,
|
|
10718
10860
|
providers,
|
|
10719
|
-
samples
|
|
10861
|
+
samples,
|
|
10720
10862
|
sampleLimit: ACCESS_BLOCK_SAMPLE_LIMIT,
|
|
10721
|
-
truncated:
|
|
10863
|
+
truncated: blockedRequests > ACCESS_BLOCK_SAMPLE_LIMIT
|
|
10722
10864
|
};
|
|
10723
10865
|
}
|
|
10724
10866
|
function crawlRunStats(pages, stats = {}) {
|
|
10725
|
-
const
|
|
10867
|
+
const needsDiscoveredUrls = stats.discoveredUrls === void 0;
|
|
10868
|
+
const needsObservedInternalLinks = stats.observedInternalLinks === void 0;
|
|
10869
|
+
const needsFailedUrls = stats.failedUrls === void 0;
|
|
10870
|
+
const discovered = needsDiscoveredUrls ? /* @__PURE__ */ new Set() : void 0;
|
|
10726
10871
|
let observedInternalLinks = 0;
|
|
10727
|
-
|
|
10728
|
-
|
|
10729
|
-
|
|
10730
|
-
|
|
10731
|
-
|
|
10872
|
+
let failedUrls = 0;
|
|
10873
|
+
if (needsDiscoveredUrls || needsObservedInternalLinks || needsFailedUrls) for (const page of pages) {
|
|
10874
|
+
if (discovered) {
|
|
10875
|
+
discovered.add(page.url);
|
|
10876
|
+
for (const url of page.sampleInternalLinks ?? []) discovered.add(url);
|
|
10877
|
+
for (const url of page.sampleExternalLinks ?? []) discovered.add(url);
|
|
10878
|
+
}
|
|
10879
|
+
if (needsObservedInternalLinks) observedInternalLinks += page.outgoingInternalCount;
|
|
10880
|
+
if (needsFailedUrls && (page.status === 0 || page.status >= 400)) failedUrls += 1;
|
|
10732
10881
|
}
|
|
10733
10882
|
const skips = normalizeCrawlSkipReasonCounts({
|
|
10734
10883
|
skippedUrls: stats.skippedUrls,
|
|
10735
10884
|
skipReasons: stats.skipReasons ?? crawlSkipReasonCountsFromRecord(stats.skipReasonCounts)
|
|
10736
10885
|
});
|
|
10737
10886
|
return {
|
|
10738
|
-
discoveredUrls: stats.discoveredUrls ?? discovered
|
|
10887
|
+
discoveredUrls: stats.discoveredUrls ?? discovered?.size ?? 0,
|
|
10739
10888
|
queuedUrls: stats.queuedUrls ?? pages.length,
|
|
10740
10889
|
crawledUrls: stats.crawledUrls ?? pages.length,
|
|
10741
10890
|
skippedUrls: skips.skippedUrls,
|
|
10742
10891
|
skipReasons: skips.skipReasons,
|
|
10743
10892
|
skippedUrlsByImpact: skips.skippedUrlsByImpact,
|
|
10744
|
-
failedUrls: stats.failedUrls ??
|
|
10893
|
+
failedUrls: stats.failedUrls ?? failedUrls,
|
|
10745
10894
|
observedInternalLinks: stats.observedInternalLinks ?? observedInternalLinks,
|
|
10746
10895
|
pageLimitReached: stats.pageLimitReached ?? false
|
|
10747
10896
|
};
|
|
@@ -10758,11 +10907,14 @@ function normalizeLinkUrl(value) {
|
|
|
10758
10907
|
function deriveInternalLinkAuthority(pages, linkGraph) {
|
|
10759
10908
|
const hasLinkGraph = linkGraph && Object.keys(linkGraph).length > 0;
|
|
10760
10909
|
const hasStoredScores = pages.some((page) => page.internalInlinkCount !== void 0 || page.internalLinkAuthorityScore !== void 0);
|
|
10761
|
-
if (!hasLinkGraph && hasStoredScores)
|
|
10762
|
-
|
|
10763
|
-
|
|
10764
|
-
|
|
10765
|
-
|
|
10910
|
+
if (!hasLinkGraph && hasStoredScores) {
|
|
10911
|
+
if (pages.every((page) => page.internalInlinkCount !== void 0 && page.internalLinkAuthorityScore !== void 0)) return pages;
|
|
10912
|
+
return pages.map((page) => page.internalInlinkCount !== void 0 && page.internalLinkAuthorityScore !== void 0 ? page : {
|
|
10913
|
+
...page,
|
|
10914
|
+
internalInlinkCount: page.internalInlinkCount ?? 0,
|
|
10915
|
+
internalLinkAuthorityScore: page.internalLinkAuthorityScore ?? 0
|
|
10916
|
+
});
|
|
10917
|
+
}
|
|
10766
10918
|
const pageByUrl = /* @__PURE__ */ new Map();
|
|
10767
10919
|
for (const page of pages) for (const value of [page.url, page.finalUrl]) {
|
|
10768
10920
|
const normalized = normalizeLinkUrl(value);
|
|
@@ -10781,7 +10933,8 @@ function deriveInternalLinkAuthority(pages, linkGraph) {
|
|
|
10781
10933
|
}
|
|
10782
10934
|
for (const linkedPage of linkedPages) inlinks.set(linkedPage, (inlinks.get(linkedPage) ?? 0) + 1);
|
|
10783
10935
|
}
|
|
10784
|
-
|
|
10936
|
+
let maxInlinks = 0;
|
|
10937
|
+
for (const count of inlinks.values()) maxInlinks = Math.max(maxInlinks, count);
|
|
10785
10938
|
return pages.map((page) => {
|
|
10786
10939
|
const internalInlinkCount = inlinks.get(page.url) ?? 0;
|
|
10787
10940
|
return {
|
|
@@ -11030,6 +11183,7 @@ function crawlCaveats(input) {
|
|
|
11030
11183
|
const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
|
|
11031
11184
|
if (input.queueSafetySkippedUrls > 0) caveats.push(`Left ${input.queueSafetySkippedUrls} eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
|
|
11032
11185
|
if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
|
|
11186
|
+
if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
|
|
11033
11187
|
return caveats;
|
|
11034
11188
|
}
|
|
11035
11189
|
//#endregion
|
|
@@ -11056,6 +11210,26 @@ function cancellationRace(signal) {
|
|
|
11056
11210
|
});
|
|
11057
11211
|
}
|
|
11058
11212
|
//#endregion
|
|
11213
|
+
//#region packages/core/dist/analyze/crawler/crawl-memory.js
|
|
11214
|
+
const MEBIBYTE$1 = 1024 * 1024;
|
|
11215
|
+
const MIN_CRAWL_MEMORY_LIMIT = 384 * MEBIBYTE$1;
|
|
11216
|
+
const MAX_CRAWL_MEMORY_LIMIT = 640 * MEBIBYTE$1;
|
|
11217
|
+
const MIN_CRAWL_RSS_LIMIT = 512 * MEBIBYTE$1;
|
|
11218
|
+
const MAX_CRAWL_RSS_LIMIT = 896 * MEBIBYTE$1;
|
|
11219
|
+
const SYSTEM_MEMORY_FRACTION = .1;
|
|
11220
|
+
const SYSTEM_RSS_FRACTION = .2;
|
|
11221
|
+
function crawlMemoryLimitBytes(totalMemoryBytes) {
|
|
11222
|
+
if (!Number.isFinite(totalMemoryBytes) || totalMemoryBytes <= 0) return MAX_CRAWL_MEMORY_LIMIT;
|
|
11223
|
+
return Math.min(MAX_CRAWL_MEMORY_LIMIT, Math.max(MIN_CRAWL_MEMORY_LIMIT, Math.floor(totalMemoryBytes * SYSTEM_MEMORY_FRACTION)));
|
|
11224
|
+
}
|
|
11225
|
+
function crawlRssLimitBytes(totalMemoryBytes) {
|
|
11226
|
+
if (!Number.isFinite(totalMemoryBytes) || totalMemoryBytes <= 0) return MAX_CRAWL_RSS_LIMIT;
|
|
11227
|
+
return Math.min(MAX_CRAWL_RSS_LIMIT, Math.max(MIN_CRAWL_RSS_LIMIT, Math.floor(totalMemoryBytes * SYSTEM_RSS_FRACTION)));
|
|
11228
|
+
}
|
|
11229
|
+
function crawlMemoryPressure(input) {
|
|
11230
|
+
return input.memoryUsage.heapUsed + input.memoryUsage.external >= crawlMemoryLimitBytes(input.totalMemoryBytes) || input.memoryUsage.rss >= crawlRssLimitBytes(input.totalMemoryBytes);
|
|
11231
|
+
}
|
|
11232
|
+
//#endregion
|
|
11059
11233
|
//#region packages/core/dist/analyze/crawler/crawl-url-queue.js
|
|
11060
11234
|
function before(left, right) {
|
|
11061
11235
|
return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
|
|
@@ -11773,6 +11947,7 @@ function crawlProviderLimits(input) {
|
|
|
11773
11947
|
}
|
|
11774
11948
|
async function crawlDataSources(input) {
|
|
11775
11949
|
const window = crawlMetricsWindow(input.now());
|
|
11950
|
+
const skippedForMemory = input.skipReason === "memory-pressure";
|
|
11776
11951
|
const dataSources = {
|
|
11777
11952
|
searchConsole: {
|
|
11778
11953
|
status: "skipped",
|
|
@@ -11783,7 +11958,7 @@ async function crawlDataSources(input) {
|
|
|
11783
11958
|
pageLimit: input.searchMetricsLimit,
|
|
11784
11959
|
pageLimitReached: false,
|
|
11785
11960
|
retainedRowLimitReached: false,
|
|
11786
|
-
warning: input.cancelled ? "Search Console join skipped because the crawl was cancelled." : "Search Console join skipped because no property was selected."
|
|
11961
|
+
warning: skippedForMemory ? "Search Console join skipped after the local memory safety limit was reached." : input.skipReason === "cancelled" ? "Search Console join skipped because the crawl was cancelled." : "Search Console join skipped because no property was selected."
|
|
11787
11962
|
},
|
|
11788
11963
|
analytics: {
|
|
11789
11964
|
status: "skipped",
|
|
@@ -11792,10 +11967,10 @@ async function crawlDataSources(input) {
|
|
|
11792
11967
|
joinedPages: 0,
|
|
11793
11968
|
retainedRowLimit: input.analyticsLimit,
|
|
11794
11969
|
retainedRowLimitReached: false,
|
|
11795
|
-
warning: input.cancelled ? "Google Analytics join skipped because the crawl was cancelled." : "Google Analytics join skipped because no property was selected."
|
|
11970
|
+
warning: skippedForMemory ? "Google Analytics join skipped after the local memory safety limit was reached." : input.skipReason === "cancelled" ? "Google Analytics join skipped because the crawl was cancelled." : "Google Analytics join skipped because no property was selected."
|
|
11796
11971
|
}
|
|
11797
11972
|
};
|
|
11798
|
-
if (!input.
|
|
11973
|
+
if (!input.skipReason && input.site) dataSources.searchConsole = await joinSearchMetrics({
|
|
11799
11974
|
site: input.site,
|
|
11800
11975
|
pages: input.pages,
|
|
11801
11976
|
warnings: input.warnings,
|
|
@@ -11808,7 +11983,7 @@ async function crawlDataSources(input) {
|
|
|
11808
11983
|
queryPagesMetricsBatch: input.queryPagesMetricsBatch,
|
|
11809
11984
|
queryPagesTopQueriesBatch: input.queryPagesTopQueriesBatch
|
|
11810
11985
|
});
|
|
11811
|
-
if (!input.
|
|
11986
|
+
if (!input.skipReason && input.googleAnalyticsPropertyId) dataSources.analytics = await joinAnalytics({
|
|
11812
11987
|
propertyId: input.googleAnalyticsPropertyId,
|
|
11813
11988
|
pages: input.pages,
|
|
11814
11989
|
warnings: input.warnings,
|
|
@@ -11821,17 +11996,6 @@ async function crawlDataSources(input) {
|
|
|
11821
11996
|
}
|
|
11822
11997
|
//#endregion
|
|
11823
11998
|
//#region packages/core/dist/analyze/monitoring/crawl-page.js
|
|
11824
|
-
function sameOriginUrl(href, base) {
|
|
11825
|
-
try {
|
|
11826
|
-
const url = new URL(href, base);
|
|
11827
|
-
url.hash = "";
|
|
11828
|
-
if (url.origin !== base.origin) return void 0;
|
|
11829
|
-
if (!["http:", "https:"].includes(url.protocol)) return void 0;
|
|
11830
|
-
return url.toString();
|
|
11831
|
-
} catch {
|
|
11832
|
-
return;
|
|
11833
|
-
}
|
|
11834
|
-
}
|
|
11835
11999
|
function resolvedUrl(href, base) {
|
|
11836
12000
|
if (!href) return void 0;
|
|
11837
12001
|
try {
|
|
@@ -11861,49 +12025,6 @@ function safeResponseHeaders(headers) {
|
|
|
11861
12025
|
function truncate$1(value, max) {
|
|
11862
12026
|
return value.length > max ? value.slice(0, max).trimEnd() : value;
|
|
11863
12027
|
}
|
|
11864
|
-
function anchorSamples(links, internal) {
|
|
11865
|
-
const seen = /* @__PURE__ */ new Set();
|
|
11866
|
-
const samples = [];
|
|
11867
|
-
for (const link of links) {
|
|
11868
|
-
if (link.internal !== internal) continue;
|
|
11869
|
-
const text = truncate$1(link.text.replace(/\s+/g, " ").trim(), 120);
|
|
11870
|
-
if (!text) continue;
|
|
11871
|
-
const key = `${link.href}\n${text}`;
|
|
11872
|
-
if (seen.has(key)) continue;
|
|
11873
|
-
seen.add(key);
|
|
11874
|
-
samples.push({
|
|
11875
|
-
href: link.href,
|
|
11876
|
-
text
|
|
11877
|
-
});
|
|
11878
|
-
if (samples.length >= 25) break;
|
|
11879
|
-
}
|
|
11880
|
-
return samples;
|
|
11881
|
-
}
|
|
11882
|
-
function socialProfileLinks(links) {
|
|
11883
|
-
const hosts = [
|
|
11884
|
-
"facebook.com",
|
|
11885
|
-
"instagram.com",
|
|
11886
|
-
"linkedin.com",
|
|
11887
|
-
"pinterest.com",
|
|
11888
|
-
"tiktok.com",
|
|
11889
|
-
"twitter.com",
|
|
11890
|
-
"x.com",
|
|
11891
|
-
"youtube.com"
|
|
11892
|
-
];
|
|
11893
|
-
const profiles = /* @__PURE__ */ new Set();
|
|
11894
|
-
for (const link of links) {
|
|
11895
|
-
if (link.internal) continue;
|
|
11896
|
-
try {
|
|
11897
|
-
const url = new URL(link.href);
|
|
11898
|
-
const host = url.hostname.replace(/^www\./, "").toLowerCase();
|
|
11899
|
-
if (hosts.some((value) => host === value || host.endsWith(`.${value}`))) {
|
|
11900
|
-
url.hash = "";
|
|
11901
|
-
profiles.add(url.toString());
|
|
11902
|
-
}
|
|
11903
|
-
} catch {}
|
|
11904
|
-
}
|
|
11905
|
-
return [...profiles].slice(0, 50);
|
|
11906
|
-
}
|
|
11907
12028
|
function sameDocumentUrl(left, right) {
|
|
11908
12029
|
if (!left || !right) return false;
|
|
11909
12030
|
try {
|
|
@@ -12003,17 +12124,9 @@ function statusOnlyPage(fetched) {
|
|
|
12003
12124
|
finalUrl: fetched.finalUrl,
|
|
12004
12125
|
status: fetched.status,
|
|
12005
12126
|
contentType,
|
|
12006
|
-
responseHeaders: safeResponseHeaders(fetched.headers),
|
|
12007
12127
|
responseTimeMs: fetched.diagnostics.durationMs,
|
|
12008
|
-
usedJs: false,
|
|
12009
|
-
fetchSource: "network",
|
|
12010
|
-
cacheState: "bypass",
|
|
12011
|
-
fetchDiagnostics: fetched.diagnostics,
|
|
12012
12128
|
auditScope: "status",
|
|
12013
|
-
accessBlock: fetched.diagnostics.accessBlock,
|
|
12014
|
-
blocked: fetched.diagnostics.blocked,
|
|
12015
12129
|
contentAuditAllowed: false,
|
|
12016
|
-
robotsTxt: fetched.robotsTxt,
|
|
12017
12130
|
indexable: false,
|
|
12018
12131
|
indexability: "Not evaluated by status-only health probe",
|
|
12019
12132
|
declaredIndexability: "unknown",
|
|
@@ -12021,8 +12134,7 @@ function statusOnlyPage(fetched) {
|
|
|
12021
12134
|
wordCount: 0,
|
|
12022
12135
|
contentHash: "",
|
|
12023
12136
|
outgoingInternalCount: 0,
|
|
12024
|
-
outgoingExternalCount: 0
|
|
12025
|
-
isHttps: new URL(fetched.finalUrl).protocol === "https:"
|
|
12137
|
+
outgoingExternalCount: 0
|
|
12026
12138
|
};
|
|
12027
12139
|
}
|
|
12028
12140
|
function indexabilityReason(page) {
|
|
@@ -12098,12 +12210,11 @@ async function crawlOne(url, opts = {}) {
|
|
|
12098
12210
|
};
|
|
12099
12211
|
try {
|
|
12100
12212
|
const extracted = await extractPage(fetched, "crawler");
|
|
12101
|
-
const
|
|
12102
|
-
|
|
12103
|
-
const
|
|
12104
|
-
const
|
|
12105
|
-
const
|
|
12106
|
-
const uniqueExternalLinks = [...new Set(externalLinks)];
|
|
12213
|
+
const crawlerEvidence = extracted.crawlerEvidence;
|
|
12214
|
+
if (!crawlerEvidence) throw new Error("Crawler extraction did not return crawl evidence.");
|
|
12215
|
+
const h1 = crawlerEvidence.h1;
|
|
12216
|
+
const uniqueInternalLinks = crawlerEvidence.internalLinks;
|
|
12217
|
+
const uniqueExternalLinks = crawlerEvidence.externalLinks;
|
|
12107
12218
|
const canonical = extracted.canonicalEvidence?.selectedUrl ?? resolvedUrl(extracted.canonical, extracted.finalUrl);
|
|
12108
12219
|
const richResults = selectGoogleRichResultAssessments(extracted.googleRichResults ?? []);
|
|
12109
12220
|
const page = {
|
|
@@ -12129,9 +12240,9 @@ async function crawlOne(url, opts = {}) {
|
|
|
12129
12240
|
metaRobots: extracted.metaRobots,
|
|
12130
12241
|
xRobotsTag: extracted.xRobotsTag,
|
|
12131
12242
|
h1,
|
|
12132
|
-
h1Count:
|
|
12133
|
-
h2Count:
|
|
12134
|
-
h3Count:
|
|
12243
|
+
h1Count: crawlerEvidence.h1Count,
|
|
12244
|
+
h2Count: crawlerEvidence.h2Count,
|
|
12245
|
+
h3Count: crawlerEvidence.h3Count,
|
|
12135
12246
|
indexable: false,
|
|
12136
12247
|
extractionStatus: "complete",
|
|
12137
12248
|
wordCount: extracted.wordCount,
|
|
@@ -12167,8 +12278,8 @@ async function crawlOne(url, opts = {}) {
|
|
|
12167
12278
|
outgoingExternalCount: uniqueExternalLinks.length,
|
|
12168
12279
|
sampleInternalLinks: uniqueInternalLinks.slice(0, 25),
|
|
12169
12280
|
sampleExternalLinks: uniqueExternalLinks.slice(0, 25),
|
|
12170
|
-
internalAnchorSamples:
|
|
12171
|
-
externalAnchorSamples:
|
|
12281
|
+
internalAnchorSamples: crawlerEvidence.internalAnchorSamples,
|
|
12282
|
+
externalAnchorSamples: crawlerEvidence.externalAnchorSamples,
|
|
12172
12283
|
markdownAlternates: extracted.markdownAlternates,
|
|
12173
12284
|
schemaTypes: extracted.schemaTypes,
|
|
12174
12285
|
structuredDataFormats: extracted.structuredDataFormats ?? [],
|
|
@@ -12177,7 +12288,7 @@ async function crawlOne(url, opts = {}) {
|
|
|
12177
12288
|
schemaSameAs: [...new Set((extracted.schemaSameAsEvidence ?? []).map((evidence) => evidence.url))].slice(0, 50),
|
|
12178
12289
|
schemaSameAsEvidence: (extracted.schemaSameAsEvidence ?? []).slice(0, 50),
|
|
12179
12290
|
invalidSchemaSameAs: (extracted.invalidSchemaSameAs ?? []).slice(0, 25),
|
|
12180
|
-
socialProfileLinks: socialProfileLinks
|
|
12291
|
+
socialProfileLinks: crawlerEvidence.socialProfileLinks,
|
|
12181
12292
|
invalidJsonLdCount: extracted.invalidJsonLdCount,
|
|
12182
12293
|
invalidJsonLdSamples: extracted.invalidJsonLdSamples,
|
|
12183
12294
|
unrecognizedJsonLdTypes: extracted.unrecognizedJsonLdTypes ?? [],
|
|
@@ -12830,7 +12941,9 @@ function resolveCrawlSiteDependencies(dependencies = {}) {
|
|
|
12830
12941
|
fetchLandingPageValues: dependencies.fetchLandingPageValues ?? fetchLandingPageValues,
|
|
12831
12942
|
landingValueForUrl: dependencies.landingValueForUrl ?? landingValueForUrl,
|
|
12832
12943
|
collectAgentDiscovery: dependencies.collectAgentDiscovery ?? collectAgentDiscovery,
|
|
12833
|
-
now: dependencies.now ?? (() => /* @__PURE__ */ new Date())
|
|
12944
|
+
now: dependencies.now ?? (() => /* @__PURE__ */ new Date()),
|
|
12945
|
+
memoryUsage: dependencies.memoryUsage ?? (() => process.memoryUsage()),
|
|
12946
|
+
totalMemory: dependencies.totalMemory ?? totalmem
|
|
12834
12947
|
};
|
|
12835
12948
|
}
|
|
12836
12949
|
//#endregion
|
|
@@ -12986,15 +13099,26 @@ async function crawlSite(input, dependencies) {
|
|
|
12986
13099
|
let queuedUrls = 0;
|
|
12987
13100
|
let skippedUrls = 0;
|
|
12988
13101
|
const skipReasonCounts = {};
|
|
12989
|
-
const
|
|
12990
|
-
|
|
12991
|
-
|
|
13102
|
+
const recordSkips = (reason, count = 1) => {
|
|
13103
|
+
if (count <= 0) return;
|
|
13104
|
+
skippedUrls += count;
|
|
13105
|
+
skipReasonCounts[reason] = (skipReasonCounts[reason] ?? 0) + count;
|
|
12992
13106
|
};
|
|
13107
|
+
const recordSkip = (reason) => recordSkips(reason);
|
|
12993
13108
|
let robotsDeferredUrls = 0;
|
|
12994
13109
|
let originBackpressureSkippedUrls = 0;
|
|
12995
13110
|
let queueSafetySkippedUrls = 0;
|
|
12996
13111
|
let failedUrls = 0;
|
|
12997
13112
|
let observedInternalLinks = 0;
|
|
13113
|
+
let memoryLimited = false;
|
|
13114
|
+
const memoryDeferredUrls = /* @__PURE__ */ new Set();
|
|
13115
|
+
const checkMemoryPressure = () => {
|
|
13116
|
+
if (memoryLimited) return;
|
|
13117
|
+
memoryLimited = crawlMemoryPressure({
|
|
13118
|
+
memoryUsage: deps.memoryUsage(),
|
|
13119
|
+
totalMemoryBytes: deps.totalMemory()
|
|
13120
|
+
});
|
|
13121
|
+
};
|
|
12998
13122
|
let sitemapDiscovery;
|
|
12999
13123
|
let externalLinkVerification;
|
|
13000
13124
|
let statusEventChain = Promise.resolve();
|
|
@@ -13124,6 +13248,7 @@ async function crawlSite(input, dependencies) {
|
|
|
13124
13248
|
for (const url of sitemap.urls) enqueue(url, 0);
|
|
13125
13249
|
sitemapDiscovery = sitemap.discovery;
|
|
13126
13250
|
}
|
|
13251
|
+
checkMemoryPressure();
|
|
13127
13252
|
const nextQueueItem = () => {
|
|
13128
13253
|
while (queue.size) {
|
|
13129
13254
|
const item = queue.take();
|
|
@@ -13149,6 +13274,7 @@ async function crawlSite(input, dependencies) {
|
|
|
13149
13274
|
timeoutMs: config.timeoutMs,
|
|
13150
13275
|
rate: config.fetchRate,
|
|
13151
13276
|
signal,
|
|
13277
|
+
writeCache: false,
|
|
13152
13278
|
respectRobots: config.respectRobots,
|
|
13153
13279
|
robotsResolver,
|
|
13154
13280
|
renderer
|
|
@@ -13176,8 +13302,8 @@ async function crawlSite(input, dependencies) {
|
|
|
13176
13302
|
depth: item.depth
|
|
13177
13303
|
});
|
|
13178
13304
|
};
|
|
13179
|
-
while (!isCancelled() && (queue.size || inFlight.size) && pages.length < config.maxPages) {
|
|
13180
|
-
while (!isCancelled() && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
|
|
13305
|
+
while (!isCancelled() && (queue.size || inFlight.size) && (!memoryLimited || inFlight.size > 0) && pages.length < config.maxPages) {
|
|
13306
|
+
while (!isCancelled() && !memoryLimited && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
|
|
13181
13307
|
const item = nextQueueItem();
|
|
13182
13308
|
if (!item) break;
|
|
13183
13309
|
startTask(item);
|
|
@@ -13205,6 +13331,7 @@ async function crawlSite(input, dependencies) {
|
|
|
13205
13331
|
healthyProbeStreak = 0;
|
|
13206
13332
|
activeConcurrency = 1;
|
|
13207
13333
|
}
|
|
13334
|
+
checkMemoryPressure();
|
|
13208
13335
|
if (request.outcome === "skipped") {
|
|
13209
13336
|
recordSkip(request.reason === "robots-disallowed" ? "robots-disallowed" : request.reason === "robots-deferred" ? "robots-uncertain" : "origin-backpressure");
|
|
13210
13337
|
if (request.reason === "robots-deferred") {
|
|
@@ -13335,10 +13462,24 @@ async function crawlSite(input, dependencies) {
|
|
|
13335
13462
|
depth: task.depth,
|
|
13336
13463
|
statusCode: page.status
|
|
13337
13464
|
});
|
|
13338
|
-
if (followLinks && task.depth < config.maxDepth) for (const
|
|
13465
|
+
if (followLinks && task.depth < config.maxDepth) if (memoryLimited) for (const value of result.urls) {
|
|
13466
|
+
const normalized = normalizeUrl$1(value, finalUrl);
|
|
13467
|
+
if (normalized && sameOrigin(normalized, origin) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
|
|
13468
|
+
discovered.add(normalized);
|
|
13469
|
+
memoryDeferredUrls.add(normalized);
|
|
13470
|
+
}
|
|
13471
|
+
}
|
|
13472
|
+
else for (const url of result.urls) enqueue(url, task.depth + 1);
|
|
13473
|
+
}
|
|
13474
|
+
const memoryPressureSkippedUrls = memoryLimited ? queue.size + memoryDeferredUrls.size : 0;
|
|
13475
|
+
const stoppedForMemory = memoryPressureSkippedUrls > 0;
|
|
13476
|
+
if (stoppedForMemory) {
|
|
13477
|
+
recordSkips("memory-pressure", memoryPressureSkippedUrls);
|
|
13478
|
+
warnings.push("Crawl stopped before local memory use could become unsafe. The retained report is partial.");
|
|
13339
13479
|
}
|
|
13340
13480
|
const resolvedInlinkCounts = resolveLinkCountAliases(incomingLinkCounts, fetchedAliases);
|
|
13341
|
-
|
|
13481
|
+
let maxInlinks = 0;
|
|
13482
|
+
for (const page of pages) maxInlinks = Math.max(maxInlinks, resolvedInlinkCounts.get(page.finalUrl) ?? 0);
|
|
13342
13483
|
for (const page of pages) {
|
|
13343
13484
|
const internalInlinkCount = resolvedInlinkCounts.get(page.finalUrl) ?? 0;
|
|
13344
13485
|
page.internalInlinkCount = internalInlinkCount;
|
|
@@ -13358,11 +13499,11 @@ async function crawlSite(input, dependencies) {
|
|
|
13358
13499
|
const sitemapEvidencePartial = sitemapDiscovery?.dataStatus === "partial";
|
|
13359
13500
|
const accessEvidencePartial = requests.some((request) => request.outcome === "response" && request.accessBlock);
|
|
13360
13501
|
const requiredSitemapUnavailable = config.mode === "sitemap" && sitemapDiscovery?.dataStatus === "unavailable";
|
|
13361
|
-
const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
|
|
13502
|
+
const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
|
|
13362
13503
|
const failed = !cancelled && pages.length === 0 && (requiredSitemapUnavailable || requests.length > 0 && requests.every((request) => request.outcome === "failure"));
|
|
13363
13504
|
const requestEvidenceStatus = cancelled && inFlight.size > 0 ? "partial" : "available";
|
|
13364
13505
|
const dataSources = await crawlDataSources({
|
|
13365
|
-
cancelled,
|
|
13506
|
+
skipReason: cancelled ? "cancelled" : stoppedForMemory ? "memory-pressure" : void 0,
|
|
13366
13507
|
site: healthStrategy ? void 0 : site,
|
|
13367
13508
|
googleAnalyticsPropertyId: healthStrategy ? void 0 : input.googleAnalyticsPropertyId,
|
|
13368
13509
|
pages,
|
|
@@ -13381,7 +13522,7 @@ async function crawlSite(input, dependencies) {
|
|
|
13381
13522
|
});
|
|
13382
13523
|
const sourceEvidencePartial = [dataSources.searchConsole.status, dataSources.analytics.status].some((status) => status === "partial" || status === "unavailable");
|
|
13383
13524
|
const reportStatus = failed ? "failed" : partial || sourceEvidencePartial ? "partial" : "completed";
|
|
13384
|
-
if (!cancelled && config.checkExternal) {
|
|
13525
|
+
if (!cancelled && !stoppedForMemory && config.checkExternal) {
|
|
13385
13526
|
emitStatus("external_links_started", {
|
|
13386
13527
|
url: config.url,
|
|
13387
13528
|
message: "Started external link checks."
|
|
@@ -13398,7 +13539,7 @@ async function crawlSite(input, dependencies) {
|
|
|
13398
13539
|
message: "Finished external link checks."
|
|
13399
13540
|
});
|
|
13400
13541
|
}
|
|
13401
|
-
const agentDiscovery = !cancelled && config.checkAgentDiscovery ? await deps.collectAgentDiscovery({
|
|
13542
|
+
const agentDiscovery = !cancelled && !stoppedForMemory && config.checkAgentDiscovery ? await deps.collectAgentDiscovery({
|
|
13402
13543
|
startUrl: config.url,
|
|
13403
13544
|
pages,
|
|
13404
13545
|
timeoutMs: config.timeoutMs,
|
|
@@ -13439,7 +13580,8 @@ async function crawlSite(input, dependencies) {
|
|
|
13439
13580
|
pageLimitReached,
|
|
13440
13581
|
maxPages: config.maxPages,
|
|
13441
13582
|
queueSafetySkippedUrls,
|
|
13442
|
-
originBackpressureSkippedUrls
|
|
13583
|
+
originBackpressureSkippedUrls,
|
|
13584
|
+
memoryPressureSkippedUrls
|
|
13443
13585
|
}),
|
|
13444
13586
|
stats: {
|
|
13445
13587
|
discoveredUrls: discovered.size,
|
|
@@ -15440,7 +15582,7 @@ function compareText$10(left, right) {
|
|
|
15440
15582
|
}
|
|
15441
15583
|
return leftPoints.length - rightPoints.length;
|
|
15442
15584
|
}
|
|
15443
|
-
function boundedInteger$
|
|
15585
|
+
function boundedInteger$6(value, fallback, minimum, maximum) {
|
|
15444
15586
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
15445
15587
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
15446
15588
|
}
|
|
@@ -15517,8 +15659,8 @@ function initialSelection(sourceRows) {
|
|
|
15517
15659
|
};
|
|
15518
15660
|
}
|
|
15519
15661
|
function analyzeCtrUnderperformersFromRows(input) {
|
|
15520
|
-
const minImpressions = boundedInteger$
|
|
15521
|
-
const limit = boundedInteger$
|
|
15662
|
+
const minImpressions = boundedInteger$6(input.minImpressions, 200, 1, CTR_MAX_MIN_IMPRESSIONS);
|
|
15663
|
+
const limit = boundedInteger$6(input.limit, 25, 1, 100);
|
|
15522
15664
|
const selection = initialSelection(input.rows.length);
|
|
15523
15665
|
const benchmarkRows = aggregateCtrRows(input.rows, selection).filter((row) => {
|
|
15524
15666
|
const query = row.keys[0] ?? "";
|
|
@@ -16017,7 +16159,7 @@ function analyzeSegmentRows(input) {
|
|
|
16017
16159
|
}
|
|
16018
16160
|
//#endregion
|
|
16019
16161
|
//#region packages/core/dist/analyze/segment-impact-input.js
|
|
16020
|
-
const DAY_MS$
|
|
16162
|
+
const DAY_MS$2 = 864e5;
|
|
16021
16163
|
const MAX_WINDOW_DAYS = 240;
|
|
16022
16164
|
function subtractDays$1(value, days) {
|
|
16023
16165
|
const date = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -16025,7 +16167,7 @@ function subtractDays$1(value, days) {
|
|
|
16025
16167
|
return date.toISOString().slice(0, 10);
|
|
16026
16168
|
}
|
|
16027
16169
|
function segmentRangeDays(range) {
|
|
16028
|
-
return Math.floor((Date.parse(`${range.endDate}T00:00:00.000Z`) - Date.parse(`${range.startDate}T00:00:00.000Z`)) / DAY_MS$
|
|
16170
|
+
return Math.floor((Date.parse(`${range.endDate}T00:00:00.000Z`) - Date.parse(`${range.startDate}T00:00:00.000Z`)) / DAY_MS$2) + 1;
|
|
16029
16171
|
}
|
|
16030
16172
|
function segmentComparisonRange(after, compareDays) {
|
|
16031
16173
|
const endDate = subtractDays$1(after.startDate, 1);
|
|
@@ -16605,13 +16747,13 @@ const MINIMUM_PAGE_SHARE = .1;
|
|
|
16605
16747
|
const MINIMUM_PAGE_IMPRESSIONS = 10;
|
|
16606
16748
|
const MAXIMUM_DOMINANT_SHARE = .8;
|
|
16607
16749
|
const MAX_SUPPRESSIONS = 100;
|
|
16608
|
-
function boundedInteger$
|
|
16750
|
+
function boundedInteger$5(value, fallback, minimum, maximum) {
|
|
16609
16751
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
16610
16752
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
16611
16753
|
}
|
|
16612
16754
|
function analyzeCannibalRows(input) {
|
|
16613
|
-
const minImpressions = boundedInteger$
|
|
16614
|
-
const limit = boundedInteger$
|
|
16755
|
+
const minImpressions = boundedInteger$5(input.minImpressions, 50, 0, 1e9);
|
|
16756
|
+
const limit = boundedInteger$5(input.limit, 25, 1, 100);
|
|
16615
16757
|
const selection = {
|
|
16616
16758
|
sourceRows: input.rows.length,
|
|
16617
16759
|
invalidRows: 0,
|
|
@@ -17391,7 +17533,7 @@ function groupQuickWins(items) {
|
|
|
17391
17533
|
}
|
|
17392
17534
|
//#endregion
|
|
17393
17535
|
//#region packages/core/dist/analyze/site-diagnostics/quick-wins-analysis-primitives.js
|
|
17394
|
-
function boundedInteger$
|
|
17536
|
+
function boundedInteger$4(value, fallback, minimum, maximum) {
|
|
17395
17537
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
17396
17538
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
17397
17539
|
}
|
|
@@ -17512,12 +17654,12 @@ function roundedSum(values) {
|
|
|
17512
17654
|
//#region packages/core/dist/analyze/site-diagnostics/quick-wins-analysis.js
|
|
17513
17655
|
const DEFAULT_MIN_IMPRESSIONS$2 = 200;
|
|
17514
17656
|
const MAX_MIN_IMPRESSIONS$2 = 1e9;
|
|
17515
|
-
const DEFAULT_LIMIT$
|
|
17516
|
-
const MAX_LIMIT$
|
|
17657
|
+
const DEFAULT_LIMIT$3 = 25;
|
|
17658
|
+
const MAX_LIMIT$3 = 100;
|
|
17517
17659
|
const FALLBACK_SOURCE = "builtin_position_ctr_curve_v1";
|
|
17518
17660
|
function analyzeQuickWinsFromRows(input) {
|
|
17519
|
-
const minImpressions = boundedInteger$
|
|
17520
|
-
const limit = boundedInteger$
|
|
17661
|
+
const minImpressions = boundedInteger$4(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$2, 0, MAX_MIN_IMPRESSIONS$2);
|
|
17662
|
+
const limit = boundedInteger$4(input.limit, DEFAULT_LIMIT$3, 1, MAX_LIMIT$3);
|
|
17521
17663
|
const selected = selectBenchmarkRows(input);
|
|
17522
17664
|
const benchmarkContext = createCtrBenchmarkContext(selected.rows, {
|
|
17523
17665
|
samplePopulation: "all_qualified_url_samples",
|
|
@@ -17858,7 +18000,7 @@ function compareText$5(left, right) {
|
|
|
17858
18000
|
}
|
|
17859
18001
|
return leftPoints.length - rightPoints.length;
|
|
17860
18002
|
}
|
|
17861
|
-
function isHttpUrl(value) {
|
|
18003
|
+
function isHttpUrl$1(value) {
|
|
17862
18004
|
try {
|
|
17863
18005
|
const parsed = new URL(value);
|
|
17864
18006
|
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
@@ -17867,7 +18009,7 @@ function isHttpUrl(value) {
|
|
|
17867
18009
|
}
|
|
17868
18010
|
}
|
|
17869
18011
|
function isValidStrikingDistanceRow(row, query, url) {
|
|
17870
|
-
return Boolean(query) && isHttpUrl(url) && Number.isFinite(row.clicks) && Number.isFinite(row.impressions) && Number.isFinite(row.ctr) && Number.isFinite(row.position) && row.clicks >= 0 && row.impressions > 0 && row.clicks <= row.impressions && row.ctr >= 0 && row.ctr <= 1 && row.position > 0;
|
|
18012
|
+
return Boolean(query) && isHttpUrl$1(url) && Number.isFinite(row.clicks) && Number.isFinite(row.impressions) && Number.isFinite(row.ctr) && Number.isFinite(row.position) && row.clicks >= 0 && row.impressions > 0 && row.clicks <= row.impressions && row.ctr >= 0 && row.ctr <= 1 && row.position > 0;
|
|
17871
18013
|
}
|
|
17872
18014
|
function isStrikingDistanceBrandRow(input) {
|
|
17873
18015
|
return shouldExcludeBrandQuery({
|
|
@@ -17957,8 +18099,8 @@ function groupStrikingDistanceItems(items) {
|
|
|
17957
18099
|
//#region packages/core/dist/analyze/striking-distance-analysis.js
|
|
17958
18100
|
const DEFAULT_MIN_IMPRESSIONS$1 = 100;
|
|
17959
18101
|
const MAX_MIN_IMPRESSIONS$1 = 1e9;
|
|
17960
|
-
const DEFAULT_LIMIT$
|
|
17961
|
-
const MAX_LIMIT$
|
|
18102
|
+
const DEFAULT_LIMIT$2 = 25;
|
|
18103
|
+
const MAX_LIMIT$2 = 100;
|
|
17962
18104
|
function integerInRange$1(value, fallback, minimum, maximum) {
|
|
17963
18105
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
17964
18106
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
@@ -18000,7 +18142,7 @@ function selectRows(input) {
|
|
|
18000
18142
|
}
|
|
18001
18143
|
function analyzeStrikingDistanceRows(input) {
|
|
18002
18144
|
const minImpressions = integerInRange$1(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$1, 0, MAX_MIN_IMPRESSIONS$1);
|
|
18003
|
-
const limit = integerInRange$1(input.limit, DEFAULT_LIMIT$
|
|
18145
|
+
const limit = integerInRange$1(input.limit, DEFAULT_LIMIT$2, 1, MAX_LIMIT$2);
|
|
18004
18146
|
const selected = selectRows({
|
|
18005
18147
|
...input,
|
|
18006
18148
|
minImpressions
|
|
@@ -18074,7 +18216,7 @@ const defaultDependencies$7 = {
|
|
|
18074
18216
|
verifyContent: verifyQueryContent,
|
|
18075
18217
|
now: () => /* @__PURE__ */ new Date()
|
|
18076
18218
|
};
|
|
18077
|
-
function boundedInteger$
|
|
18219
|
+
function boundedInteger$3(value, fallback, minimum, maximum) {
|
|
18078
18220
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
18079
18221
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
18080
18222
|
}
|
|
@@ -18138,7 +18280,7 @@ async function strikingDistance(input, dependencies = defaultDependencies$7) {
|
|
|
18138
18280
|
brandTerms: input.brandTerms,
|
|
18139
18281
|
includeBrand: input.includeBrand
|
|
18140
18282
|
});
|
|
18141
|
-
const verifyLimit = boundedInteger$
|
|
18283
|
+
const verifyLimit = boundedInteger$3(input.verifyLimit, DEFAULT_VERIFY_LIMIT, 0, analysis.limit);
|
|
18142
18284
|
const verificationRequested = input.verifyContent === true || input.verifyContent !== false && input.verifyLimit !== void 0;
|
|
18143
18285
|
const attempted = verificationRequested ? Math.min(verifyLimit, analysis.items.length) : 0;
|
|
18144
18286
|
const items = await Promise.all(analysis.items.map(async (item, index) => {
|
|
@@ -18327,7 +18469,7 @@ function standardDeviation(values) {
|
|
|
18327
18469
|
function toFixedNumber(value, digits = 3) {
|
|
18328
18470
|
return Number(value.toFixed(digits));
|
|
18329
18471
|
}
|
|
18330
|
-
function percentChange(baseline, comparison) {
|
|
18472
|
+
function percentChange$1(baseline, comparison) {
|
|
18331
18473
|
if (!baseline) return comparison ? null : 0;
|
|
18332
18474
|
return (comparison - baseline) / baseline * 100;
|
|
18333
18475
|
}
|
|
@@ -18513,7 +18655,7 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
18513
18655
|
const outsideFlatBaseline = zScore === null && comparisonMean !== baselineMean;
|
|
18514
18656
|
const significant = outsideFlatBaseline || zScore !== null && Math.abs(zScore) >= 2;
|
|
18515
18657
|
const direction = significant ? comparisonMean < baselineMean ? "drop" : "spike" : "normal";
|
|
18516
|
-
const relativeChange = percentChange(baselineMean, comparisonMean);
|
|
18658
|
+
const relativeChange = percentChange$1(baselineMean, comparisonMean);
|
|
18517
18659
|
return {
|
|
18518
18660
|
site: input.site,
|
|
18519
18661
|
metric,
|
|
@@ -19572,7 +19714,7 @@ function dateShift(date, days) {
|
|
|
19572
19714
|
}
|
|
19573
19715
|
//#endregion
|
|
19574
19716
|
//#region packages/core/dist/analyze/experiments/measurement/window.js
|
|
19575
|
-
const DAY_MS = 864e5;
|
|
19717
|
+
const DAY_MS$1 = 864e5;
|
|
19576
19718
|
function exactDate(value, label) {
|
|
19577
19719
|
const pattern = /^\d{4}-\d{2}-\d{2}$/;
|
|
19578
19720
|
const parsed = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -19587,7 +19729,7 @@ function boundedDays(value, fallback, label) {
|
|
|
19587
19729
|
function calendarDays(startDate, endDate) {
|
|
19588
19730
|
const start = (/* @__PURE__ */ new Date(`${startDate}T00:00:00.000Z`)).getTime();
|
|
19589
19731
|
const end = (/* @__PURE__ */ new Date(`${endDate}T00:00:00.000Z`)).getTime();
|
|
19590
|
-
return Math.floor((end - start) / DAY_MS) + 1;
|
|
19732
|
+
return Math.floor((end - start) / DAY_MS$1) + 1;
|
|
19591
19733
|
}
|
|
19592
19734
|
function subtractCalendarMonths(value, months) {
|
|
19593
19735
|
const source = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -20133,7 +20275,7 @@ function internalLinksVerdict(input) {
|
|
|
20133
20275
|
function uniqueInternalLinksWarnings(warnings) {
|
|
20134
20276
|
return [...new Map(warnings.map((warning) => [`${warning.stage}\u0000${warning.url}\u0000${warning.code}\u0000${warning.message}`, warning])).values()];
|
|
20135
20277
|
}
|
|
20136
|
-
function errorMessage(error) {
|
|
20278
|
+
function errorMessage$1(error) {
|
|
20137
20279
|
return error instanceof Error ? error.message : String(error);
|
|
20138
20280
|
}
|
|
20139
20281
|
function acceptedAlias(site, value) {
|
|
@@ -20183,7 +20325,7 @@ async function verifyInternalLinkTarget(input) {
|
|
|
20183
20325
|
stage: "target-fetch",
|
|
20184
20326
|
url: input.targetUrl,
|
|
20185
20327
|
code: "fetch-failed",
|
|
20186
|
-
message: errorMessage(error)
|
|
20328
|
+
message: errorMessage$1(error)
|
|
20187
20329
|
}]
|
|
20188
20330
|
};
|
|
20189
20331
|
}
|
|
@@ -20204,7 +20346,7 @@ async function verifyInternalLinkTarget(input) {
|
|
|
20204
20346
|
stage: "target-extract",
|
|
20205
20347
|
url: input.targetUrl,
|
|
20206
20348
|
code: "extract-failed",
|
|
20207
|
-
message: errorMessage(error)
|
|
20349
|
+
message: errorMessage$1(error)
|
|
20208
20350
|
}]
|
|
20209
20351
|
};
|
|
20210
20352
|
}
|
|
@@ -20267,7 +20409,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
20267
20409
|
stage: "source-fetch",
|
|
20268
20410
|
url: input.candidate.sourceUrl,
|
|
20269
20411
|
code: "fetch-failed",
|
|
20270
|
-
message: errorMessage(error)
|
|
20412
|
+
message: errorMessage$1(error)
|
|
20271
20413
|
}]
|
|
20272
20414
|
};
|
|
20273
20415
|
}
|
|
@@ -20285,7 +20427,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
20285
20427
|
stage: "source-fetch",
|
|
20286
20428
|
url: input.candidate.sourceUrl,
|
|
20287
20429
|
code: "fetch-failed",
|
|
20288
|
-
message: errorMessage(error)
|
|
20430
|
+
message: errorMessage$1(error)
|
|
20289
20431
|
}]
|
|
20290
20432
|
};
|
|
20291
20433
|
}
|
|
@@ -20299,7 +20441,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
20299
20441
|
stage: "source-extract",
|
|
20300
20442
|
url: input.candidate.sourceUrl,
|
|
20301
20443
|
code: "extract-failed",
|
|
20302
|
-
message: errorMessage(error)
|
|
20444
|
+
message: errorMessage$1(error)
|
|
20303
20445
|
}]
|
|
20304
20446
|
};
|
|
20305
20447
|
}
|
|
@@ -21107,7 +21249,7 @@ function compareText$3(left, right) {
|
|
|
21107
21249
|
}
|
|
21108
21250
|
return leftPoints.length - rightPoints.length;
|
|
21109
21251
|
}
|
|
21110
|
-
function boundedInteger$
|
|
21252
|
+
function boundedInteger$2(value, fallback, maximum) {
|
|
21111
21253
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
21112
21254
|
return Math.min(maximum, Math.max(1, Math.trunc(value)));
|
|
21113
21255
|
}
|
|
@@ -21300,10 +21442,10 @@ function caveats(input) {
|
|
|
21300
21442
|
return items;
|
|
21301
21443
|
}
|
|
21302
21444
|
function analyzeIndexCoverageSignals(input) {
|
|
21303
|
-
const itemLimit = boundedInteger$
|
|
21304
|
-
const clusterLimit = boundedInteger$
|
|
21305
|
-
const clusterSamples = boundedInteger$
|
|
21306
|
-
const minUrls = boundedInteger$
|
|
21445
|
+
const itemLimit = boundedInteger$2(input.limits?.itemsPerSection, DEFAULT_ITEMS_PER_SECTION, MAX_ITEMS_PER_SECTION);
|
|
21446
|
+
const clusterLimit = boundedInteger$2(input.limits?.templateClusters, DEFAULT_TEMPLATE_LIMIT, MAX_TEMPLATE_LIMIT);
|
|
21447
|
+
const clusterSamples = boundedInteger$2(input.limits?.templateSamples, DEFAULT_TEMPLATE_SAMPLES, MAX_TEMPLATE_SAMPLES);
|
|
21448
|
+
const minUrls = boundedInteger$2(input.templateThresholds?.minUrls, DEFAULT_TEMPLATE_MIN_URLS, Number.MAX_SAFE_INTEGER);
|
|
21307
21449
|
const requestedMinShare = input.templateThresholds?.minShare;
|
|
21308
21450
|
const minShare = Number.isFinite(requestedMinShare) ? Math.min(1, Math.max(0, requestedMinShare ?? DEFAULT_TEMPLATE_MIN_SHARE)) : DEFAULT_TEMPLATE_MIN_SHARE;
|
|
21309
21451
|
const crawl = crawlInput(input.crawl);
|
|
@@ -24990,16 +25132,16 @@ function benchmarkDetails(benchmark) {
|
|
|
24990
25132
|
positiveUrlSamples: benchmark.positiveUrlSamples
|
|
24991
25133
|
};
|
|
24992
25134
|
}
|
|
24993
|
-
function boundedInteger(input) {
|
|
25135
|
+
function boundedInteger$1(input) {
|
|
24994
25136
|
const value = Number.isFinite(input.value) ? input.value ?? 0 : input.fallback;
|
|
24995
25137
|
return Math.min(input.maximum ?? Number.POSITIVE_INFINITY, Math.max(1, Math.floor(value)));
|
|
24996
25138
|
}
|
|
24997
25139
|
function analyzeQueryClustersFromRows(input) {
|
|
24998
|
-
const minImpressions = boundedInteger({
|
|
25140
|
+
const minImpressions = boundedInteger$1({
|
|
24999
25141
|
value: input.minImpressions,
|
|
25000
25142
|
fallback: 25
|
|
25001
25143
|
});
|
|
25002
|
-
const limit = boundedInteger({
|
|
25144
|
+
const limit = boundedInteger$1({
|
|
25003
25145
|
value: input.limit,
|
|
25004
25146
|
fallback: 25,
|
|
25005
25147
|
maximum: 100
|
|
@@ -25641,8 +25783,8 @@ function reportPresentation(report) {
|
|
|
25641
25783
|
//#region packages/core/dist/analyze/second-page-analysis.js
|
|
25642
25784
|
const DEFAULT_MIN_IMPRESSIONS = 50;
|
|
25643
25785
|
const MAX_MIN_IMPRESSIONS = 1e9;
|
|
25644
|
-
const DEFAULT_LIMIT = 10;
|
|
25645
|
-
const MAX_LIMIT = 100;
|
|
25786
|
+
const DEFAULT_LIMIT$1 = 10;
|
|
25787
|
+
const MAX_LIMIT$1 = 100;
|
|
25646
25788
|
function integerInRange(value, fallback, minimum, maximum) {
|
|
25647
25789
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
25648
25790
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
@@ -25729,7 +25871,7 @@ function compareItems(left, right) {
|
|
|
25729
25871
|
}
|
|
25730
25872
|
function analyzeSecondPageRows(input) {
|
|
25731
25873
|
const minImpressions = integerInRange(input.minImpressions, DEFAULT_MIN_IMPRESSIONS, 0, MAX_MIN_IMPRESSIONS);
|
|
25732
|
-
const limit = integerInRange(input.limit, DEFAULT_LIMIT, 1, MAX_LIMIT);
|
|
25874
|
+
const limit = integerInRange(input.limit, DEFAULT_LIMIT$1, 1, MAX_LIMIT$1);
|
|
25733
25875
|
const selection = {
|
|
25734
25876
|
sourceRows: input.rows.length,
|
|
25735
25877
|
invalidRows: 0,
|
|
@@ -27127,138 +27269,1489 @@ async function updatePostmortemWorkflow(input, dependencies = {}) {
|
|
|
27127
27269
|
});
|
|
27128
27270
|
}
|
|
27129
27271
|
//#endregion
|
|
27130
|
-
//#region packages/core/dist/
|
|
27131
|
-
const
|
|
27132
|
-
|
|
27133
|
-
|
|
27134
|
-
|
|
27135
|
-
|
|
27136
|
-
|
|
27137
|
-
|
|
27138
|
-
|
|
27139
|
-
|
|
27140
|
-
|
|
27141
|
-
|
|
27142
|
-
|
|
27143
|
-
|
|
27144
|
-
|
|
27145
|
-
|
|
27146
|
-
|
|
27147
|
-
|
|
27148
|
-
|
|
27149
|
-
|
|
27150
|
-
|
|
27151
|
-
|
|
27152
|
-
|
|
27153
|
-
|
|
27154
|
-
|
|
27155
|
-
|
|
27156
|
-
|
|
27272
|
+
//#region packages/core/dist/bing/analysis.js
|
|
27273
|
+
const DAY_MS = 864e5;
|
|
27274
|
+
const DEFAULT_COMPARISON_DAYS = 28;
|
|
27275
|
+
const WEEKLY_PERIODS = 4;
|
|
27276
|
+
const OPPORTUNITY_MIN_IMPRESSIONS = 20;
|
|
27277
|
+
const OPPORTUNITY_MIN_POSITION = 4;
|
|
27278
|
+
const OPPORTUNITY_MAX_POSITION = 20;
|
|
27279
|
+
const RESULT_LIMIT = 10;
|
|
27280
|
+
function dateOffset(date, days) {
|
|
27281
|
+
return new Date(Date.parse(`${date}T00:00:00Z`) + days * DAY_MS).toISOString().slice(0, 10);
|
|
27282
|
+
}
|
|
27283
|
+
function percentChange(previous, current) {
|
|
27284
|
+
if (previous === 0) return null;
|
|
27285
|
+
return round((current - previous) / previous * 100);
|
|
27286
|
+
}
|
|
27287
|
+
function round(value, digits = 3) {
|
|
27288
|
+
const factor = 10 ** digits;
|
|
27289
|
+
return Math.round(value * factor) / factor;
|
|
27290
|
+
}
|
|
27291
|
+
function trafficPeriod(byDate, startDate, endDate, expectedDays) {
|
|
27292
|
+
let clicks = 0;
|
|
27293
|
+
let impressions = 0;
|
|
27294
|
+
let observedDays = 0;
|
|
27295
|
+
for (const [date, row] of byDate) {
|
|
27296
|
+
if (date < startDate || date > endDate) continue;
|
|
27297
|
+
observedDays += 1;
|
|
27298
|
+
clicks += row.clicks;
|
|
27299
|
+
impressions += row.impressions;
|
|
27157
27300
|
}
|
|
27158
|
-
checks.push({
|
|
27159
|
-
id: "config-dir",
|
|
27160
|
-
label: "Config directory",
|
|
27161
|
-
status: existsSync(paths.configDir) ? "pass" : "warn",
|
|
27162
|
-
detail: paths.configDir,
|
|
27163
|
-
fix: existsSync(paths.configDir) ? void 0 : "Run `seo start` or `seo auth setup-client`."
|
|
27164
|
-
});
|
|
27165
|
-
checks.push({
|
|
27166
|
-
id: "oauth-client",
|
|
27167
|
-
label: "Google credentials",
|
|
27168
|
-
status: serviceAccount.configured ? "pass" : tokenClientConfigured === false ? "fail" : serviceAccount.error ? "fail" : authMode.sharedConfigured || authMode.byoConfigured ? "pass" : "fail",
|
|
27169
|
-
detail: serviceAccount.configured ? `Service account configured for ${serviceAccount.identity}.` : tokenClientConfigured === false ? `Stored Google login uses the ${tokens?.client_source === "shared" ? "shared seo app" : "BYO client"}, but that client is not configured.` : serviceAccount.error ? serviceAccount.error : authMode.sharedConfigured ? "Shared client configured." : authMode.byoConfigured ? `BYO client configured at ${paths.oauthClientFile}.` : "No shared or BYO Google OAuth client configured.",
|
|
27170
|
-
fix: serviceAccount.configured ? void 0 : tokenClientConfigured === false ? tokens?.client_source === "shared" ? "Reinstall `seo`. If the shared client is still missing, report it at https://github.com/iannuttall/seo/issues." : "Restore the same client with `seo auth setup-client`, or run `seo auth logout` before signing in with a different client." : serviceAccount.error ? "Set one valid service account credential source, then run `seo doctor` again." : authMode.sharedConfigured || authMode.byoConfigured ? void 0 : "Run `seo auth setup-client` or set SEO_GOOGLE_CLIENT_ID and SEO_GOOGLE_CLIENT_SECRET."
|
|
27171
|
-
});
|
|
27172
|
-
checks.push({
|
|
27173
|
-
id: "oauth-client-file",
|
|
27174
|
-
label: "BYO OAuth file",
|
|
27175
|
-
status: oauthClient ? "pass" : "warn",
|
|
27176
|
-
detail: oauthClient ? paths.oauthClientFile : "No BYO OAuth client file. This is fine if using a shared client."
|
|
27177
|
-
});
|
|
27178
|
-
checks.push({
|
|
27179
|
-
id: "google-login",
|
|
27180
|
-
label: "Google login",
|
|
27181
|
-
status: serviceAccount.configured || tokens ? "pass" : "fail",
|
|
27182
|
-
detail: serviceAccount.configured ? `Using service account ${serviceAccount.identity}.` : tokens ? `Signed in as ${tokens.account_email}.` : "No Google token found.",
|
|
27183
|
-
fix: serviceAccount.configured || tokens ? void 0 : "Run `seo auth login`, or configure a service account for CI."
|
|
27184
|
-
});
|
|
27185
|
-
const grantedScopes = new Set(tokens?.scope.split(/\s+/).filter(Boolean) ?? []);
|
|
27186
|
-
for (const scope of REQUIRED_SCOPES) checks.push({
|
|
27187
|
-
id: `scope:${scope}`,
|
|
27188
|
-
label: scope.endsWith("webmasters.readonly") ? "Search Console scope" : "Google Analytics scope",
|
|
27189
|
-
status: serviceAccount.configured || tokens && grantedScopes.has(scope) ? "pass" : tokens ? "fail" : "warn",
|
|
27190
|
-
detail: serviceAccount.configured ? `${scope} requested by the service account.` : tokens ? grantedScopes.has(scope) ? scope : `Missing ${scope}.` : "Cannot inspect scopes until login is complete.",
|
|
27191
|
-
fix: !serviceAccount.configured && tokens && !grantedScopes.has(scope) ? "Run `seo auth logout` then `seo auth login`." : void 0
|
|
27192
|
-
});
|
|
27193
|
-
checks.push({
|
|
27194
|
-
id: "default-site",
|
|
27195
|
-
label: "Saved GSC property",
|
|
27196
|
-
status: "pass",
|
|
27197
|
-
detail: config.defaultSite ?? "No saved default. Human CLI commands will prompt; agents can pass --site."
|
|
27198
|
-
});
|
|
27199
|
-
checks.push({
|
|
27200
|
-
id: "default-google-analytics",
|
|
27201
|
-
label: "Saved Google Analytics property",
|
|
27202
|
-
status: "pass",
|
|
27203
|
-
detail: config.analytics.google.defaultPropertyId ?? "No saved default. Human CLI commands will prompt; agents can pass --property."
|
|
27204
|
-
});
|
|
27205
27301
|
return {
|
|
27206
|
-
|
|
27207
|
-
|
|
27208
|
-
|
|
27302
|
+
startDate,
|
|
27303
|
+
endDate,
|
|
27304
|
+
expectedDays,
|
|
27305
|
+
observedDays,
|
|
27306
|
+
missingDays: Math.max(0, expectedDays - observedDays),
|
|
27307
|
+
clicks,
|
|
27308
|
+
impressions,
|
|
27309
|
+
ctr: impressions ? round(clicks / impressions * 100) : null
|
|
27209
27310
|
};
|
|
27210
27311
|
}
|
|
27211
|
-
|
|
27212
|
-
|
|
27213
|
-
|
|
27214
|
-
|
|
27215
|
-
|
|
27216
|
-
|
|
27217
|
-
|
|
27218
|
-
|
|
27219
|
-
|
|
27220
|
-
|
|
27221
|
-
|
|
27222
|
-
}
|
|
27223
|
-
|
|
27224
|
-
|
|
27225
|
-
|
|
27226
|
-
|
|
27227
|
-
|
|
27228
|
-
|
|
27229
|
-
|
|
27230
|
-
|
|
27231
|
-
|
|
27232
|
-
|
|
27233
|
-
|
|
27234
|
-
|
|
27235
|
-
|
|
27236
|
-
|
|
27237
|
-
|
|
27238
|
-
|
|
27239
|
-
|
|
27240
|
-
|
|
27241
|
-
|
|
27242
|
-
|
|
27243
|
-
|
|
27244
|
-
|
|
27245
|
-
|
|
27246
|
-
|
|
27247
|
-
|
|
27248
|
-
|
|
27249
|
-
|
|
27250
|
-
|
|
27251
|
-
|
|
27252
|
-
|
|
27253
|
-
|
|
27254
|
-
|
|
27312
|
+
function analyzeBingTraffic(rows, comparisonDays = DEFAULT_COMPARISON_DAYS) {
|
|
27313
|
+
if (!rows.length) return void 0;
|
|
27314
|
+
const byDate = /* @__PURE__ */ new Map();
|
|
27315
|
+
for (const row of rows) {
|
|
27316
|
+
const existing = byDate.get(row.date) ?? {
|
|
27317
|
+
clicks: 0,
|
|
27318
|
+
impressions: 0
|
|
27319
|
+
};
|
|
27320
|
+
existing.clicks += row.clicks;
|
|
27321
|
+
existing.impressions += row.impressions;
|
|
27322
|
+
byDate.set(row.date, existing);
|
|
27323
|
+
}
|
|
27324
|
+
const latestDate = [...byDate.keys()].sort().at(-1);
|
|
27325
|
+
if (!latestDate) return void 0;
|
|
27326
|
+
const currentStart = dateOffset(latestDate, -(comparisonDays - 1));
|
|
27327
|
+
const previousEnd = dateOffset(currentStart, -1);
|
|
27328
|
+
const previousStart = dateOffset(previousEnd, -(comparisonDays - 1));
|
|
27329
|
+
const current = trafficPeriod(byDate, currentStart, latestDate, comparisonDays);
|
|
27330
|
+
const previous = trafficPeriod(byDate, previousStart, previousEnd, comparisonDays);
|
|
27331
|
+
return {
|
|
27332
|
+
comparisonDays,
|
|
27333
|
+
current,
|
|
27334
|
+
previous,
|
|
27335
|
+
changes: {
|
|
27336
|
+
clicks: current.clicks - previous.clicks,
|
|
27337
|
+
clicksPercent: percentChange(previous.clicks, current.clicks),
|
|
27338
|
+
impressions: current.impressions - previous.impressions,
|
|
27339
|
+
impressionsPercent: percentChange(previous.impressions, current.impressions),
|
|
27340
|
+
ctrPercentagePoints: current.ctr === null || previous.ctr === null ? null : round(current.ctr - previous.ctr)
|
|
27341
|
+
}
|
|
27342
|
+
};
|
|
27343
|
+
}
|
|
27344
|
+
const CRAWL_FIELDS = [
|
|
27345
|
+
"crawledPages",
|
|
27346
|
+
"inIndex",
|
|
27347
|
+
"inLinks",
|
|
27348
|
+
"crawlErrors",
|
|
27349
|
+
"code4xx",
|
|
27350
|
+
"code5xx",
|
|
27351
|
+
"blockedByRobotsTxt",
|
|
27352
|
+
"connectionTimeout",
|
|
27353
|
+
"dnsFailures",
|
|
27354
|
+
"containsMalware"
|
|
27355
|
+
];
|
|
27356
|
+
function analyzeBingCrawl(rows, comparisonDays = DEFAULT_COMPARISON_DAYS) {
|
|
27357
|
+
const ordered = [...rows].sort((a, b) => a.date.localeCompare(b.date, "en"));
|
|
27358
|
+
const current = ordered.at(-1);
|
|
27359
|
+
if (!current) return void 0;
|
|
27360
|
+
const targetDate = dateOffset(current.date, -comparisonDays);
|
|
27361
|
+
const previous = ordered.filter((row) => row.date <= targetDate).at(-1);
|
|
27362
|
+
const changes = {};
|
|
27363
|
+
if (!previous) return {
|
|
27364
|
+
comparisonDays,
|
|
27365
|
+
targetDate,
|
|
27366
|
+
current,
|
|
27367
|
+
previous: void 0,
|
|
27368
|
+
changes
|
|
27369
|
+
};
|
|
27370
|
+
for (const field of CRAWL_FIELDS) {
|
|
27371
|
+
const currentValue = current[field];
|
|
27372
|
+
const previousValue = previous[field];
|
|
27373
|
+
if (typeof currentValue !== "number" || typeof previousValue !== "number") continue;
|
|
27374
|
+
changes[field] = {
|
|
27375
|
+
previous: previousValue,
|
|
27376
|
+
current: currentValue,
|
|
27377
|
+
absolute: currentValue - previousValue,
|
|
27378
|
+
percent: percentChange(previousValue, currentValue)
|
|
27255
27379
|
};
|
|
27256
27380
|
}
|
|
27257
|
-
|
|
27258
|
-
|
|
27259
|
-
|
|
27260
|
-
|
|
27261
|
-
|
|
27381
|
+
return {
|
|
27382
|
+
comparisonDays,
|
|
27383
|
+
targetDate,
|
|
27384
|
+
current,
|
|
27385
|
+
previous,
|
|
27386
|
+
changes
|
|
27387
|
+
};
|
|
27388
|
+
}
|
|
27389
|
+
function aggregateDimensions(rows, dates) {
|
|
27390
|
+
const values = /* @__PURE__ */ new Map();
|
|
27391
|
+
for (const row of rows) {
|
|
27392
|
+
if (!dates.has(row.date)) continue;
|
|
27393
|
+
const item = values.get(row.value) ?? {
|
|
27394
|
+
clicks: 0,
|
|
27395
|
+
impressions: 0,
|
|
27396
|
+
clickPositionTotal: 0,
|
|
27397
|
+
clickPositionWeight: 0,
|
|
27398
|
+
impressionPositionTotal: 0,
|
|
27399
|
+
impressionPositionWeight: 0,
|
|
27400
|
+
observedDates: /* @__PURE__ */ new Set()
|
|
27401
|
+
};
|
|
27402
|
+
item.clicks += row.clicks;
|
|
27403
|
+
item.impressions += row.impressions;
|
|
27404
|
+
item.observedDates.add(row.date);
|
|
27405
|
+
if (row.avgClickPosition !== void 0 && row.clicks > 0) {
|
|
27406
|
+
const weight = row.clicks;
|
|
27407
|
+
item.clickPositionTotal += row.avgClickPosition * weight;
|
|
27408
|
+
item.clickPositionWeight += weight;
|
|
27409
|
+
}
|
|
27410
|
+
if (row.avgImpressionPosition !== void 0 && row.impressions > 0) {
|
|
27411
|
+
const weight = row.impressions;
|
|
27412
|
+
item.impressionPositionTotal += row.avgImpressionPosition * weight;
|
|
27413
|
+
item.impressionPositionWeight += weight;
|
|
27414
|
+
}
|
|
27415
|
+
values.set(row.value, item);
|
|
27416
|
+
}
|
|
27417
|
+
return new Map([...values].map(([value, item]) => [value, {
|
|
27418
|
+
value,
|
|
27419
|
+
observedWeeks: item.observedDates.size,
|
|
27420
|
+
clicks: item.clicks,
|
|
27421
|
+
impressions: item.impressions,
|
|
27422
|
+
avgClickPosition: item.clickPositionWeight ? round(item.clickPositionTotal / item.clickPositionWeight) : null,
|
|
27423
|
+
avgImpressionPosition: item.impressionPositionWeight ? round(item.impressionPositionTotal / item.impressionPositionWeight) : null
|
|
27424
|
+
}]));
|
|
27425
|
+
}
|
|
27426
|
+
function compareValue(a, b) {
|
|
27427
|
+
return a.value.localeCompare(b.value, "en");
|
|
27428
|
+
}
|
|
27429
|
+
function analyzeBingDimensions(rows, kind) {
|
|
27430
|
+
const dates = [...new Set(rows.map((row) => row.date))].sort().reverse();
|
|
27431
|
+
const currentDates = dates.slice(0, WEEKLY_PERIODS);
|
|
27432
|
+
const previousDates = dates.slice(WEEKLY_PERIODS, WEEKLY_PERIODS * 2);
|
|
27433
|
+
const current = aggregateDimensions(rows, new Set(currentDates));
|
|
27434
|
+
const previous = aggregateDimensions(rows, new Set(previousDates));
|
|
27435
|
+
const matchedValues = [...current.keys()].filter((value) => previous.has(value));
|
|
27436
|
+
const comparableValues = currentDates.length === WEEKLY_PERIODS && previousDates.length === WEEKLY_PERIODS ? matchedValues.filter((value) => {
|
|
27437
|
+
const currentValue = current.get(value);
|
|
27438
|
+
const previousValue = previous.get(value);
|
|
27439
|
+
return currentValue?.observedWeeks === WEEKLY_PERIODS && previousValue?.observedWeeks === WEEKLY_PERIODS;
|
|
27440
|
+
}) : [];
|
|
27441
|
+
const movements = comparableValues.map((value) => {
|
|
27442
|
+
const currentValue = current.get(value);
|
|
27443
|
+
const previousValue = previous.get(value);
|
|
27444
|
+
return {
|
|
27445
|
+
value,
|
|
27446
|
+
current: currentValue,
|
|
27447
|
+
previous: previousValue,
|
|
27448
|
+
changes: {
|
|
27449
|
+
clicks: currentValue.clicks - previousValue.clicks,
|
|
27450
|
+
clicksPercent: percentChange(previousValue.clicks, currentValue.clicks),
|
|
27451
|
+
impressions: currentValue.impressions - previousValue.impressions,
|
|
27452
|
+
impressionsPercent: percentChange(previousValue.impressions, currentValue.impressions)
|
|
27453
|
+
}
|
|
27454
|
+
};
|
|
27455
|
+
}).sort((a, b) => Math.abs(b.changes.clicks) - Math.abs(a.changes.clicks) || Math.abs(b.changes.impressions) - Math.abs(a.changes.impressions) || compareValue(a, b)).slice(0, RESULT_LIMIT);
|
|
27456
|
+
const opportunities = [...current.values()].filter((row) => row.impressions >= OPPORTUNITY_MIN_IMPRESSIONS && row.avgImpressionPosition !== null && row.avgImpressionPosition >= OPPORTUNITY_MIN_POSITION && row.avgImpressionPosition <= OPPORTUNITY_MAX_POSITION).sort((a, b) => b.impressions - a.impressions || b.clicks - a.clicks || compareValue(a, b)).slice(0, RESULT_LIMIT);
|
|
27457
|
+
return {
|
|
27458
|
+
kind,
|
|
27459
|
+
sourceSemantics: "Bing weekly top-list evidence. Missing dimensions are unknown, not zero. Movements require observation in every week of both periods.",
|
|
27460
|
+
periods: {
|
|
27461
|
+
weeksPerPeriod: WEEKLY_PERIODS,
|
|
27462
|
+
currentDates,
|
|
27463
|
+
previousDates
|
|
27464
|
+
},
|
|
27465
|
+
coverage: {
|
|
27466
|
+
sourceRows: rows.length,
|
|
27467
|
+
currentDimensions: current.size,
|
|
27468
|
+
previousDimensions: previous.size,
|
|
27469
|
+
matchedDimensions: matchedValues.length,
|
|
27470
|
+
comparableDimensions: comparableValues.length,
|
|
27471
|
+
incompleteMatchedDimensions: matchedValues.length - comparableValues.length,
|
|
27472
|
+
currentOnlyDimensions: [...current.keys()].filter((value) => !previous.has(value)).length,
|
|
27473
|
+
previousOnlyDimensions: [...previous.keys()].filter((value) => !current.has(value)).length
|
|
27474
|
+
},
|
|
27475
|
+
thresholds: {
|
|
27476
|
+
heuristic: true,
|
|
27477
|
+
opportunityMinImpressions: OPPORTUNITY_MIN_IMPRESSIONS,
|
|
27478
|
+
opportunityPositionRange: [OPPORTUNITY_MIN_POSITION, OPPORTUNITY_MAX_POSITION],
|
|
27479
|
+
resultLimit: RESULT_LIMIT
|
|
27480
|
+
},
|
|
27481
|
+
movements,
|
|
27482
|
+
opportunities
|
|
27483
|
+
};
|
|
27484
|
+
}
|
|
27485
|
+
//#endregion
|
|
27486
|
+
//#region packages/core/dist/bing/client.js
|
|
27487
|
+
const DEFAULT_BASE_URL = "https://ssl.bing.com/webmaster/api.svc/json";
|
|
27488
|
+
const MAX_RESPONSE_BYTES$1 = 2e6;
|
|
27489
|
+
const MAX_STAT_ROWS = 400;
|
|
27490
|
+
const MAX_DIMENSION_ROWS = 8e3;
|
|
27491
|
+
const MAX_LINK_ROWS = 200;
|
|
27492
|
+
function bingDate(value) {
|
|
27493
|
+
if (typeof value !== "string") return void 0;
|
|
27494
|
+
const legacy = /^\/Date\((-?\d+)(?:[+-]\d{4})?\)\/$/.exec(value);
|
|
27495
|
+
const date = legacy ? new Date(Number(legacy[1])) : new Date(value);
|
|
27496
|
+
return Number.isNaN(date.getTime()) ? void 0 : date.toISOString().slice(0, 10);
|
|
27497
|
+
}
|
|
27498
|
+
function nonnegativeNumber(value) {
|
|
27499
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
|
|
27500
|
+
}
|
|
27501
|
+
function positionNumber(value) {
|
|
27502
|
+
if (value === void 0 || value === null || value === -1) return { valid: true };
|
|
27503
|
+
const number = nonnegativeNumber(value);
|
|
27504
|
+
return number === void 0 ? { valid: false } : {
|
|
27505
|
+
valid: true,
|
|
27506
|
+
value: number
|
|
27507
|
+
};
|
|
27508
|
+
}
|
|
27509
|
+
function isHttpUrl(value) {
|
|
27510
|
+
if (typeof value !== "string" || value.length > 2e3) return false;
|
|
27511
|
+
try {
|
|
27512
|
+
const url = new URL(value);
|
|
27513
|
+
return url.protocol === "http:" || url.protocol === "https:";
|
|
27514
|
+
} catch {
|
|
27515
|
+
return false;
|
|
27516
|
+
}
|
|
27517
|
+
}
|
|
27518
|
+
function boundedRows(rows, invalidRows, limit = MAX_STAT_ROWS) {
|
|
27519
|
+
const capped = rows.length > limit;
|
|
27520
|
+
return {
|
|
27521
|
+
rows: capped ? rows.slice(-limit) : rows,
|
|
27522
|
+
invalidRows,
|
|
27523
|
+
capped,
|
|
27524
|
+
returnedRows: rows.length
|
|
27525
|
+
};
|
|
27526
|
+
}
|
|
27527
|
+
function dimensionValue(value, kind) {
|
|
27528
|
+
if (typeof value !== "string") return void 0;
|
|
27529
|
+
const normalized = value.trim();
|
|
27530
|
+
if (!normalized || normalized.length > 2e3) return void 0;
|
|
27531
|
+
if (kind === "page" && !isHttpUrl(normalized)) return void 0;
|
|
27532
|
+
return normalized;
|
|
27533
|
+
}
|
|
27534
|
+
function errorMessage(body) {
|
|
27535
|
+
try {
|
|
27536
|
+
const parsed = JSON.parse(body);
|
|
27537
|
+
return typeof parsed.Message === "string" ? parsed.Message : void 0;
|
|
27538
|
+
} catch {
|
|
27539
|
+
return;
|
|
27540
|
+
}
|
|
27541
|
+
}
|
|
27542
|
+
var BingWebmasterClient = class {
|
|
27543
|
+
credentials;
|
|
27544
|
+
authentication;
|
|
27545
|
+
baseUrl;
|
|
27546
|
+
fetchImpl;
|
|
27547
|
+
timeoutMs;
|
|
27548
|
+
constructor(credentials, options = {}) {
|
|
27549
|
+
this.credentials = credentials;
|
|
27550
|
+
if (!(credentials.apiKey?.trim() || credentials.accessToken?.trim())) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster credentials are empty.");
|
|
27551
|
+
this.authentication = credentials.apiKey ? "api-key" : "oauth";
|
|
27552
|
+
this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, "");
|
|
27553
|
+
this.fetchImpl = options.fetchImpl ?? publicHttpFetch;
|
|
27554
|
+
this.timeoutMs = options.timeoutMs ?? 3e4;
|
|
27555
|
+
}
|
|
27556
|
+
async request(method, params = {}) {
|
|
27557
|
+
const url = new URL(`${this.baseUrl}/${method}`);
|
|
27558
|
+
for (const [key, value] of Object.entries(params)) url.searchParams.set(key, value);
|
|
27559
|
+
if (this.credentials.apiKey) url.searchParams.set("apikey", this.credentials.apiKey);
|
|
27560
|
+
let response;
|
|
27561
|
+
try {
|
|
27562
|
+
response = await this.fetchImpl(url.toString(), {
|
|
27563
|
+
headers: {
|
|
27564
|
+
accept: "application/json",
|
|
27565
|
+
...this.credentials.accessToken ? { authorization: `Bearer ${this.credentials.accessToken}` } : {}
|
|
27566
|
+
},
|
|
27567
|
+
signal: AbortSignal.timeout(this.timeoutMs)
|
|
27568
|
+
});
|
|
27569
|
+
} catch (error) {
|
|
27570
|
+
if (error instanceof SeoError) throw error;
|
|
27571
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster did not respond: ${error instanceof Error ? error.message : String(error)}`);
|
|
27572
|
+
}
|
|
27573
|
+
const body = await readBoundedResponseText(response, MAX_RESPONSE_BYTES$1, "Bing Webmaster response");
|
|
27574
|
+
if (!response.ok) {
|
|
27575
|
+
const message = errorMessage(body);
|
|
27576
|
+
if (response.status === 401 || /invalidapikey|unauthorized/i.test(message ?? "")) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster rejected the saved credential. Run `seo providers bing connect` again.");
|
|
27577
|
+
if (response.status === 403) throw new SeoError("ACCESS_DENIED", "Bing Webmaster denied access to this site. Check that it is verified for the connected account.");
|
|
27578
|
+
if (response.status === 429) throw new SeoError("RATE_LIMITED", "Bing Webmaster rate limited this request. Try again later.");
|
|
27579
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster request failed with HTTP ${response.status}${message ? ` (${message})` : ""}.`);
|
|
27580
|
+
}
|
|
27581
|
+
try {
|
|
27582
|
+
return JSON.parse(body).d;
|
|
27583
|
+
} catch {
|
|
27584
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid JSON.");
|
|
27585
|
+
}
|
|
27586
|
+
}
|
|
27587
|
+
async listSites() {
|
|
27588
|
+
const data = await this.request("GetUserSites");
|
|
27589
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned an invalid site list.");
|
|
27590
|
+
const sites = [];
|
|
27591
|
+
let invalidRows = 0;
|
|
27592
|
+
for (const item of data) {
|
|
27593
|
+
if (!item || typeof item !== "object") {
|
|
27594
|
+
invalidRows += 1;
|
|
27595
|
+
continue;
|
|
27596
|
+
}
|
|
27597
|
+
const row = item;
|
|
27598
|
+
if (typeof row.Url !== "string" || typeof row.IsVerified !== "boolean") {
|
|
27599
|
+
invalidRows += 1;
|
|
27600
|
+
continue;
|
|
27601
|
+
}
|
|
27602
|
+
sites.push({
|
|
27603
|
+
url: row.Url,
|
|
27604
|
+
isVerified: row.IsVerified
|
|
27605
|
+
});
|
|
27606
|
+
}
|
|
27607
|
+
sites.sort((a, b) => a.url.localeCompare(b.url, "en"));
|
|
27608
|
+
return {
|
|
27609
|
+
sites,
|
|
27610
|
+
invalidRows
|
|
27611
|
+
};
|
|
27612
|
+
}
|
|
27613
|
+
async getTraffic(siteUrl) {
|
|
27614
|
+
const data = await this.request("GetRankAndTrafficStats", { siteUrl });
|
|
27615
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid traffic statistics.");
|
|
27616
|
+
const rows = [];
|
|
27617
|
+
let invalidRows = 0;
|
|
27618
|
+
for (const item of data) {
|
|
27619
|
+
if (!item || typeof item !== "object") {
|
|
27620
|
+
invalidRows += 1;
|
|
27621
|
+
continue;
|
|
27622
|
+
}
|
|
27623
|
+
const row = item;
|
|
27624
|
+
const date = bingDate(row.Date);
|
|
27625
|
+
const clicks = nonnegativeNumber(row.Clicks);
|
|
27626
|
+
const impressions = nonnegativeNumber(row.Impressions);
|
|
27627
|
+
if (!date || clicks === void 0 || impressions === void 0) {
|
|
27628
|
+
invalidRows += 1;
|
|
27629
|
+
continue;
|
|
27630
|
+
}
|
|
27631
|
+
rows.push({
|
|
27632
|
+
date,
|
|
27633
|
+
clicks,
|
|
27634
|
+
impressions
|
|
27635
|
+
});
|
|
27636
|
+
}
|
|
27637
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en") || a.clicks - b.clicks || a.impressions - b.impressions);
|
|
27638
|
+
return boundedRows(rows, invalidRows);
|
|
27639
|
+
}
|
|
27640
|
+
async getCrawlStats(siteUrl) {
|
|
27641
|
+
const data = await this.request("GetCrawlStats", { siteUrl });
|
|
27642
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid crawl statistics.");
|
|
27643
|
+
const fieldMap = {
|
|
27644
|
+
AllOtherCodes: "allOtherCodes",
|
|
27645
|
+
BlockedByRobotsTxt: "blockedByRobotsTxt",
|
|
27646
|
+
Code2xx: "code2xx",
|
|
27647
|
+
Code301: "code301",
|
|
27648
|
+
Code302: "code302",
|
|
27649
|
+
Code4xx: "code4xx",
|
|
27650
|
+
Code5xx: "code5xx",
|
|
27651
|
+
ConnectionTimeout: "connectionTimeout",
|
|
27652
|
+
ContainsMalware: "containsMalware",
|
|
27653
|
+
CrawledPages: "crawledPages",
|
|
27654
|
+
CrawlErrors: "crawlErrors",
|
|
27655
|
+
DnsFailures: "dnsFailures",
|
|
27656
|
+
InIndex: "inIndex",
|
|
27657
|
+
InLinks: "inLinks"
|
|
27658
|
+
};
|
|
27659
|
+
const rows = [];
|
|
27660
|
+
let invalidRows = 0;
|
|
27661
|
+
for (const item of data) {
|
|
27662
|
+
if (!item || typeof item !== "object") {
|
|
27663
|
+
invalidRows += 1;
|
|
27664
|
+
continue;
|
|
27665
|
+
}
|
|
27666
|
+
const source = item;
|
|
27667
|
+
const date = bingDate(source.Date);
|
|
27668
|
+
if (!date) {
|
|
27669
|
+
invalidRows += 1;
|
|
27670
|
+
continue;
|
|
27671
|
+
}
|
|
27672
|
+
const row = { date };
|
|
27673
|
+
let invalid = false;
|
|
27674
|
+
for (const [providerField, field] of Object.entries(fieldMap)) {
|
|
27675
|
+
const value = source[providerField];
|
|
27676
|
+
if (value === void 0 || value === null) continue;
|
|
27677
|
+
const number = nonnegativeNumber(value);
|
|
27678
|
+
if (number === void 0) {
|
|
27679
|
+
invalid = true;
|
|
27680
|
+
break;
|
|
27681
|
+
}
|
|
27682
|
+
row[field] = number;
|
|
27683
|
+
}
|
|
27684
|
+
if (invalid) {
|
|
27685
|
+
invalidRows += 1;
|
|
27686
|
+
continue;
|
|
27687
|
+
}
|
|
27688
|
+
rows.push(row);
|
|
27689
|
+
}
|
|
27690
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en"));
|
|
27691
|
+
return boundedRows(rows, invalidRows);
|
|
27692
|
+
}
|
|
27693
|
+
async getQueryStats(siteUrl) {
|
|
27694
|
+
return this.getDimensionStats("GetQueryStats", siteUrl, "query");
|
|
27695
|
+
}
|
|
27696
|
+
async getPageStats(siteUrl) {
|
|
27697
|
+
return this.getDimensionStats("GetPageStats", siteUrl, "page");
|
|
27698
|
+
}
|
|
27699
|
+
async getDimensionStats(method, siteUrl, kind) {
|
|
27700
|
+
const data = await this.request(method, { siteUrl });
|
|
27701
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster returned invalid ${kind} statistics.`);
|
|
27702
|
+
const rows = [];
|
|
27703
|
+
let invalidRows = 0;
|
|
27704
|
+
for (const item of data) {
|
|
27705
|
+
if (!item || typeof item !== "object") {
|
|
27706
|
+
invalidRows += 1;
|
|
27707
|
+
continue;
|
|
27708
|
+
}
|
|
27709
|
+
const source = item;
|
|
27710
|
+
const date = bingDate(source.Date);
|
|
27711
|
+
const value = dimensionValue(source.Query, kind);
|
|
27712
|
+
const clicks = nonnegativeNumber(source.Clicks);
|
|
27713
|
+
const impressions = nonnegativeNumber(source.Impressions);
|
|
27714
|
+
const clickPosition = positionNumber(source.AvgClickPosition);
|
|
27715
|
+
const impressionPosition = positionNumber(source.AvgImpressionPosition);
|
|
27716
|
+
if (!date || !value || clicks === void 0 || impressions === void 0 || !clickPosition.valid || !impressionPosition.valid) {
|
|
27717
|
+
invalidRows += 1;
|
|
27718
|
+
continue;
|
|
27719
|
+
}
|
|
27720
|
+
rows.push({
|
|
27721
|
+
date,
|
|
27722
|
+
value,
|
|
27723
|
+
clicks,
|
|
27724
|
+
impressions,
|
|
27725
|
+
avgClickPosition: clickPosition.value,
|
|
27726
|
+
avgImpressionPosition: impressionPosition.value
|
|
27727
|
+
});
|
|
27728
|
+
}
|
|
27729
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en") || a.value.localeCompare(b.value, "en"));
|
|
27730
|
+
return boundedRows(rows, invalidRows, MAX_DIMENSION_ROWS);
|
|
27731
|
+
}
|
|
27732
|
+
async getLinkCounts(siteUrl, page) {
|
|
27733
|
+
const data = await this.request("GetLinkCounts", {
|
|
27734
|
+
siteUrl,
|
|
27735
|
+
page: String(page)
|
|
27736
|
+
});
|
|
27737
|
+
if (!data || typeof data !== "object") throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link counts.");
|
|
27738
|
+
const response = data;
|
|
27739
|
+
if (!Array.isArray(response.Links)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link counts.");
|
|
27740
|
+
const rows = [];
|
|
27741
|
+
let invalidRows = 0;
|
|
27742
|
+
for (const item of response.Links) {
|
|
27743
|
+
if (!item || typeof item !== "object") {
|
|
27744
|
+
invalidRows += 1;
|
|
27745
|
+
continue;
|
|
27746
|
+
}
|
|
27747
|
+
const row = item;
|
|
27748
|
+
const count = nonnegativeNumber(row.Count);
|
|
27749
|
+
if (!isHttpUrl(row.Url) || count === void 0) {
|
|
27750
|
+
invalidRows += 1;
|
|
27751
|
+
continue;
|
|
27752
|
+
}
|
|
27753
|
+
rows.push({
|
|
27754
|
+
url: row.Url,
|
|
27755
|
+
count
|
|
27756
|
+
});
|
|
27757
|
+
}
|
|
27758
|
+
rows.sort((a, b) => b.count - a.count || a.url.localeCompare(b.url, "en"));
|
|
27759
|
+
return linkPage(rows, response.TotalPages, invalidRows);
|
|
27760
|
+
}
|
|
27761
|
+
async getUrlLinks(siteUrl, link, page) {
|
|
27762
|
+
const data = await this.request("GetUrlLinks", {
|
|
27763
|
+
siteUrl,
|
|
27764
|
+
link,
|
|
27765
|
+
page: String(page)
|
|
27766
|
+
});
|
|
27767
|
+
if (!data || typeof data !== "object") throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid referring links.");
|
|
27768
|
+
const response = data;
|
|
27769
|
+
if (!Array.isArray(response.Details)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid referring links.");
|
|
27770
|
+
const rows = [];
|
|
27771
|
+
let invalidRows = 0;
|
|
27772
|
+
for (const item of response.Details) {
|
|
27773
|
+
if (!item || typeof item !== "object") {
|
|
27774
|
+
invalidRows += 1;
|
|
27775
|
+
continue;
|
|
27776
|
+
}
|
|
27777
|
+
const row = item;
|
|
27778
|
+
if (!isHttpUrl(row.Url)) {
|
|
27779
|
+
invalidRows += 1;
|
|
27780
|
+
continue;
|
|
27781
|
+
}
|
|
27782
|
+
rows.push({
|
|
27783
|
+
url: row.Url,
|
|
27784
|
+
anchorText: typeof row.AnchorText === "string" && row.AnchorText.trim() ? row.AnchorText.trim() : void 0
|
|
27785
|
+
});
|
|
27786
|
+
}
|
|
27787
|
+
rows.sort((a, b) => a.url.localeCompare(b.url, "en") || (a.anchorText ?? "").localeCompare(b.anchorText ?? "", "en"));
|
|
27788
|
+
return linkPage(rows, response.TotalPages, invalidRows);
|
|
27789
|
+
}
|
|
27790
|
+
};
|
|
27791
|
+
function linkPage(rows, totalPagesValue, invalidRows) {
|
|
27792
|
+
const totalPages = nonnegativeNumber(totalPagesValue);
|
|
27793
|
+
if (totalPages === void 0 || !Number.isInteger(totalPages)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link pagination.");
|
|
27794
|
+
const capped = rows.length > MAX_LINK_ROWS;
|
|
27795
|
+
return {
|
|
27796
|
+
rows: capped ? rows.slice(0, MAX_LINK_ROWS) : rows,
|
|
27797
|
+
totalPages,
|
|
27798
|
+
invalidRows,
|
|
27799
|
+
capped,
|
|
27800
|
+
returnedRows: rows.length
|
|
27801
|
+
};
|
|
27802
|
+
}
|
|
27803
|
+
//#endregion
|
|
27804
|
+
//#region packages/core/dist/storage/provider-secrets.js
|
|
27805
|
+
const KEYRING_SERVICE = "seo";
|
|
27806
|
+
function account(name) {
|
|
27807
|
+
return `provider:${name}`;
|
|
27808
|
+
}
|
|
27809
|
+
function readFile$1() {
|
|
27810
|
+
return readJsonFile(getSeoCliPaths().providerSecretsFile) ?? {
|
|
27811
|
+
version: 1,
|
|
27812
|
+
secrets: {}
|
|
27813
|
+
};
|
|
27814
|
+
}
|
|
27815
|
+
async function readProviderSecret(input) {
|
|
27816
|
+
const environment = input.env ?? process.env;
|
|
27817
|
+
const environmentValue = input.envVar ? environment[input.envVar]?.trim() : void 0;
|
|
27818
|
+
if (environmentValue) return {
|
|
27819
|
+
value: environmentValue,
|
|
27820
|
+
source: "environment"
|
|
27821
|
+
};
|
|
27822
|
+
if (readConfig().security.useKeychain) try {
|
|
27823
|
+
const value = await getKeyringPassword(KEYRING_SERVICE, account(input.name));
|
|
27824
|
+
if (value) return {
|
|
27825
|
+
value,
|
|
27826
|
+
source: "keychain"
|
|
27827
|
+
};
|
|
27828
|
+
} catch {}
|
|
27829
|
+
const value = readFile$1().secrets[input.name];
|
|
27830
|
+
return value ? {
|
|
27831
|
+
value,
|
|
27832
|
+
source: "file"
|
|
27833
|
+
} : void 0;
|
|
27834
|
+
}
|
|
27835
|
+
//#endregion
|
|
27836
|
+
//#region packages/core/dist/bing/credentials.js
|
|
27837
|
+
const BING_API_KEY_SECRET = "bing-api-key";
|
|
27838
|
+
const BING_API_KEY_ENV = "SEO_BING_API_KEY";
|
|
27839
|
+
async function createBingWebmasterClient(input = {}) {
|
|
27840
|
+
const secret = await readProviderSecret({
|
|
27841
|
+
name: BING_API_KEY_SECRET,
|
|
27842
|
+
envVar: BING_API_KEY_ENV,
|
|
27843
|
+
env: input.env
|
|
27844
|
+
});
|
|
27845
|
+
if (!secret) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster is not connected. Run `seo providers bing connect`, or set SEO_BING_API_KEY for this process.");
|
|
27846
|
+
return {
|
|
27847
|
+
client: new BingWebmasterClient({ apiKey: secret.value }),
|
|
27848
|
+
credentialSource: secret.source
|
|
27849
|
+
};
|
|
27850
|
+
}
|
|
27851
|
+
//#endregion
|
|
27852
|
+
//#region packages/core/dist/bing/overview.js
|
|
27853
|
+
const DAILY_OUTPUT_ROWS = 14;
|
|
27854
|
+
function message(error) {
|
|
27855
|
+
return error instanceof Error ? error.message : String(error);
|
|
27856
|
+
}
|
|
27857
|
+
function range(rows) {
|
|
27858
|
+
return rows.length ? {
|
|
27859
|
+
startDate: rows[0]?.date ?? "",
|
|
27860
|
+
endDate: rows.at(-1)?.date ?? ""
|
|
27861
|
+
} : void 0;
|
|
27862
|
+
}
|
|
27863
|
+
function dailyOutput(data) {
|
|
27864
|
+
const rows = data.rows.slice(-14);
|
|
27865
|
+
return {
|
|
27866
|
+
...data,
|
|
27867
|
+
rows,
|
|
27868
|
+
retainedRows: data.rows.length,
|
|
27869
|
+
outputSelection: {
|
|
27870
|
+
strategy: "most-recent",
|
|
27871
|
+
availableRows: data.rows.length,
|
|
27872
|
+
returnedRows: rows.length,
|
|
27873
|
+
omittedRows: Math.max(0, data.rows.length - rows.length)
|
|
27874
|
+
}
|
|
27875
|
+
};
|
|
27876
|
+
}
|
|
27877
|
+
async function section(work, options) {
|
|
27878
|
+
try {
|
|
27879
|
+
const data = await work();
|
|
27880
|
+
const reasons = [
|
|
27881
|
+
...data.invalidRows ? [`${data.invalidRows} invalid provider rows were excluded.`] : [],
|
|
27882
|
+
...data.capped ? [`The provider response exceeded the ${options.rowLimit}-row limit.`] : [],
|
|
27883
|
+
...options.sourceWarning ? [options.sourceWarning] : []
|
|
27884
|
+
];
|
|
27885
|
+
return reasons.length ? {
|
|
27886
|
+
status: "partial",
|
|
27887
|
+
data,
|
|
27888
|
+
warning: reasons.join(" ")
|
|
27889
|
+
} : {
|
|
27890
|
+
status: "complete",
|
|
27891
|
+
data
|
|
27892
|
+
};
|
|
27893
|
+
} catch (error) {
|
|
27894
|
+
if (error instanceof SeoError && (error.code === "AUTH_REQUIRED" || error.code === "ACCESS_DENIED")) throw error;
|
|
27895
|
+
return {
|
|
27896
|
+
status: "unavailable",
|
|
27897
|
+
warning: message(error)
|
|
27898
|
+
};
|
|
27899
|
+
}
|
|
27900
|
+
}
|
|
27901
|
+
function changeFinding(input) {
|
|
27902
|
+
return {
|
|
27903
|
+
code: input.code,
|
|
27904
|
+
severity: "warning",
|
|
27905
|
+
title: input.title,
|
|
27906
|
+
evidence: {
|
|
27907
|
+
current: input.current,
|
|
27908
|
+
previous: input.previous,
|
|
27909
|
+
absoluteChange: input.absolute,
|
|
27910
|
+
percentChange: input.percent,
|
|
27911
|
+
currentDate: input.dates.current,
|
|
27912
|
+
previousDate: input.dates.previous
|
|
27913
|
+
},
|
|
27914
|
+
interpretation: input.interpretation,
|
|
27915
|
+
verification: input.verification
|
|
27916
|
+
};
|
|
27917
|
+
}
|
|
27918
|
+
function buildFindings(input) {
|
|
27919
|
+
const findings = [];
|
|
27920
|
+
const traffic = input.traffic;
|
|
27921
|
+
if (traffic && traffic.current.observedDays === traffic.current.expectedDays && traffic.previous.observedDays === traffic.previous.expectedDays && (traffic.changes.clicksPercent !== null && Math.abs(traffic.changes.clicksPercent) >= 10 || traffic.changes.impressionsPercent !== null && Math.abs(traffic.changes.impressionsPercent) >= 10)) {
|
|
27922
|
+
const metric = traffic.changes.clicksPercent !== null && Math.abs(traffic.changes.clicksPercent) >= 10 ? "clicks" : "impressions";
|
|
27923
|
+
const declined = traffic.changes[metric] < 0;
|
|
27924
|
+
findings.push({
|
|
27925
|
+
code: `bing_${metric}_${declined ? "declined" : "increased"}`,
|
|
27926
|
+
severity: declined ? "warning" : "info",
|
|
27927
|
+
title: `Bing ${metric} ${declined ? "declined" : "increased"} in the latest complete period`,
|
|
27928
|
+
evidence: {
|
|
27929
|
+
currentClicks: traffic.current.clicks,
|
|
27930
|
+
previousClicks: traffic.previous.clicks,
|
|
27931
|
+
clicksChange: traffic.changes.clicks,
|
|
27932
|
+
clicksPercentChange: traffic.changes.clicksPercent,
|
|
27933
|
+
currentImpressions: traffic.current.impressions,
|
|
27934
|
+
previousImpressions: traffic.previous.impressions,
|
|
27935
|
+
impressionsChange: traffic.changes.impressions,
|
|
27936
|
+
impressionsPercentChange: traffic.changes.impressionsPercent,
|
|
27937
|
+
ctrPercentagePointChange: traffic.changes.ctrPercentagePoints,
|
|
27938
|
+
currentRange: {
|
|
27939
|
+
startDate: traffic.current.startDate,
|
|
27940
|
+
endDate: traffic.current.endDate
|
|
27941
|
+
},
|
|
27942
|
+
previousRange: {
|
|
27943
|
+
startDate: traffic.previous.startDate,
|
|
27944
|
+
endDate: traffic.previous.endDate
|
|
27945
|
+
}
|
|
27946
|
+
},
|
|
27947
|
+
interpretation: "This is a Bing-observed traffic change. It does not establish the cause or describe Google traffic.",
|
|
27948
|
+
verification: "Review the query and page evidence for the same dates, then compare current pages and crawl health before changing content."
|
|
27949
|
+
});
|
|
27950
|
+
}
|
|
27951
|
+
const crawl = input.crawl;
|
|
27952
|
+
if (crawl?.previous) {
|
|
27953
|
+
const dates = {
|
|
27954
|
+
current: crawl.current.date,
|
|
27955
|
+
previous: crawl.previous.date
|
|
27956
|
+
};
|
|
27957
|
+
const errors = crawl.changes.crawlErrors;
|
|
27958
|
+
if (errors && errors.absolute >= Math.max(10, Math.ceil(errors.previous * .5))) findings.push(changeFinding({
|
|
27959
|
+
code: "bing_crawl_errors_increased",
|
|
27960
|
+
title: "Bing crawl errors increased",
|
|
27961
|
+
...errors,
|
|
27962
|
+
dates,
|
|
27963
|
+
interpretation: "Bing reported more crawl errors in its latest daily snapshot. The count may include URLs outside the current sitemap.",
|
|
27964
|
+
verification: "Run the fast sitemap health check, inspect Bing crawl issue samples, then audit any affected URLs directly."
|
|
27965
|
+
}));
|
|
27966
|
+
const code4xx = crawl.changes.code4xx;
|
|
27967
|
+
if (code4xx && code4xx.absolute >= Math.max(10, Math.ceil(code4xx.previous * .5))) findings.push(changeFinding({
|
|
27968
|
+
code: "bing_4xx_increased",
|
|
27969
|
+
title: "Bing-observed 4xx responses increased",
|
|
27970
|
+
...code4xx,
|
|
27971
|
+
dates,
|
|
27972
|
+
interpretation: "Bing encountered more 4xx responses. Intentional removals and obsolete discovered URLs can contribute to this count.",
|
|
27973
|
+
verification: "Inspect Bing crawl issue URLs and recover or redirect only URLs that should still exist or retain search value."
|
|
27974
|
+
}));
|
|
27975
|
+
const code5xx = crawl.changes.code5xx;
|
|
27976
|
+
if (code5xx && code5xx.current > 0 && code5xx.absolute > 0) findings.push(changeFinding({
|
|
27977
|
+
code: "bing_5xx_increased",
|
|
27978
|
+
title: "Bing-observed server errors increased",
|
|
27979
|
+
...code5xx,
|
|
27980
|
+
dates,
|
|
27981
|
+
interpretation: "Bing encountered more server errors in the latest snapshot. Even a small increase deserves a live check.",
|
|
27982
|
+
verification: "Check current server logs and run targeted URL audits before treating the provider count as an active outage."
|
|
27983
|
+
}));
|
|
27984
|
+
}
|
|
27985
|
+
for (const [kind, analysis] of [["query", input.queries], ["page", input.pages]]) {
|
|
27986
|
+
if (!analysis?.opportunities.length) continue;
|
|
27987
|
+
findings.push({
|
|
27988
|
+
code: `bing_${kind}_opportunities`,
|
|
27989
|
+
severity: "review",
|
|
27990
|
+
title: `${analysis.opportunities.length} Bing ${kind} opportunities need review`,
|
|
27991
|
+
evidence: {
|
|
27992
|
+
retainedOpportunities: analysis.opportunities.length,
|
|
27993
|
+
currentDimensions: analysis.coverage.currentDimensions,
|
|
27994
|
+
matchedDimensions: analysis.coverage.matchedDimensions,
|
|
27995
|
+
comparableDimensions: analysis.coverage.comparableDimensions,
|
|
27996
|
+
thresholds: analysis.thresholds
|
|
27997
|
+
},
|
|
27998
|
+
interpretation: "These are high-impression top-list entries currently observed around positions 4 to 20. The threshold is a prioritization heuristic, not a search-engine rule.",
|
|
27999
|
+
verification: kind === "query" ? "Check intent and the ranking page before changing copy or creating content." : "Audit the selected page and its matching queries before changing internal links, titles, or content."
|
|
28000
|
+
});
|
|
28001
|
+
}
|
|
28002
|
+
return findings;
|
|
28003
|
+
}
|
|
28004
|
+
async function bingWebmasterOverview(input) {
|
|
28005
|
+
if (!input.site.trim()) throw new SeoError("INVALID_INPUT", "Pass a Bing Webmaster site URL.");
|
|
28006
|
+
const resolved = input.client ? {
|
|
28007
|
+
client: input.client,
|
|
28008
|
+
credentialSource: input.credentialSource ?? "environment"
|
|
28009
|
+
} : await createBingWebmasterClient();
|
|
28010
|
+
const observedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
28011
|
+
const topListWarning = "Bing returns weekly top-list rows, so dimensions missing from a period remain unknown.";
|
|
28012
|
+
const [traffic, crawl, queries, pages] = await Promise.all([
|
|
28013
|
+
section(() => resolved.client.getTraffic(input.site), { rowLimit: 400 }),
|
|
28014
|
+
section(() => resolved.client.getCrawlStats(input.site), { rowLimit: 400 }),
|
|
28015
|
+
section(() => resolved.client.getQueryStats(input.site), {
|
|
28016
|
+
rowLimit: 8e3,
|
|
28017
|
+
sourceWarning: topListWarning
|
|
28018
|
+
}),
|
|
28019
|
+
section(() => resolved.client.getPageStats(input.site), {
|
|
28020
|
+
rowLimit: 8e3,
|
|
28021
|
+
sourceWarning: topListWarning
|
|
28022
|
+
})
|
|
28023
|
+
]);
|
|
28024
|
+
const statuses = [
|
|
28025
|
+
traffic.status,
|
|
28026
|
+
crawl.status,
|
|
28027
|
+
queries.status,
|
|
28028
|
+
pages.status
|
|
28029
|
+
];
|
|
28030
|
+
const dataStatus = statuses.every((status) => status === "complete") ? "complete" : statuses.every((status) => status === "unavailable") ? "unavailable" : "partial";
|
|
28031
|
+
const trafficAnalysis = traffic.status === "unavailable" ? void 0 : analyzeBingTraffic(traffic.data.rows);
|
|
28032
|
+
const crawlAnalysis = crawl.status === "unavailable" ? void 0 : analyzeBingCrawl(crawl.data.rows);
|
|
28033
|
+
const queryAnalysis = queries.status === "unavailable" ? void 0 : analyzeBingDimensions(queries.data.rows, "query");
|
|
28034
|
+
const pageAnalysis = pages.status === "unavailable" ? void 0 : analyzeBingDimensions(pages.data.rows, "page");
|
|
28035
|
+
const findings = buildFindings({
|
|
28036
|
+
traffic: trafficAnalysis,
|
|
28037
|
+
crawl: crawlAnalysis,
|
|
28038
|
+
queries: queryAnalysis,
|
|
28039
|
+
pages: pageAnalysis
|
|
28040
|
+
});
|
|
28041
|
+
return {
|
|
28042
|
+
schemaVersion: 2,
|
|
28043
|
+
site: input.site,
|
|
28044
|
+
generatedAt: observedAt,
|
|
28045
|
+
dataStatus,
|
|
28046
|
+
summary: {
|
|
28047
|
+
findings: findings.length,
|
|
28048
|
+
warnings: findings.filter((finding) => finding.severity === "warning").length,
|
|
28049
|
+
reviewItems: findings.filter((finding) => finding.severity === "review").length
|
|
28050
|
+
},
|
|
28051
|
+
findings,
|
|
28052
|
+
outputBudget: {
|
|
28053
|
+
maxFindings: 6,
|
|
28054
|
+
maxDailyRowsPerSection: DAILY_OUTPUT_ROWS,
|
|
28055
|
+
maxDailyRowsTotal: DAILY_OUTPUT_ROWS * 2,
|
|
28056
|
+
maxDimensionItemsPerList: 10,
|
|
28057
|
+
maxDimensionItemsTotal: 40
|
|
28058
|
+
},
|
|
28059
|
+
provenance: {
|
|
28060
|
+
provider: "bing-webmaster",
|
|
28061
|
+
authentication: resolved.client.authentication,
|
|
28062
|
+
credentialSource: resolved.credentialSource,
|
|
28063
|
+
observedAt,
|
|
28064
|
+
cached: false,
|
|
28065
|
+
rowLimits: {
|
|
28066
|
+
traffic: 400,
|
|
28067
|
+
crawl: 400,
|
|
28068
|
+
queries: 8e3,
|
|
28069
|
+
pages: 8e3
|
|
28070
|
+
},
|
|
28071
|
+
methods: [
|
|
28072
|
+
"GetRankAndTrafficStats",
|
|
28073
|
+
"GetCrawlStats",
|
|
28074
|
+
"GetQueryStats",
|
|
28075
|
+
"GetPageStats"
|
|
28076
|
+
]
|
|
28077
|
+
},
|
|
28078
|
+
traffic: traffic.status === "unavailable" ? traffic : {
|
|
28079
|
+
...traffic,
|
|
28080
|
+
data: {
|
|
28081
|
+
...dailyOutput(traffic.data),
|
|
28082
|
+
range: range(traffic.data.rows),
|
|
28083
|
+
clicks: traffic.data.rows.reduce((total, row) => total + row.clicks, 0),
|
|
28084
|
+
impressions: traffic.data.rows.reduce((total, row) => total + row.impressions, 0),
|
|
28085
|
+
analysis: trafficAnalysis
|
|
28086
|
+
}
|
|
28087
|
+
},
|
|
28088
|
+
crawl: crawl.status === "unavailable" ? crawl : {
|
|
28089
|
+
...crawl,
|
|
28090
|
+
data: {
|
|
28091
|
+
...dailyOutput(crawl.data),
|
|
28092
|
+
range: range(crawl.data.rows),
|
|
28093
|
+
latest: crawl.data.rows.at(-1),
|
|
28094
|
+
analysis: crawlAnalysis
|
|
28095
|
+
}
|
|
28096
|
+
},
|
|
28097
|
+
queries: queries.status === "unavailable" ? queries : {
|
|
28098
|
+
...queries,
|
|
28099
|
+
data: {
|
|
28100
|
+
returnedRows: queries.data.returnedRows,
|
|
28101
|
+
retainedRows: queries.data.rows.length,
|
|
28102
|
+
invalidRows: queries.data.invalidRows,
|
|
28103
|
+
capped: queries.data.capped,
|
|
28104
|
+
range: range(queries.data.rows),
|
|
28105
|
+
analysis: queryAnalysis
|
|
28106
|
+
}
|
|
28107
|
+
},
|
|
28108
|
+
pages: pages.status === "unavailable" ? pages : {
|
|
28109
|
+
...pages,
|
|
28110
|
+
data: {
|
|
28111
|
+
returnedRows: pages.data.returnedRows,
|
|
28112
|
+
retainedRows: pages.data.rows.length,
|
|
28113
|
+
invalidRows: pages.data.invalidRows,
|
|
28114
|
+
capped: pages.data.capped,
|
|
28115
|
+
range: range(pages.data.rows),
|
|
28116
|
+
analysis: pageAnalysis
|
|
28117
|
+
}
|
|
28118
|
+
},
|
|
28119
|
+
caveats: [
|
|
28120
|
+
"Bing reports its own observed search and crawl evidence. It is not a complete view of every search engine.",
|
|
28121
|
+
"The inIndex field is provider-reported crawl statistics, not independent proof that a URL is indexed.",
|
|
28122
|
+
"Query and page movements include only dimensions observed in every weekly top list in both periods. Missing entries are not treated as zero.",
|
|
28123
|
+
"Opportunity thresholds prioritize review. They are heuristics, not search-engine requirements."
|
|
28124
|
+
]
|
|
28125
|
+
};
|
|
28126
|
+
}
|
|
28127
|
+
//#endregion
|
|
28128
|
+
//#region packages/core/dist/doctor.js
|
|
28129
|
+
const REQUIRED_SCOPES = ["https://www.googleapis.com/auth/webmasters.readonly", "https://www.googleapis.com/auth/analytics.readonly"];
|
|
28130
|
+
async function runDoctor(dependencies = {}) {
|
|
28131
|
+
const paths = getSeoCliPaths();
|
|
28132
|
+
const config = readConfig();
|
|
28133
|
+
const authMode = getAuthModeStatus();
|
|
28134
|
+
const serviceAccount = getServiceAccountStatus();
|
|
28135
|
+
const tokens = await readTokens();
|
|
28136
|
+
const oauthClient = readOauthClient();
|
|
28137
|
+
const tokenClientConfigured = tokens ? Boolean(getClientConfig(tokens.client_source)) : void 0;
|
|
28138
|
+
const checks = [];
|
|
28139
|
+
try {
|
|
28140
|
+
const database = (dependencies.checkDatabase ?? checkDatabaseReadiness)();
|
|
28141
|
+
checks.push({
|
|
28142
|
+
id: "local-database",
|
|
28143
|
+
label: "Local database",
|
|
28144
|
+
status: "pass",
|
|
28145
|
+
detail: database.dbPath
|
|
28146
|
+
});
|
|
28147
|
+
} catch (error) {
|
|
28148
|
+
checks.push({
|
|
28149
|
+
id: "local-database",
|
|
28150
|
+
label: "Local database",
|
|
28151
|
+
status: "fail",
|
|
28152
|
+
detail: error instanceof Error ? error.message : "The local database could not be opened.",
|
|
28153
|
+
fix: "Upgrade or reinstall `seo`. If this still fails, report it at https://github.com/iannuttall/seo/issues."
|
|
28154
|
+
});
|
|
28155
|
+
}
|
|
28156
|
+
checks.push({
|
|
28157
|
+
id: "config-dir",
|
|
28158
|
+
label: "Config directory",
|
|
28159
|
+
status: existsSync(paths.configDir) ? "pass" : "warn",
|
|
28160
|
+
detail: paths.configDir,
|
|
28161
|
+
fix: existsSync(paths.configDir) ? void 0 : "Run `seo start` or `seo auth setup-client`."
|
|
28162
|
+
});
|
|
28163
|
+
checks.push({
|
|
28164
|
+
id: "oauth-client",
|
|
28165
|
+
label: "Google credentials",
|
|
28166
|
+
status: serviceAccount.configured ? "pass" : tokenClientConfigured === false ? "fail" : serviceAccount.error ? "fail" : authMode.sharedConfigured || authMode.byoConfigured ? "pass" : "fail",
|
|
28167
|
+
detail: serviceAccount.configured ? `Service account configured for ${serviceAccount.identity}.` : tokenClientConfigured === false ? `Stored Google login uses the ${tokens?.client_source === "shared" ? "shared seo app" : "BYO client"}, but that client is not configured.` : serviceAccount.error ? serviceAccount.error : authMode.sharedConfigured ? "Shared client configured." : authMode.byoConfigured ? `BYO client configured at ${paths.oauthClientFile}.` : "No shared or BYO Google OAuth client configured.",
|
|
28168
|
+
fix: serviceAccount.configured ? void 0 : tokenClientConfigured === false ? tokens?.client_source === "shared" ? "Reinstall `seo`. If the shared client is still missing, report it at https://github.com/iannuttall/seo/issues." : "Restore the same client with `seo auth setup-client`, or run `seo auth logout` before signing in with a different client." : serviceAccount.error ? "Set one valid service account credential source, then run `seo doctor` again." : authMode.sharedConfigured || authMode.byoConfigured ? void 0 : "Run `seo auth setup-client` or set SEO_GOOGLE_CLIENT_ID and SEO_GOOGLE_CLIENT_SECRET."
|
|
28169
|
+
});
|
|
28170
|
+
checks.push({
|
|
28171
|
+
id: "oauth-client-file",
|
|
28172
|
+
label: "BYO OAuth file",
|
|
28173
|
+
status: oauthClient ? "pass" : "warn",
|
|
28174
|
+
detail: oauthClient ? paths.oauthClientFile : "No BYO OAuth client file. This is fine if using a shared client."
|
|
28175
|
+
});
|
|
28176
|
+
checks.push({
|
|
28177
|
+
id: "google-login",
|
|
28178
|
+
label: "Google login",
|
|
28179
|
+
status: serviceAccount.configured || tokens ? "pass" : "fail",
|
|
28180
|
+
detail: serviceAccount.configured ? `Using service account ${serviceAccount.identity}.` : tokens ? `Signed in as ${tokens.account_email}.` : "No Google token found.",
|
|
28181
|
+
fix: serviceAccount.configured || tokens ? void 0 : "Run `seo auth login`, or configure a service account for CI."
|
|
28182
|
+
});
|
|
28183
|
+
const grantedScopes = new Set(tokens?.scope.split(/\s+/).filter(Boolean) ?? []);
|
|
28184
|
+
for (const scope of REQUIRED_SCOPES) checks.push({
|
|
28185
|
+
id: `scope:${scope}`,
|
|
28186
|
+
label: scope.endsWith("webmasters.readonly") ? "Search Console scope" : "Google Analytics scope",
|
|
28187
|
+
status: serviceAccount.configured || tokens && grantedScopes.has(scope) ? "pass" : tokens ? "fail" : "warn",
|
|
28188
|
+
detail: serviceAccount.configured ? `${scope} requested by the service account.` : tokens ? grantedScopes.has(scope) ? scope : `Missing ${scope}.` : "Cannot inspect scopes until login is complete.",
|
|
28189
|
+
fix: !serviceAccount.configured && tokens && !grantedScopes.has(scope) ? "Run `seo auth logout` then `seo auth login`." : void 0
|
|
28190
|
+
});
|
|
28191
|
+
checks.push({
|
|
28192
|
+
id: "default-site",
|
|
28193
|
+
label: "Saved GSC property",
|
|
28194
|
+
status: "pass",
|
|
28195
|
+
detail: config.defaultSite ?? "No saved default. Human CLI commands will prompt; agents can pass --site."
|
|
28196
|
+
});
|
|
28197
|
+
checks.push({
|
|
28198
|
+
id: "default-google-analytics",
|
|
28199
|
+
label: "Saved Google Analytics property",
|
|
28200
|
+
status: "pass",
|
|
28201
|
+
detail: config.analytics.google.defaultPropertyId ?? "No saved default. Human CLI commands will prompt; agents can pass --property."
|
|
28202
|
+
});
|
|
28203
|
+
return {
|
|
28204
|
+
ok: checks.every((check) => check.status !== "fail"),
|
|
28205
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
28206
|
+
checks
|
|
28207
|
+
};
|
|
28208
|
+
}
|
|
28209
|
+
//#endregion
|
|
28210
|
+
//#region packages/core/dist/links/normalize.js
|
|
28211
|
+
const FIELD_ALIASES = {
|
|
28212
|
+
sourceUrl: [
|
|
28213
|
+
"sourceurl",
|
|
28214
|
+
"source",
|
|
28215
|
+
"referringurl",
|
|
28216
|
+
"referringpage",
|
|
28217
|
+
"fromurl"
|
|
28218
|
+
],
|
|
28219
|
+
targetUrl: [
|
|
28220
|
+
"targeturl",
|
|
28221
|
+
"target",
|
|
28222
|
+
"destinationurl",
|
|
28223
|
+
"destination",
|
|
28224
|
+
"tourl"
|
|
28225
|
+
],
|
|
28226
|
+
anchorText: ["anchortext", "anchor"],
|
|
28227
|
+
firstSeenAt: ["firstseenat", "firstseen"],
|
|
28228
|
+
lastSeenAt: ["lastseenat", "lastseen"],
|
|
28229
|
+
nofollow: ["nofollow", "isnofollow"]
|
|
28230
|
+
};
|
|
28231
|
+
function key(value) {
|
|
28232
|
+
return value.toLowerCase().replaceAll(/[^a-z0-9]/g, "");
|
|
28233
|
+
}
|
|
28234
|
+
function valueFor(row, field) {
|
|
28235
|
+
const aliases = new Set(FIELD_ALIASES[field]);
|
|
28236
|
+
for (const [name, value] of Object.entries(row)) if (aliases.has(key(name))) return value;
|
|
28237
|
+
}
|
|
28238
|
+
function httpUrl(value) {
|
|
28239
|
+
if (typeof value !== "string" || !value.trim() || value.trim().length > 2e3) return;
|
|
28240
|
+
try {
|
|
28241
|
+
const url = new URL(value.trim());
|
|
28242
|
+
return url.protocol === "http:" || url.protocol === "https:" ? url : void 0;
|
|
28243
|
+
} catch {
|
|
28244
|
+
return;
|
|
28245
|
+
}
|
|
28246
|
+
}
|
|
28247
|
+
function optionalString(value) {
|
|
28248
|
+
return typeof value === "string" && value.trim() ? value.trim().slice(0, 1e3) : void 0;
|
|
28249
|
+
}
|
|
28250
|
+
function optionalBoolean(value) {
|
|
28251
|
+
if (typeof value === "boolean") return value;
|
|
28252
|
+
if (typeof value !== "string") return void 0;
|
|
28253
|
+
if (/^(true|yes|1)$/i.test(value.trim())) return true;
|
|
28254
|
+
if (/^(false|no|0)$/i.test(value.trim())) return false;
|
|
28255
|
+
}
|
|
28256
|
+
function normalizeLinkEvidenceRow(raw) {
|
|
28257
|
+
const source = httpUrl(valueFor(raw, "sourceUrl"));
|
|
28258
|
+
const target = httpUrl(valueFor(raw, "targetUrl"));
|
|
28259
|
+
if (!source || !target) return void 0;
|
|
28260
|
+
return {
|
|
28261
|
+
sourceUrl: source.toString(),
|
|
28262
|
+
sourceDomain: source.hostname.toLowerCase(),
|
|
28263
|
+
targetUrl: target.toString(),
|
|
28264
|
+
anchorText: optionalString(valueFor(raw, "anchorText")),
|
|
28265
|
+
firstSeenAt: optionalString(valueFor(raw, "firstSeenAt")),
|
|
28266
|
+
lastSeenAt: optionalString(valueFor(raw, "lastSeenAt")),
|
|
28267
|
+
nofollow: optionalBoolean(valueFor(raw, "nofollow"))
|
|
28268
|
+
};
|
|
28269
|
+
}
|
|
28270
|
+
function linkEvidenceKey(row) {
|
|
28271
|
+
return `${row.sourceUrl}\u0000${row.targetUrl}\u0000${row.anchorText ?? ""}`;
|
|
28272
|
+
}
|
|
28273
|
+
function compareLinkEvidence(a, b) {
|
|
28274
|
+
return a.targetUrl.localeCompare(b.targetUrl, "en") || a.sourceUrl.localeCompare(b.sourceUrl, "en") || (a.anchorText ?? "").localeCompare(b.anchorText ?? "", "en");
|
|
28275
|
+
}
|
|
28276
|
+
//#endregion
|
|
28277
|
+
//#region packages/core/dist/links/bing.js
|
|
28278
|
+
const MAX_COUNT_PAGES = 5;
|
|
28279
|
+
const DEFAULT_TARGET_LIMIT = 20;
|
|
28280
|
+
const MAX_TARGET_LIMIT = 50;
|
|
28281
|
+
const DEFAULT_DETAIL_PAGES = 1;
|
|
28282
|
+
const MAX_DETAIL_PAGES = 3;
|
|
28283
|
+
const DEFAULT_ROW_LIMIT$2 = 500;
|
|
28284
|
+
const MAX_ROW_LIMIT$2 = 1e3;
|
|
28285
|
+
const MAX_CONCURRENCY = 3;
|
|
28286
|
+
function integer(value, fallback, max, name) {
|
|
28287
|
+
const resolved = value ?? fallback;
|
|
28288
|
+
if (!Number.isInteger(resolved) || resolved < 1 || resolved > max) throw new SeoError("INVALID_INPUT", `${name} must be between 1 and ${max}.`);
|
|
28289
|
+
return resolved;
|
|
28290
|
+
}
|
|
28291
|
+
async function mapConcurrent(values, concurrency, work) {
|
|
28292
|
+
const output = new Array(values.length);
|
|
28293
|
+
let next = 0;
|
|
28294
|
+
async function worker() {
|
|
28295
|
+
while (next < values.length) {
|
|
28296
|
+
const index = next;
|
|
28297
|
+
next += 1;
|
|
28298
|
+
const value = values[index];
|
|
28299
|
+
if (value !== void 0) output[index] = await work(value);
|
|
28300
|
+
}
|
|
28301
|
+
}
|
|
28302
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, () => worker()));
|
|
28303
|
+
return output;
|
|
28304
|
+
}
|
|
28305
|
+
async function targetDetails(input) {
|
|
28306
|
+
const rows = [];
|
|
28307
|
+
let requestedPages = 0;
|
|
28308
|
+
let invalidRows = 0;
|
|
28309
|
+
let returnedRows = 0;
|
|
28310
|
+
let partial = false;
|
|
28311
|
+
let totalPages = 0;
|
|
28312
|
+
for (let page = 0; page < input.pages; page += 1) {
|
|
28313
|
+
requestedPages += 1;
|
|
28314
|
+
let response;
|
|
28315
|
+
try {
|
|
28316
|
+
response = await input.client.getUrlLinks(input.site, input.target.url, page);
|
|
28317
|
+
} catch (error) {
|
|
28318
|
+
if (error instanceof SeoError && (error.code === "AUTH_REQUIRED" || error.code === "ACCESS_DENIED")) throw error;
|
|
28319
|
+
return {
|
|
28320
|
+
rows,
|
|
28321
|
+
requestedPages,
|
|
28322
|
+
invalidRows,
|
|
28323
|
+
returnedRows,
|
|
28324
|
+
partial: true,
|
|
28325
|
+
failed: true
|
|
28326
|
+
};
|
|
28327
|
+
}
|
|
28328
|
+
invalidRows += response.invalidRows;
|
|
28329
|
+
returnedRows += response.returnedRows;
|
|
28330
|
+
rows.push(...response.rows);
|
|
28331
|
+
totalPages = response.totalPages;
|
|
28332
|
+
partial ||= response.capped;
|
|
28333
|
+
if (page + 1 >= response.totalPages || response.rows.length === 0) break;
|
|
28334
|
+
}
|
|
28335
|
+
partial ||= requestedPages < totalPages;
|
|
28336
|
+
return {
|
|
28337
|
+
rows,
|
|
28338
|
+
requestedPages,
|
|
28339
|
+
invalidRows,
|
|
28340
|
+
returnedRows,
|
|
28341
|
+
partial,
|
|
28342
|
+
failed: false
|
|
28343
|
+
};
|
|
28344
|
+
}
|
|
28345
|
+
async function collectBingLinkEvidence(input) {
|
|
28346
|
+
if (!input.site.trim()) throw new SeoError("INVALID_INPUT", "Pass a Bing Webmaster site URL.");
|
|
28347
|
+
const targetLimit = integer(input.targetLimit, DEFAULT_TARGET_LIMIT, MAX_TARGET_LIMIT, "Bing link target limit");
|
|
28348
|
+
const detailPages = integer(input.detailPagesPerTarget, DEFAULT_DETAIL_PAGES, MAX_DETAIL_PAGES, "Bing detail page limit");
|
|
28349
|
+
const rowLimit = integer(input.rowLimit, DEFAULT_ROW_LIMIT$2, MAX_ROW_LIMIT$2, "Bing link row limit");
|
|
28350
|
+
const client = input.client ?? (await createBingWebmasterClient()).client;
|
|
28351
|
+
const countRows = [];
|
|
28352
|
+
let countPagesRequested = 0;
|
|
28353
|
+
let invalidRows = 0;
|
|
28354
|
+
let countInvalidRows = 0;
|
|
28355
|
+
let countRowsReturned = 0;
|
|
28356
|
+
let partial = false;
|
|
28357
|
+
let totalCountPages = 0;
|
|
28358
|
+
for (let page = 0; page < MAX_COUNT_PAGES; page += 1) {
|
|
28359
|
+
const response = await client.getLinkCounts(input.site, page);
|
|
28360
|
+
countPagesRequested += 1;
|
|
28361
|
+
invalidRows += response.invalidRows;
|
|
28362
|
+
countInvalidRows += response.invalidRows;
|
|
28363
|
+
countRowsReturned += response.returnedRows;
|
|
28364
|
+
countRows.push(...response.rows);
|
|
28365
|
+
totalCountPages = response.totalPages;
|
|
28366
|
+
partial ||= response.capped;
|
|
28367
|
+
if (countRows.length >= targetLimit || page + 1 >= response.totalPages || response.rows.length === 0) break;
|
|
28368
|
+
}
|
|
28369
|
+
partial ||= countPagesRequested < totalCountPages;
|
|
28370
|
+
const countsByUrl = /* @__PURE__ */ new Map();
|
|
28371
|
+
for (const row of countRows) countsByUrl.set(row.url, Math.max(row.count, countsByUrl.get(row.url) ?? 0));
|
|
28372
|
+
const counts = [...countsByUrl].map(([url, count]) => ({
|
|
28373
|
+
url,
|
|
28374
|
+
count
|
|
28375
|
+
}));
|
|
28376
|
+
counts.sort((a, b) => b.count - a.count || a.url.localeCompare(b.url, "en"));
|
|
28377
|
+
const selectedTargets = counts.slice(0, targetLimit);
|
|
28378
|
+
partial ||= counts.length > targetLimit;
|
|
28379
|
+
const details = await mapConcurrent(selectedTargets, MAX_CONCURRENCY, async (target) => targetDetails({
|
|
28380
|
+
client,
|
|
28381
|
+
site: input.site,
|
|
28382
|
+
target,
|
|
28383
|
+
pages: detailPages
|
|
28384
|
+
}));
|
|
28385
|
+
const rows = [];
|
|
28386
|
+
const keys = /* @__PURE__ */ new Set();
|
|
28387
|
+
let duplicateRows = 0;
|
|
28388
|
+
let detailPagesRequested = 0;
|
|
28389
|
+
let detailInvalidRows = 0;
|
|
28390
|
+
let detailRowsReturned = 0;
|
|
28391
|
+
let failedTargets = 0;
|
|
28392
|
+
for (const [index, detail] of details.entries()) {
|
|
28393
|
+
const target = selectedTargets[index];
|
|
28394
|
+
if (!target) continue;
|
|
28395
|
+
detailPagesRequested += detail.requestedPages;
|
|
28396
|
+
if (detail.failed) failedTargets += 1;
|
|
28397
|
+
invalidRows += detail.invalidRows;
|
|
28398
|
+
detailInvalidRows += detail.invalidRows;
|
|
28399
|
+
detailRowsReturned += detail.returnedRows;
|
|
28400
|
+
partial ||= detail.partial;
|
|
28401
|
+
for (const source of detail.rows) {
|
|
28402
|
+
if (rows.length >= rowLimit) {
|
|
28403
|
+
partial = true;
|
|
28404
|
+
break;
|
|
28405
|
+
}
|
|
28406
|
+
const row = normalizeLinkEvidenceRow({
|
|
28407
|
+
sourceUrl: source.url,
|
|
28408
|
+
targetUrl: target.url,
|
|
28409
|
+
anchorText: source.anchorText
|
|
28410
|
+
});
|
|
28411
|
+
if (!row) {
|
|
28412
|
+
invalidRows += 1;
|
|
28413
|
+
continue;
|
|
28414
|
+
}
|
|
28415
|
+
const key = linkEvidenceKey(row);
|
|
28416
|
+
if (keys.has(key)) {
|
|
28417
|
+
duplicateRows += 1;
|
|
28418
|
+
continue;
|
|
28419
|
+
}
|
|
28420
|
+
keys.add(key);
|
|
28421
|
+
rows.push(row);
|
|
28422
|
+
}
|
|
28423
|
+
}
|
|
28424
|
+
rows.sort(compareLinkEvidence);
|
|
28425
|
+
return {
|
|
28426
|
+
rows,
|
|
28427
|
+
targetCounts: selectedTargets.map((target) => ({
|
|
28428
|
+
targetUrl: target.url,
|
|
28429
|
+
providerReportedLinks: target.count,
|
|
28430
|
+
observedLinks: rows.filter((row) => row.targetUrl === target.url).length
|
|
28431
|
+
})),
|
|
28432
|
+
provenance: {
|
|
28433
|
+
provider: "bing-webmaster",
|
|
28434
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
28435
|
+
cached: false,
|
|
28436
|
+
suppliedRows: detailRowsReturned + detailInvalidRows,
|
|
28437
|
+
validRows: rows.length,
|
|
28438
|
+
invalidRows,
|
|
28439
|
+
duplicateRows,
|
|
28440
|
+
capped: partial,
|
|
28441
|
+
rowLimit,
|
|
28442
|
+
completeness: partial ? "partial" : "unknown",
|
|
28443
|
+
providerRequests: {
|
|
28444
|
+
methods: ["GetLinkCounts", "GetUrlLinks"],
|
|
28445
|
+
targetPagesRequested: countPagesRequested,
|
|
28446
|
+
detailPagesRequested,
|
|
28447
|
+
maxConcurrentRequests: MAX_CONCURRENCY
|
|
28448
|
+
},
|
|
28449
|
+
providerCoverage: {
|
|
28450
|
+
targetCountRows: {
|
|
28451
|
+
returnedRows: countRowsReturned,
|
|
28452
|
+
retainedRows: counts.length,
|
|
28453
|
+
invalidRows: countInvalidRows
|
|
28454
|
+
},
|
|
28455
|
+
detailRows: {
|
|
28456
|
+
returnedRows: detailRowsReturned,
|
|
28457
|
+
retainedRows: rows.length,
|
|
28458
|
+
invalidRows: detailInvalidRows
|
|
28459
|
+
}
|
|
28460
|
+
}
|
|
28461
|
+
},
|
|
28462
|
+
warnings: [
|
|
28463
|
+
...countRowsReturned === 0 ? ["Bing returned no target-link rows. This does not prove that the site has no links."] : [],
|
|
28464
|
+
...partial ? ["Bing link evidence was bounded by provider pagination, failed requests, or report limits."] : [],
|
|
28465
|
+
...failedTargets ? [`${failedTargets} target detail request${failedTargets === 1 ? "" : "s"} failed while other evidence was retained.`] : []
|
|
28466
|
+
]
|
|
28467
|
+
};
|
|
28468
|
+
}
|
|
28469
|
+
//#endregion
|
|
28470
|
+
//#region packages/core/dist/links/import.js
|
|
28471
|
+
const DEFAULT_ROW_LIMIT$1 = 1e4;
|
|
28472
|
+
const MAX_ROW_LIMIT$1 = 1e5;
|
|
28473
|
+
const MAX_STREAM_BYTES = 5e7;
|
|
28474
|
+
const MAX_JSON_BYTES = 1e7;
|
|
28475
|
+
const MAX_CELL_BYTES = 1e6;
|
|
28476
|
+
function formatFor$1(path, explicit) {
|
|
28477
|
+
if (explicit) return explicit;
|
|
28478
|
+
const extension = extname(path).toLowerCase();
|
|
28479
|
+
if (extension === ".csv") return "csv";
|
|
28480
|
+
if (extension === ".jsonl" || extension === ".ndjson") return "jsonl";
|
|
28481
|
+
if (extension === ".json") return "json";
|
|
28482
|
+
throw new SeoError("INVALID_INPUT", "Link imports must be CSV, JSON, JSONL, or NDJSON.");
|
|
28483
|
+
}
|
|
28484
|
+
function boundedRowLimit(value) {
|
|
28485
|
+
const limit = value ?? DEFAULT_ROW_LIMIT$1;
|
|
28486
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_ROW_LIMIT$1) throw new SeoError("INVALID_INPUT", `Link import row limit must be between 1 and ${MAX_ROW_LIMIT$1}.`);
|
|
28487
|
+
return limit;
|
|
28488
|
+
}
|
|
28489
|
+
async function* csvRecords(path, byteLimit, progress) {
|
|
28490
|
+
const stream = createReadStream(path, { encoding: "utf8" });
|
|
28491
|
+
let record = [];
|
|
28492
|
+
let field = "";
|
|
28493
|
+
let quoted = false;
|
|
28494
|
+
let pendingQuote = false;
|
|
28495
|
+
for await (const chunk of stream) {
|
|
28496
|
+
progress.bytesRead += Buffer.byteLength(chunk);
|
|
28497
|
+
if (progress.bytesRead > byteLimit) {
|
|
28498
|
+
stream.destroy();
|
|
28499
|
+
throw new SeoError("INVALID_INPUT", `Link import exceeds the ${byteLimit}-byte streaming limit.`);
|
|
28500
|
+
}
|
|
28501
|
+
for (const character of chunk) {
|
|
28502
|
+
if (pendingQuote) {
|
|
28503
|
+
if (character === "\"") {
|
|
28504
|
+
field += "\"";
|
|
28505
|
+
pendingQuote = false;
|
|
28506
|
+
continue;
|
|
28507
|
+
}
|
|
28508
|
+
quoted = false;
|
|
28509
|
+
pendingQuote = false;
|
|
28510
|
+
}
|
|
28511
|
+
if (quoted) if (character === "\"") pendingQuote = true;
|
|
28512
|
+
else field += character;
|
|
28513
|
+
else if (character === "\"" && field.length === 0) quoted = true;
|
|
28514
|
+
else if (character === ",") {
|
|
28515
|
+
record.push(field);
|
|
28516
|
+
field = "";
|
|
28517
|
+
} else if (character === "\n") {
|
|
28518
|
+
record.push(field.replace(/\r$/, ""));
|
|
28519
|
+
field = "";
|
|
28520
|
+
if (record.some((value) => value.length > 0)) yield record;
|
|
28521
|
+
record = [];
|
|
28522
|
+
} else field += character;
|
|
28523
|
+
if (Buffer.byteLength(field) > MAX_CELL_BYTES) {
|
|
28524
|
+
stream.destroy();
|
|
28525
|
+
throw new SeoError("INVALID_INPUT", `Link import contains a cell larger than ${MAX_CELL_BYTES} bytes.`);
|
|
28526
|
+
}
|
|
28527
|
+
}
|
|
28528
|
+
}
|
|
28529
|
+
if (quoted && !pendingQuote) throw new SeoError("INVALID_INPUT", "Link CSV contains an unclosed quote.");
|
|
28530
|
+
if (field || record.length) {
|
|
28531
|
+
record.push(field.replace(/\r$/, ""));
|
|
28532
|
+
if (record.some((value) => value.length > 0)) yield record;
|
|
28533
|
+
}
|
|
28534
|
+
}
|
|
28535
|
+
async function* jsonlRecords(path, byteLimit, progress) {
|
|
28536
|
+
const stream = createReadStream(path, { encoding: "utf8" });
|
|
28537
|
+
let remainder = "";
|
|
28538
|
+
for await (const chunk of stream) {
|
|
28539
|
+
progress.bytesRead += Buffer.byteLength(chunk);
|
|
28540
|
+
if (progress.bytesRead > byteLimit) {
|
|
28541
|
+
stream.destroy();
|
|
28542
|
+
throw new SeoError("INVALID_INPUT", `Link import exceeds the ${byteLimit}-byte streaming limit.`);
|
|
28543
|
+
}
|
|
28544
|
+
remainder += chunk;
|
|
28545
|
+
let newline = remainder.indexOf("\n");
|
|
28546
|
+
while (newline >= 0) {
|
|
28547
|
+
const line = remainder.slice(0, newline).trim();
|
|
28548
|
+
remainder = remainder.slice(newline + 1);
|
|
28549
|
+
if (line) yield parseObject(line);
|
|
28550
|
+
newline = remainder.indexOf("\n");
|
|
28551
|
+
}
|
|
28552
|
+
if (Buffer.byteLength(remainder) > MAX_CELL_BYTES) {
|
|
28553
|
+
stream.destroy();
|
|
28554
|
+
throw new SeoError("INVALID_INPUT", `Link import contains a line larger than ${MAX_CELL_BYTES} bytes.`);
|
|
28555
|
+
}
|
|
28556
|
+
}
|
|
28557
|
+
if (remainder.trim()) yield parseObject(remainder);
|
|
28558
|
+
}
|
|
28559
|
+
function parseObject(value) {
|
|
28560
|
+
try {
|
|
28561
|
+
const parsed = JSON.parse(value);
|
|
28562
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("not an object");
|
|
28563
|
+
return parsed;
|
|
28564
|
+
} catch {
|
|
28565
|
+
throw new SeoError("INVALID_INPUT", "Link JSONL must contain one JSON object per line.");
|
|
28566
|
+
}
|
|
28567
|
+
}
|
|
28568
|
+
function rowFromCsv(headers, values) {
|
|
28569
|
+
return Object.fromEntries(headers.map((header, index) => [header, values[index]]));
|
|
28570
|
+
}
|
|
28571
|
+
async function importLinkEvidence(input) {
|
|
28572
|
+
const path = resolve(input.file);
|
|
28573
|
+
const file = await stat(path).catch(() => void 0);
|
|
28574
|
+
if (!file?.isFile()) throw new SeoError("INVALID_INPUT", `Link import file was not found: ${path}`);
|
|
28575
|
+
const format = formatFor$1(path, input.format);
|
|
28576
|
+
const rowLimit = boundedRowLimit(input.rowLimit);
|
|
28577
|
+
if (format === "json" && file.size > MAX_JSON_BYTES) throw new SeoError("INVALID_INPUT", `JSON link imports are limited to ${MAX_JSON_BYTES} bytes. Use CSV or JSONL for larger files.`);
|
|
28578
|
+
const progress = { bytesRead: 0 };
|
|
28579
|
+
let records;
|
|
28580
|
+
if (format === "csv") {
|
|
28581
|
+
const csv = csvRecords(path, MAX_STREAM_BYTES, progress);
|
|
28582
|
+
const first = await csv.next();
|
|
28583
|
+
const headers = (Array.isArray(first.value) ? first.value : []).map((value, index) => index === 0 ? value.replace(/^\uFEFF/, "") : value);
|
|
28584
|
+
if (!headers.length) throw new SeoError("INVALID_INPUT", "Link CSV has no header row.");
|
|
28585
|
+
records = (async function* () {
|
|
28586
|
+
for await (const values of csv) yield rowFromCsv(headers, values);
|
|
28587
|
+
})();
|
|
28588
|
+
} else if (format === "jsonl") records = jsonlRecords(path, MAX_STREAM_BYTES, progress);
|
|
28589
|
+
else {
|
|
28590
|
+
const body = await readFile(path, "utf8");
|
|
28591
|
+
progress.bytesRead = Buffer.byteLength(body);
|
|
28592
|
+
let parsed;
|
|
28593
|
+
try {
|
|
28594
|
+
parsed = JSON.parse(body);
|
|
28595
|
+
} catch {
|
|
28596
|
+
throw new SeoError("INVALID_INPUT", "Link JSON is not valid JSON.");
|
|
28597
|
+
}
|
|
28598
|
+
if (!Array.isArray(parsed)) throw new SeoError("INVALID_INPUT", "Link JSON must be an array of objects.");
|
|
28599
|
+
records = (async function* () {
|
|
28600
|
+
for (const row of parsed) yield row && typeof row === "object" && !Array.isArray(row) ? row : {};
|
|
28601
|
+
})();
|
|
28602
|
+
}
|
|
28603
|
+
const rows = [];
|
|
28604
|
+
const keys = /* @__PURE__ */ new Set();
|
|
28605
|
+
let suppliedRows = 0;
|
|
28606
|
+
let invalidRows = 0;
|
|
28607
|
+
let duplicateRows = 0;
|
|
28608
|
+
let capped = false;
|
|
28609
|
+
for await (const raw of records) {
|
|
28610
|
+
if (suppliedRows >= rowLimit) {
|
|
28611
|
+
capped = true;
|
|
28612
|
+
break;
|
|
28613
|
+
}
|
|
28614
|
+
suppliedRows += 1;
|
|
28615
|
+
const row = normalizeLinkEvidenceRow(raw);
|
|
28616
|
+
if (!row) {
|
|
28617
|
+
invalidRows += 1;
|
|
28618
|
+
continue;
|
|
28619
|
+
}
|
|
28620
|
+
const key = linkEvidenceKey(row);
|
|
28621
|
+
if (keys.has(key)) {
|
|
28622
|
+
duplicateRows += 1;
|
|
28623
|
+
continue;
|
|
28624
|
+
}
|
|
28625
|
+
keys.add(key);
|
|
28626
|
+
rows.push(row);
|
|
28627
|
+
}
|
|
28628
|
+
rows.sort(compareLinkEvidence);
|
|
28629
|
+
const observedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
28630
|
+
return {
|
|
28631
|
+
rows,
|
|
28632
|
+
targetCounts: [],
|
|
28633
|
+
provenance: {
|
|
28634
|
+
provider: `${format}-import`,
|
|
28635
|
+
observedAt,
|
|
28636
|
+
cached: false,
|
|
28637
|
+
suppliedRows,
|
|
28638
|
+
validRows: rows.length,
|
|
28639
|
+
invalidRows,
|
|
28640
|
+
duplicateRows,
|
|
28641
|
+
capped,
|
|
28642
|
+
rowLimit,
|
|
28643
|
+
completeness: capped ? "partial" : "unknown",
|
|
28644
|
+
file: {
|
|
28645
|
+
path,
|
|
28646
|
+
format,
|
|
28647
|
+
bytesRead: progress.bytesRead,
|
|
28648
|
+
fileBytes: file.size
|
|
28649
|
+
}
|
|
28650
|
+
},
|
|
28651
|
+
warnings: capped ? [`Import stopped after ${rowLimit} rows. The file contains more data.`] : []
|
|
28652
|
+
};
|
|
28653
|
+
}
|
|
28654
|
+
//#endregion
|
|
28655
|
+
//#region packages/core/dist/links/report.js
|
|
28656
|
+
const DEFAULT_OUTPUT_LIMIT = 100;
|
|
28657
|
+
const MAX_OUTPUT_LIMIT = 500;
|
|
28658
|
+
function boundedLimit(value) {
|
|
28659
|
+
const limit = value ?? DEFAULT_OUTPUT_LIMIT;
|
|
28660
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_OUTPUT_LIMIT) throw new SeoError("INVALID_INPUT", `Link report limit must be between 1 and ${MAX_OUTPUT_LIMIT}.`);
|
|
28661
|
+
return limit;
|
|
28662
|
+
}
|
|
28663
|
+
function targetCounts(rows) {
|
|
28664
|
+
const counts = /* @__PURE__ */ new Map();
|
|
28665
|
+
for (const row of rows) counts.set(row.targetUrl, (counts.get(row.targetUrl) ?? 0) + 1);
|
|
28666
|
+
return [...counts].map(([targetUrl, observedLinks]) => ({
|
|
28667
|
+
targetUrl,
|
|
28668
|
+
observedLinks
|
|
28669
|
+
})).sort((a, b) => b.observedLinks - a.observedLinks || a.targetUrl.localeCompare(b.targetUrl, "en"));
|
|
28670
|
+
}
|
|
28671
|
+
function linkEvidenceReport(input) {
|
|
28672
|
+
const limit = boundedLimit(input.limit);
|
|
28673
|
+
const rows = input.evidence.rows.slice(0, limit);
|
|
28674
|
+
const domains = new Set(input.evidence.rows.map((row) => row.sourceDomain));
|
|
28675
|
+
const observedTargets = targetCounts(input.evidence.rows);
|
|
28676
|
+
const counts = input.evidence.targetCounts.length ? input.evidence.targetCounts : observedTargets;
|
|
28677
|
+
return {
|
|
28678
|
+
schemaVersion: 1,
|
|
28679
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
28680
|
+
dataStatus: input.evidence.provenance.completeness === "complete" ? "complete" : "partial",
|
|
28681
|
+
summary: {
|
|
28682
|
+
observedLinks: input.evidence.rows.length,
|
|
28683
|
+
referringDomains: domains.size,
|
|
28684
|
+
targetPages: new Set(input.evidence.rows.map((row) => row.targetUrl)).size,
|
|
28685
|
+
providerTargetPages: counts.length
|
|
28686
|
+
},
|
|
28687
|
+
provenance: input.evidence.provenance,
|
|
28688
|
+
selection: {
|
|
28689
|
+
availableRows: input.evidence.rows.length,
|
|
28690
|
+
returnedRows: rows.length,
|
|
28691
|
+
omittedRows: Math.max(0, input.evidence.rows.length - rows.length),
|
|
28692
|
+
limit
|
|
28693
|
+
},
|
|
28694
|
+
targetCounts: counts.slice(0, 100),
|
|
28695
|
+
links: rows,
|
|
28696
|
+
warnings: input.evidence.warnings,
|
|
28697
|
+
caveats: [
|
|
28698
|
+
"This is bounded provider or imported evidence, not a complete backlink index.",
|
|
28699
|
+
"A missing link is not proof that no link exists, and an observed link is not a quality or ranking verdict.",
|
|
28700
|
+
"Referring-domain counts describe the retained rows only."
|
|
28701
|
+
]
|
|
28702
|
+
};
|
|
28703
|
+
}
|
|
28704
|
+
//#endregion
|
|
28705
|
+
//#region packages/core/dist/providers/dataforseo.js
|
|
28706
|
+
var DataForSeoProvider = class {
|
|
28707
|
+
name = "dataforseo";
|
|
28708
|
+
capabilities = { overview: true };
|
|
28709
|
+
getAuthHeader() {
|
|
28710
|
+
const config = readConfig();
|
|
28711
|
+
const login = config.providers.dataForSeoLogin;
|
|
28712
|
+
const password = config.providers.dataForSeoPassword;
|
|
28713
|
+
if (!login || !password) throw new Error("DataForSEO credentials missing. Add login and password to config.json.");
|
|
28714
|
+
return `Basic ${Buffer.from(`${login}:${password}`).toString("base64")}`;
|
|
28715
|
+
}
|
|
28716
|
+
async keywordOverview(phrase, opts = {}) {
|
|
28717
|
+
const response = await fetch$1("https://api.dataforseo.com/v3/dataforseo_labs/google/keyword_overview/live", {
|
|
28718
|
+
method: "POST",
|
|
28719
|
+
headers: {
|
|
28720
|
+
authorization: this.getAuthHeader(),
|
|
28721
|
+
"content-type": "application/json"
|
|
28722
|
+
},
|
|
28723
|
+
body: JSON.stringify([{
|
|
28724
|
+
keyword: phrase,
|
|
28725
|
+
language_code: "en",
|
|
28726
|
+
location_code: 2840
|
|
28727
|
+
}])
|
|
28728
|
+
});
|
|
28729
|
+
if (!response.ok) throw new Error(`DataForSEO request failed with ${response.status}.`);
|
|
28730
|
+
const json = await response.json();
|
|
28731
|
+
const row = json.tasks?.[0]?.result?.[0];
|
|
28732
|
+
return {
|
|
28733
|
+
data: {
|
|
28734
|
+
phrase: row?.keyword ?? phrase,
|
|
28735
|
+
volume: row?.search_volume,
|
|
28736
|
+
competition: row?.competition,
|
|
28737
|
+
cpc: row?.cpc,
|
|
28738
|
+
difficulty: row?.keyword_difficulty
|
|
28739
|
+
},
|
|
28740
|
+
usage: {
|
|
28741
|
+
provider: "DataForSEO",
|
|
28742
|
+
units: 1,
|
|
28743
|
+
unitLabel: "tasks",
|
|
28744
|
+
estimatedUsd: json.cost,
|
|
28745
|
+
calls: 1
|
|
28746
|
+
},
|
|
28747
|
+
warnings: opts.database ? ["DataForSEO overview currently ignores the database option."] : void 0
|
|
28748
|
+
};
|
|
28749
|
+
}
|
|
28750
|
+
};
|
|
28751
|
+
//#endregion
|
|
28752
|
+
//#region packages/core/dist/providers/semrush/csv.js
|
|
28753
|
+
function parseSemicolonCsv(text) {
|
|
28754
|
+
return text.trim().split("\n").map((line) => line.split(";").map((cell) => cell.replace(/^"|"$/g, "").trim()));
|
|
27262
28755
|
}
|
|
27263
28756
|
//#endregion
|
|
27264
28757
|
//#region packages/core/dist/providers/semrush/cache.js
|
|
@@ -27463,6 +28956,472 @@ var LocalCredentialsProvider = class {
|
|
|
27463
28956
|
}
|
|
27464
28957
|
};
|
|
27465
28958
|
new LocalCredentialsProvider();
|
|
28959
|
+
//#endregion
|
|
28960
|
+
//#region packages/core/dist/server-logs/parse.js
|
|
28961
|
+
const MONTHS = {
|
|
28962
|
+
Jan: 0,
|
|
28963
|
+
Feb: 1,
|
|
28964
|
+
Mar: 2,
|
|
28965
|
+
Apr: 3,
|
|
28966
|
+
May: 4,
|
|
28967
|
+
Jun: 5,
|
|
28968
|
+
Jul: 6,
|
|
28969
|
+
Aug: 7,
|
|
28970
|
+
Sep: 8,
|
|
28971
|
+
Oct: 9,
|
|
28972
|
+
Nov: 10,
|
|
28973
|
+
Dec: 11
|
|
28974
|
+
};
|
|
28975
|
+
const CRAWLERS = [
|
|
28976
|
+
{
|
|
28977
|
+
family: "Googlebot",
|
|
28978
|
+
category: "search",
|
|
28979
|
+
patterns: ["googlebot", "google-inspectiontool"]
|
|
28980
|
+
},
|
|
28981
|
+
{
|
|
28982
|
+
family: "Bingbot",
|
|
28983
|
+
category: "search",
|
|
28984
|
+
patterns: ["bingbot", "bingpreview"]
|
|
28985
|
+
},
|
|
28986
|
+
{
|
|
28987
|
+
family: "Applebot",
|
|
28988
|
+
category: "search",
|
|
28989
|
+
patterns: ["applebot"]
|
|
28990
|
+
},
|
|
28991
|
+
{
|
|
28992
|
+
family: "DuckDuckBot",
|
|
28993
|
+
category: "search",
|
|
28994
|
+
patterns: ["duckduckbot"]
|
|
28995
|
+
},
|
|
28996
|
+
{
|
|
28997
|
+
family: "YandexBot",
|
|
28998
|
+
category: "search",
|
|
28999
|
+
patterns: ["yandexbot"]
|
|
29000
|
+
},
|
|
29001
|
+
{
|
|
29002
|
+
family: "Baiduspider",
|
|
29003
|
+
category: "search",
|
|
29004
|
+
patterns: ["baiduspider"]
|
|
29005
|
+
},
|
|
29006
|
+
{
|
|
29007
|
+
family: "PetalBot",
|
|
29008
|
+
category: "search",
|
|
29009
|
+
patterns: ["petalbot"]
|
|
29010
|
+
},
|
|
29011
|
+
{
|
|
29012
|
+
family: "OpenAI",
|
|
29013
|
+
category: "ai",
|
|
29014
|
+
patterns: [
|
|
29015
|
+
"gptbot",
|
|
29016
|
+
"chatgpt-user",
|
|
29017
|
+
"oai-searchbot"
|
|
29018
|
+
]
|
|
29019
|
+
},
|
|
29020
|
+
{
|
|
29021
|
+
family: "Anthropic",
|
|
29022
|
+
category: "ai",
|
|
29023
|
+
patterns: [
|
|
29024
|
+
"claudebot",
|
|
29025
|
+
"claude-searchbot",
|
|
29026
|
+
"claude-user"
|
|
29027
|
+
]
|
|
29028
|
+
},
|
|
29029
|
+
{
|
|
29030
|
+
family: "Perplexity",
|
|
29031
|
+
category: "ai",
|
|
29032
|
+
patterns: ["perplexitybot", "perplexity-user"]
|
|
29033
|
+
},
|
|
29034
|
+
{
|
|
29035
|
+
family: "Amazonbot",
|
|
29036
|
+
category: "ai",
|
|
29037
|
+
patterns: ["amazonbot"]
|
|
29038
|
+
},
|
|
29039
|
+
{
|
|
29040
|
+
family: "Bytespider",
|
|
29041
|
+
category: "ai",
|
|
29042
|
+
patterns: ["bytespider"]
|
|
29043
|
+
},
|
|
29044
|
+
{
|
|
29045
|
+
family: "CCBot",
|
|
29046
|
+
category: "ai",
|
|
29047
|
+
patterns: ["ccbot"]
|
|
29048
|
+
},
|
|
29049
|
+
{
|
|
29050
|
+
family: "Cohere",
|
|
29051
|
+
category: "ai",
|
|
29052
|
+
patterns: ["cohere-ai"]
|
|
29053
|
+
}
|
|
29054
|
+
];
|
|
29055
|
+
const COMBINED_LOG = /^\S+\s+\S+\s+\S+\s+\[([^\]]+)]\s+"([^"\s]+)\s+([^"\s]+)(?:\s+[^"\s]+)?"\s+(\d{3})\s+(\d+|-)\s*(?:"[^"]*"\s+"([^"]*)")?\s*$/;
|
|
29056
|
+
function classifyCrawler(userAgent) {
|
|
29057
|
+
if (!userAgent) return void 0;
|
|
29058
|
+
const value = userAgent.toLowerCase();
|
|
29059
|
+
const match = CRAWLERS.find((crawler) => crawler.patterns.some((pattern) => value.includes(pattern)));
|
|
29060
|
+
return match ? {
|
|
29061
|
+
family: match.family,
|
|
29062
|
+
category: match.category
|
|
29063
|
+
} : void 0;
|
|
29064
|
+
}
|
|
29065
|
+
function logTimestamp(value) {
|
|
29066
|
+
const match = /^(\d{2})\/([A-Z][a-z]{2})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})$/.exec(value);
|
|
29067
|
+
if (!match) return void 0;
|
|
29068
|
+
const month = MONTHS[match[2] ?? ""];
|
|
29069
|
+
if (month === void 0) return void 0;
|
|
29070
|
+
const utc = Date.UTC(Number(match[3]), month, Number(match[1]), Number(match[4]), Number(match[5]), Number(match[6]));
|
|
29071
|
+
const offsetMinutes = Number(match[8]) * 60 + Number(match[9]);
|
|
29072
|
+
const adjusted = utc + (match[7] === "+" ? -1 : 1) * offsetMinutes * 6e4;
|
|
29073
|
+
const date = new Date(adjusted);
|
|
29074
|
+
return Number.isNaN(date.getTime()) ? void 0 : date.toISOString();
|
|
29075
|
+
}
|
|
29076
|
+
function normalizedPath(value) {
|
|
29077
|
+
if (typeof value !== "string" || value.length < 1 || value.length > 2e3) return;
|
|
29078
|
+
try {
|
|
29079
|
+
if (value.startsWith("/")) return new URL(value, "https://server-log.invalid").pathname;
|
|
29080
|
+
const url = new URL(value);
|
|
29081
|
+
return ["http:", "https:"].includes(url.protocol) ? url.pathname : void 0;
|
|
29082
|
+
} catch {
|
|
29083
|
+
return;
|
|
29084
|
+
}
|
|
29085
|
+
}
|
|
29086
|
+
function normalizedMethod(value) {
|
|
29087
|
+
if (typeof value !== "string" || !/^[A-Za-z]{1,16}$/.test(value)) return;
|
|
29088
|
+
return value.toUpperCase();
|
|
29089
|
+
}
|
|
29090
|
+
function normalizedStatus(value) {
|
|
29091
|
+
const status = typeof value === "number" ? value : Number(value);
|
|
29092
|
+
return Number.isInteger(status) && status >= 100 && status <= 599 ? status : void 0;
|
|
29093
|
+
}
|
|
29094
|
+
function normalizedBytes(value) {
|
|
29095
|
+
if (value === void 0 || value === null || value === "" || value === "-") return;
|
|
29096
|
+
const bytes = typeof value === "number" ? value : Number(value);
|
|
29097
|
+
return Number.isSafeInteger(bytes) && bytes >= 0 ? bytes : void 0;
|
|
29098
|
+
}
|
|
29099
|
+
function record(input) {
|
|
29100
|
+
let timestamp;
|
|
29101
|
+
if (typeof input.timestamp === "string") {
|
|
29102
|
+
const date = new Date(input.timestamp);
|
|
29103
|
+
if (!Number.isNaN(date.getTime())) timestamp = date.toISOString();
|
|
29104
|
+
}
|
|
29105
|
+
const method = normalizedMethod(input.method);
|
|
29106
|
+
const path = normalizedPath(input.path);
|
|
29107
|
+
const status = normalizedStatus(input.status);
|
|
29108
|
+
if (!timestamp || !method || !path || status === void 0) return void 0;
|
|
29109
|
+
const userAgent = typeof input.userAgent === "string" && input.userAgent.length <= 2e3 ? input.userAgent : void 0;
|
|
29110
|
+
return {
|
|
29111
|
+
timestamp,
|
|
29112
|
+
method,
|
|
29113
|
+
path,
|
|
29114
|
+
status,
|
|
29115
|
+
bytes: normalizedBytes(input.bytes),
|
|
29116
|
+
userAgent,
|
|
29117
|
+
crawler: classifyCrawler(userAgent)
|
|
29118
|
+
};
|
|
29119
|
+
}
|
|
29120
|
+
function parseCombinedLogLine(line) {
|
|
29121
|
+
const match = COMBINED_LOG.exec(line);
|
|
29122
|
+
if (!match) return void 0;
|
|
29123
|
+
const timestamp = logTimestamp(match[1] ?? "");
|
|
29124
|
+
if (!timestamp) return void 0;
|
|
29125
|
+
return record({
|
|
29126
|
+
timestamp,
|
|
29127
|
+
method: match[2],
|
|
29128
|
+
path: match[3],
|
|
29129
|
+
status: match[4],
|
|
29130
|
+
bytes: match[5],
|
|
29131
|
+
userAgent: match[6]
|
|
29132
|
+
});
|
|
29133
|
+
}
|
|
29134
|
+
function firstValue(value, names) {
|
|
29135
|
+
for (const name of names) if (value[name] !== void 0) return value[name];
|
|
29136
|
+
}
|
|
29137
|
+
function parseJsonLogLine(line) {
|
|
29138
|
+
let value;
|
|
29139
|
+
try {
|
|
29140
|
+
value = JSON.parse(line);
|
|
29141
|
+
} catch {
|
|
29142
|
+
return;
|
|
29143
|
+
}
|
|
29144
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return;
|
|
29145
|
+
const row = value;
|
|
29146
|
+
return record({
|
|
29147
|
+
timestamp: firstValue(row, [
|
|
29148
|
+
"timestamp",
|
|
29149
|
+
"time",
|
|
29150
|
+
"datetime",
|
|
29151
|
+
"@timestamp"
|
|
29152
|
+
]),
|
|
29153
|
+
method: firstValue(row, [
|
|
29154
|
+
"method",
|
|
29155
|
+
"requestMethod",
|
|
29156
|
+
"http_method"
|
|
29157
|
+
]),
|
|
29158
|
+
path: firstValue(row, [
|
|
29159
|
+
"path",
|
|
29160
|
+
"url",
|
|
29161
|
+
"requestUri",
|
|
29162
|
+
"request_uri"
|
|
29163
|
+
]),
|
|
29164
|
+
status: firstValue(row, [
|
|
29165
|
+
"status",
|
|
29166
|
+
"statusCode",
|
|
29167
|
+
"status_code"
|
|
29168
|
+
]),
|
|
29169
|
+
bytes: firstValue(row, [
|
|
29170
|
+
"bytes",
|
|
29171
|
+
"bodyBytesSent",
|
|
29172
|
+
"body_bytes_sent"
|
|
29173
|
+
]),
|
|
29174
|
+
userAgent: firstValue(row, [
|
|
29175
|
+
"userAgent",
|
|
29176
|
+
"user_agent",
|
|
29177
|
+
"httpUserAgent",
|
|
29178
|
+
"http_user_agent"
|
|
29179
|
+
])
|
|
29180
|
+
});
|
|
29181
|
+
}
|
|
29182
|
+
function emptyStatusBreakdown() {
|
|
29183
|
+
return {
|
|
29184
|
+
success: 0,
|
|
29185
|
+
redirect: 0,
|
|
29186
|
+
clientError: 0,
|
|
29187
|
+
serverError: 0,
|
|
29188
|
+
other: 0
|
|
29189
|
+
};
|
|
29190
|
+
}
|
|
29191
|
+
function addStatus(breakdown, status) {
|
|
29192
|
+
if (status >= 200 && status < 300) breakdown.success += 1;
|
|
29193
|
+
else if (status >= 300 && status < 400) breakdown.redirect += 1;
|
|
29194
|
+
else if (status >= 400 && status < 500) breakdown.clientError += 1;
|
|
29195
|
+
else if (status >= 500) breakdown.serverError += 1;
|
|
29196
|
+
else breakdown.other += 1;
|
|
29197
|
+
}
|
|
29198
|
+
//#endregion
|
|
29199
|
+
//#region packages/core/dist/server-logs/import.js
|
|
29200
|
+
const DEFAULT_ROW_LIMIT = 1e6;
|
|
29201
|
+
const MAX_ROW_LIMIT = 1e7;
|
|
29202
|
+
const DEFAULT_PATH_LIMIT = 25e3;
|
|
29203
|
+
const MAX_PATH_LIMIT = 1e5;
|
|
29204
|
+
const BYTE_LIMIT = 1e9;
|
|
29205
|
+
const MAX_LINE_BYTES = 65536;
|
|
29206
|
+
function boundedInteger(value, fallback, maximum, label) {
|
|
29207
|
+
const result = value ?? fallback;
|
|
29208
|
+
if (!Number.isInteger(result) || result < 1 || result > maximum) throw new SeoError("INVALID_INPUT", `${label} must be between 1 and ${maximum}.`);
|
|
29209
|
+
return result;
|
|
29210
|
+
}
|
|
29211
|
+
function formatFor(path, explicit) {
|
|
29212
|
+
if (explicit) return explicit;
|
|
29213
|
+
const extension = extname(path).toLowerCase();
|
|
29214
|
+
return [".jsonl", ".ndjson"].includes(extension) ? "jsonl" : "combined";
|
|
29215
|
+
}
|
|
29216
|
+
async function* lines(path, progress) {
|
|
29217
|
+
const stream = createReadStream(path, {
|
|
29218
|
+
encoding: "utf8",
|
|
29219
|
+
highWaterMark: 64 * 1024
|
|
29220
|
+
});
|
|
29221
|
+
let remainder = "";
|
|
29222
|
+
let discarding = false;
|
|
29223
|
+
for await (const chunk of stream) {
|
|
29224
|
+
const remainingBytes = BYTE_LIMIT - progress.bytesRead;
|
|
29225
|
+
if (remainingBytes <= 0) {
|
|
29226
|
+
progress.bytesCapped = true;
|
|
29227
|
+
break;
|
|
29228
|
+
}
|
|
29229
|
+
const chunkBytes = Buffer.byteLength(chunk);
|
|
29230
|
+
const accepted = chunkBytes > remainingBytes ? Buffer.from(chunk).subarray(0, remainingBytes).toString("utf8") : chunk;
|
|
29231
|
+
progress.bytesRead += Buffer.byteLength(accepted);
|
|
29232
|
+
if (chunkBytes > remainingBytes) progress.bytesCapped = true;
|
|
29233
|
+
remainder += accepted;
|
|
29234
|
+
let newline = remainder.indexOf("\n");
|
|
29235
|
+
while (newline >= 0) {
|
|
29236
|
+
const line = remainder.slice(0, newline).replace(/\r$/, "");
|
|
29237
|
+
remainder = remainder.slice(newline + 1);
|
|
29238
|
+
if (discarding || Buffer.byteLength(line) > MAX_LINE_BYTES) {
|
|
29239
|
+
yield { tooLong: true };
|
|
29240
|
+
discarding = false;
|
|
29241
|
+
} else if (line.trim()) yield { line };
|
|
29242
|
+
newline = remainder.indexOf("\n");
|
|
29243
|
+
}
|
|
29244
|
+
if (Buffer.byteLength(remainder) > MAX_LINE_BYTES) {
|
|
29245
|
+
remainder = "";
|
|
29246
|
+
discarding = true;
|
|
29247
|
+
}
|
|
29248
|
+
if (progress.bytesCapped) break;
|
|
29249
|
+
}
|
|
29250
|
+
if (discarding || Buffer.byteLength(remainder) > MAX_LINE_BYTES) yield { tooLong: true };
|
|
29251
|
+
else if (remainder.trim()) yield { line: remainder.replace(/\r$/, "") };
|
|
29252
|
+
}
|
|
29253
|
+
function compareCrawler(a, b) {
|
|
29254
|
+
return b.requests - a.requests || a.family.localeCompare(b.family, "en");
|
|
29255
|
+
}
|
|
29256
|
+
function compareCrawlerPath(a, b) {
|
|
29257
|
+
return b.requests - a.requests || a.family.localeCompare(b.family, "en") || a.path.localeCompare(b.path, "en");
|
|
29258
|
+
}
|
|
29259
|
+
function updatePeriod(timestamp, period) {
|
|
29260
|
+
if (!period.firstSeenAt || timestamp < period.firstSeenAt) period.firstSeenAt = timestamp;
|
|
29261
|
+
if (!period.lastSeenAt || timestamp > period.lastSeenAt) period.lastSeenAt = timestamp;
|
|
29262
|
+
}
|
|
29263
|
+
function newCrawler(record) {
|
|
29264
|
+
const crawler = record.crawler;
|
|
29265
|
+
if (!crawler) throw new Error("Crawler evidence is required.");
|
|
29266
|
+
return {
|
|
29267
|
+
family: crawler.family,
|
|
29268
|
+
category: crawler.category,
|
|
29269
|
+
requests: 0,
|
|
29270
|
+
lastSeenAt: record.timestamp,
|
|
29271
|
+
...emptyStatusBreakdown()
|
|
29272
|
+
};
|
|
29273
|
+
}
|
|
29274
|
+
function newCrawlerPath(record) {
|
|
29275
|
+
return {
|
|
29276
|
+
...newCrawler(record),
|
|
29277
|
+
path: record.path
|
|
29278
|
+
};
|
|
29279
|
+
}
|
|
29280
|
+
async function importServerLog(input) {
|
|
29281
|
+
const path = resolve(input.file);
|
|
29282
|
+
const file = await stat(path).catch(() => void 0);
|
|
29283
|
+
if (!file?.isFile()) throw new SeoError("INVALID_INPUT", `Server log was not found: ${path}`);
|
|
29284
|
+
const format = formatFor(path, input.format);
|
|
29285
|
+
const rowLimit = boundedInteger(input.rowLimit, DEFAULT_ROW_LIMIT, MAX_ROW_LIMIT, "Server log row limit");
|
|
29286
|
+
const pathLimit = boundedInteger(input.pathLimit, DEFAULT_PATH_LIMIT, MAX_PATH_LIMIT, "Server log path limit");
|
|
29287
|
+
const parse = format === "jsonl" ? parseJsonLogLine : parseCombinedLogLine;
|
|
29288
|
+
const progress = {
|
|
29289
|
+
bytesRead: 0,
|
|
29290
|
+
bytesCapped: false
|
|
29291
|
+
};
|
|
29292
|
+
const period = {};
|
|
29293
|
+
const statusCodes = /* @__PURE__ */ new Map();
|
|
29294
|
+
const crawlers = /* @__PURE__ */ new Map();
|
|
29295
|
+
const crawlerPaths = /* @__PURE__ */ new Map();
|
|
29296
|
+
let suppliedRows = 0;
|
|
29297
|
+
let parsedRows = 0;
|
|
29298
|
+
let invalidRows = 0;
|
|
29299
|
+
let crawlerRows = 0;
|
|
29300
|
+
let responseBytes = 0;
|
|
29301
|
+
let rowsCapped = false;
|
|
29302
|
+
let pathsCapped = false;
|
|
29303
|
+
let untrackedCrawlerPathRows = 0;
|
|
29304
|
+
for await (const item of lines(path, progress)) {
|
|
29305
|
+
if (suppliedRows >= rowLimit) {
|
|
29306
|
+
rowsCapped = true;
|
|
29307
|
+
break;
|
|
29308
|
+
}
|
|
29309
|
+
suppliedRows += 1;
|
|
29310
|
+
const parsed = item.line ? parse(item.line) : void 0;
|
|
29311
|
+
if (!parsed) {
|
|
29312
|
+
invalidRows += 1;
|
|
29313
|
+
continue;
|
|
29314
|
+
}
|
|
29315
|
+
parsedRows += 1;
|
|
29316
|
+
responseBytes += parsed.bytes ?? 0;
|
|
29317
|
+
updatePeriod(parsed.timestamp, period);
|
|
29318
|
+
statusCodes.set(parsed.status, (statusCodes.get(parsed.status) ?? 0) + 1);
|
|
29319
|
+
if (!parsed.crawler) continue;
|
|
29320
|
+
crawlerRows += 1;
|
|
29321
|
+
const crawler = crawlers.get(parsed.crawler.family) ?? newCrawler(parsed);
|
|
29322
|
+
crawler.requests += 1;
|
|
29323
|
+
if (parsed.timestamp > crawler.lastSeenAt) crawler.lastSeenAt = parsed.timestamp;
|
|
29324
|
+
addStatus(crawler, parsed.status);
|
|
29325
|
+
crawlers.set(parsed.crawler.family, crawler);
|
|
29326
|
+
const key = `${parsed.crawler.family}\u0000${parsed.path}`;
|
|
29327
|
+
let crawlerPath = crawlerPaths.get(key);
|
|
29328
|
+
if (!crawlerPath && crawlerPaths.size >= pathLimit) {
|
|
29329
|
+
pathsCapped = true;
|
|
29330
|
+
untrackedCrawlerPathRows += 1;
|
|
29331
|
+
continue;
|
|
29332
|
+
}
|
|
29333
|
+
crawlerPath ??= newCrawlerPath(parsed);
|
|
29334
|
+
crawlerPath.requests += 1;
|
|
29335
|
+
if (parsed.timestamp > crawlerPath.lastSeenAt) crawlerPath.lastSeenAt = parsed.timestamp;
|
|
29336
|
+
addStatus(crawlerPath, parsed.status);
|
|
29337
|
+
crawlerPaths.set(key, crawlerPath);
|
|
29338
|
+
}
|
|
29339
|
+
const fileReadCompletely = !rowsCapped && !progress.bytesCapped && progress.bytesRead >= file.size;
|
|
29340
|
+
const completeness = fileReadCompletely && invalidRows === 0 && !pathsCapped ? "complete" : "partial";
|
|
29341
|
+
const warnings = [];
|
|
29342
|
+
if (rowsCapped) warnings.push(`Analysis stopped after ${rowLimit} rows.`);
|
|
29343
|
+
if (progress.bytesCapped) warnings.push(`Analysis stopped after ${BYTE_LIMIT} input bytes.`);
|
|
29344
|
+
if (invalidRows) warnings.push(`${invalidRows} malformed or unsupported rows were skipped.`);
|
|
29345
|
+
if (pathsCapped) warnings.push(`Crawler path tracking stopped at ${pathLimit} unique crawler and path pairs. Overall crawler and status totals still include later valid rows.`);
|
|
29346
|
+
return {
|
|
29347
|
+
summary: {
|
|
29348
|
+
suppliedRows,
|
|
29349
|
+
parsedRows,
|
|
29350
|
+
invalidRows,
|
|
29351
|
+
crawlerRows,
|
|
29352
|
+
nonCrawlerRows: parsedRows - crawlerRows,
|
|
29353
|
+
responseBytes,
|
|
29354
|
+
...period
|
|
29355
|
+
},
|
|
29356
|
+
statusCodes: [...statusCodes].map(([status, requests]) => ({
|
|
29357
|
+
status,
|
|
29358
|
+
requests
|
|
29359
|
+
})).sort((a, b) => a.status - b.status),
|
|
29360
|
+
crawlers: [...crawlers.values()].sort(compareCrawler),
|
|
29361
|
+
crawlerPaths: [...crawlerPaths.values()].sort(compareCrawlerPath),
|
|
29362
|
+
provenance: {
|
|
29363
|
+
source: "local-server-log",
|
|
29364
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
29365
|
+
cached: false,
|
|
29366
|
+
file: {
|
|
29367
|
+
path,
|
|
29368
|
+
format,
|
|
29369
|
+
bytesRead: progress.bytesRead,
|
|
29370
|
+
fileBytes: file.size
|
|
29371
|
+
},
|
|
29372
|
+
limits: {
|
|
29373
|
+
rowLimit,
|
|
29374
|
+
pathLimit,
|
|
29375
|
+
byteLimit: BYTE_LIMIT,
|
|
29376
|
+
maxLineBytes: MAX_LINE_BYTES
|
|
29377
|
+
},
|
|
29378
|
+
coverage: {
|
|
29379
|
+
fileReadCompletely,
|
|
29380
|
+
rowsCapped,
|
|
29381
|
+
bytesCapped: progress.bytesCapped,
|
|
29382
|
+
pathsCapped,
|
|
29383
|
+
untrackedCrawlerPathRows
|
|
29384
|
+
},
|
|
29385
|
+
completeness
|
|
29386
|
+
},
|
|
29387
|
+
warnings
|
|
29388
|
+
};
|
|
29389
|
+
}
|
|
29390
|
+
//#endregion
|
|
29391
|
+
//#region packages/core/dist/server-logs/report.js
|
|
29392
|
+
const DEFAULT_LIMIT = 100;
|
|
29393
|
+
const MAX_LIMIT = 500;
|
|
29394
|
+
function outputLimit(value) {
|
|
29395
|
+
const limit = value ?? DEFAULT_LIMIT;
|
|
29396
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) throw new SeoError("INVALID_INPUT", `Server log output limit must be between 1 and ${MAX_LIMIT}.`);
|
|
29397
|
+
return limit;
|
|
29398
|
+
}
|
|
29399
|
+
function serverLogReport(input) {
|
|
29400
|
+
const limit = outputLimit(input.limit);
|
|
29401
|
+
const crawlerPaths = input.evidence.crawlerPaths.slice(0, limit);
|
|
29402
|
+
return {
|
|
29403
|
+
schemaVersion: 1,
|
|
29404
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
29405
|
+
dataStatus: input.evidence.provenance.completeness,
|
|
29406
|
+
summary: input.evidence.summary,
|
|
29407
|
+
statusCodes: input.evidence.statusCodes,
|
|
29408
|
+
crawlers: input.evidence.crawlers,
|
|
29409
|
+
crawlerPaths,
|
|
29410
|
+
selection: {
|
|
29411
|
+
availableCrawlerPaths: input.evidence.crawlerPaths.length,
|
|
29412
|
+
returnedCrawlerPaths: crawlerPaths.length,
|
|
29413
|
+
omittedCrawlerPaths: Math.max(0, input.evidence.crawlerPaths.length - crawlerPaths.length),
|
|
29414
|
+
limit
|
|
29415
|
+
},
|
|
29416
|
+
provenance: input.evidence.provenance,
|
|
29417
|
+
warnings: input.evidence.warnings,
|
|
29418
|
+
caveats: [
|
|
29419
|
+
"Crawler families are identified from user-agent strings, which can be missing or spoofed.",
|
|
29420
|
+
"This report describes only the supplied log file and cannot prove that every server or time period is represented.",
|
|
29421
|
+
"Crawler path rankings describe retained path aggregates when the path limit is reached."
|
|
29422
|
+
]
|
|
29423
|
+
};
|
|
29424
|
+
}
|
|
27466
29425
|
const TELEMETRY_TIMEOUT_MS = 2e3;
|
|
27467
29426
|
const TELEMETRY_REPORTS = [
|
|
27468
29427
|
"affected-urls",
|
|
@@ -27472,6 +29431,7 @@ const TELEMETRY_REPORTS = [
|
|
|
27472
29431
|
"ai-search-scorecard",
|
|
27473
29432
|
"audit-page",
|
|
27474
29433
|
"audit-urls",
|
|
29434
|
+
"bing-webmaster-overview",
|
|
27475
29435
|
"cannibalisation",
|
|
27476
29436
|
"community-intent",
|
|
27477
29437
|
"compare-crawls",
|
|
@@ -27490,6 +29450,8 @@ const TELEMETRY_REPORTS = [
|
|
|
27490
29450
|
"index-monitor",
|
|
27491
29451
|
"index-watch",
|
|
27492
29452
|
"internal-links",
|
|
29453
|
+
"link-evidence",
|
|
29454
|
+
"server-log-analysis",
|
|
27493
29455
|
"link-recovery",
|
|
27494
29456
|
"crawl-history",
|
|
27495
29457
|
"crawler-rules",
|
|
@@ -29462,6 +31424,87 @@ function registerOpportunityTools(server, dependencies = {}) {
|
|
|
29462
31424
|
});
|
|
29463
31425
|
}
|
|
29464
31426
|
//#endregion
|
|
31427
|
+
//#region packages/mcp/src/provider-tools.ts
|
|
31428
|
+
const MAX_AGENT_ROWS_PER_SECTION = 14;
|
|
31429
|
+
function compactRows(data) {
|
|
31430
|
+
if (data.rows.length <= MAX_AGENT_ROWS_PER_SECTION && data.outputSelection) return data;
|
|
31431
|
+
const rows = data.rows.slice(-14);
|
|
31432
|
+
return {
|
|
31433
|
+
...data,
|
|
31434
|
+
rows,
|
|
31435
|
+
outputSelection: {
|
|
31436
|
+
strategy: "most-recent",
|
|
31437
|
+
availableRows: data.rows.length,
|
|
31438
|
+
returnedRows: rows.length,
|
|
31439
|
+
omittedRows: Math.max(0, data.rows.length - rows.length)
|
|
31440
|
+
}
|
|
31441
|
+
};
|
|
31442
|
+
}
|
|
31443
|
+
function compactBingWebmasterOverview(result) {
|
|
31444
|
+
return {
|
|
31445
|
+
...result,
|
|
31446
|
+
traffic: result.traffic.status === "unavailable" ? result.traffic : {
|
|
31447
|
+
...result.traffic,
|
|
31448
|
+
data: compactRows(result.traffic.data)
|
|
31449
|
+
},
|
|
31450
|
+
crawl: result.crawl.status === "unavailable" ? result.crawl : {
|
|
31451
|
+
...result.crawl,
|
|
31452
|
+
data: compactRows(result.crawl.data)
|
|
31453
|
+
},
|
|
31454
|
+
outputBudget: result.outputBudget
|
|
31455
|
+
};
|
|
31456
|
+
}
|
|
31457
|
+
function registerProviderTools(server) {
|
|
31458
|
+
server.registerTool("seo_bing_webmaster_overview", {
|
|
31459
|
+
description: "Find bounded Bing traffic, crawl, query, and page insights for one verified site",
|
|
31460
|
+
inputSchema: { site: z$1.string().url().max(2e3) }
|
|
31461
|
+
}, async ({ site }) => {
|
|
31462
|
+
try {
|
|
31463
|
+
const result = await bingWebmasterOverview({ site });
|
|
31464
|
+
const firstFinding = result.findings[0]?.title;
|
|
31465
|
+
return toolSuccess(`Bing evidence is ${result.dataStatus} with ${result.summary.findings} findings.${firstFinding ? ` Review first: ${firstFinding}.` : ""}`, compactBingWebmasterOverview(result));
|
|
31466
|
+
} catch (error) {
|
|
31467
|
+
return toolError(error);
|
|
31468
|
+
}
|
|
31469
|
+
});
|
|
31470
|
+
server.registerTool("seo_link_evidence", {
|
|
31471
|
+
description: "Review bounded referring-link evidence from Bing Webmaster or a local export",
|
|
31472
|
+
inputSchema: {
|
|
31473
|
+
site: z$1.string().url().max(2e3).optional(),
|
|
31474
|
+
file: z$1.string().min(1).max(4096).optional(),
|
|
31475
|
+
format: z$1.enum([
|
|
31476
|
+
"csv",
|
|
31477
|
+
"json",
|
|
31478
|
+
"jsonl"
|
|
31479
|
+
]).optional(),
|
|
31480
|
+
rowLimit: z$1.number().int().min(1).max(1e5).optional(),
|
|
31481
|
+
targetLimit: z$1.number().int().min(1).max(50).optional(),
|
|
31482
|
+
detailPagesPerTarget: z$1.number().int().min(1).max(3).optional(),
|
|
31483
|
+
limit: z$1.number().int().min(1).max(500).optional()
|
|
31484
|
+
}
|
|
31485
|
+
}, async ({ site, file, format, rowLimit, targetLimit, detailPagesPerTarget, limit }) => {
|
|
31486
|
+
try {
|
|
31487
|
+
if (Boolean(site) === Boolean(file)) throw new SeoError("INVALID_INPUT", "Pass exactly one of site for Bing or file for a local import.");
|
|
31488
|
+
const report = linkEvidenceReport({
|
|
31489
|
+
evidence: file ? await importLinkEvidence({
|
|
31490
|
+
file,
|
|
31491
|
+
format,
|
|
31492
|
+
rowLimit
|
|
31493
|
+
}) : await collectBingLinkEvidence({
|
|
31494
|
+
site: site ?? "",
|
|
31495
|
+
rowLimit,
|
|
31496
|
+
targetLimit,
|
|
31497
|
+
detailPagesPerTarget
|
|
31498
|
+
}),
|
|
31499
|
+
limit
|
|
31500
|
+
});
|
|
31501
|
+
return toolSuccess(`${report.summary.observedLinks} referring links were retained from ${report.provenance.provider}.`, report);
|
|
31502
|
+
} catch (error) {
|
|
31503
|
+
return toolError(error);
|
|
31504
|
+
}
|
|
31505
|
+
});
|
|
31506
|
+
}
|
|
31507
|
+
//#endregion
|
|
29465
31508
|
//#region packages/mcp/src/pseo-tools.ts
|
|
29466
31509
|
function compactPseoReport(result) {
|
|
29467
31510
|
return {
|
|
@@ -29754,6 +31797,91 @@ const REPORT_DEPTH = {
|
|
|
29754
31797
|
}
|
|
29755
31798
|
]
|
|
29756
31799
|
},
|
|
31800
|
+
"bing-webmaster-overview": {
|
|
31801
|
+
readOrder: [
|
|
31802
|
+
"dataStatus and provenance",
|
|
31803
|
+
"summary and findings, including verification steps",
|
|
31804
|
+
"traffic analysis current and previous periods, then changes",
|
|
31805
|
+
"crawl analysis current and previous snapshots, then changes",
|
|
31806
|
+
"query and page coverage before movements or opportunities",
|
|
31807
|
+
"outputBudget, per-section outputSelection, and caveats"
|
|
31808
|
+
],
|
|
31809
|
+
doNotClaim: [
|
|
31810
|
+
"Bing evidence does not describe Google or the whole search market, and inIndex is not URL-level proof of indexing.",
|
|
31811
|
+
"A missing weekly top-list entry is not zero traffic, and movements require observation in every week of both periods.",
|
|
31812
|
+
"Opportunity thresholds prioritize review and are not Bing requirements.",
|
|
31813
|
+
"Partial, capped, invalid, or unavailable rows cannot support an all-clear."
|
|
31814
|
+
],
|
|
31815
|
+
verify: "Check the same verified site in Bing Webmaster Tools and investigate the exact dates behind any crawl or traffic change.",
|
|
31816
|
+
related: [
|
|
31817
|
+
{
|
|
31818
|
+
id: "search-performance-overview",
|
|
31819
|
+
reason: "Adds Search Console evidence."
|
|
31820
|
+
},
|
|
31821
|
+
{
|
|
31822
|
+
id: "site-crawl",
|
|
31823
|
+
reason: "Checks the current site directly."
|
|
31824
|
+
},
|
|
31825
|
+
{
|
|
31826
|
+
id: "link-recovery",
|
|
31827
|
+
reason: "Finds broken search-value URLs."
|
|
31828
|
+
}
|
|
31829
|
+
]
|
|
31830
|
+
},
|
|
31831
|
+
"link-evidence": {
|
|
31832
|
+
readOrder: [
|
|
31833
|
+
"dataStatus, provenance, warnings, and caveats",
|
|
31834
|
+
"selection and summary coverage counts",
|
|
31835
|
+
"targetCounts, then the bounded links list"
|
|
31836
|
+
],
|
|
31837
|
+
doNotClaim: [
|
|
31838
|
+
"The retained rows are not a complete backlink index.",
|
|
31839
|
+
"Missing rows do not prove a link does not exist.",
|
|
31840
|
+
"Observed links do not establish quality, authority, value, or ranking impact."
|
|
31841
|
+
],
|
|
31842
|
+
verify: "Open a selected referring URL and confirm the link is still present before acting on it.",
|
|
31843
|
+
related: [
|
|
31844
|
+
{
|
|
31845
|
+
id: "link-recovery",
|
|
31846
|
+
reason: "Finds broken search-value URLs."
|
|
31847
|
+
},
|
|
31848
|
+
{
|
|
31849
|
+
id: "redirect-trace",
|
|
31850
|
+
reason: "Checks a target redirect chain."
|
|
31851
|
+
},
|
|
31852
|
+
{
|
|
31853
|
+
id: "site-crawl",
|
|
31854
|
+
reason: "Checks internal links and live pages."
|
|
31855
|
+
}
|
|
31856
|
+
]
|
|
31857
|
+
},
|
|
31858
|
+
"server-log-analysis": {
|
|
31859
|
+
readOrder: [
|
|
31860
|
+
"dataStatus, provenance coverage, warnings, and caveats",
|
|
31861
|
+
"summary and statusCodes",
|
|
31862
|
+
"crawlers, then the bounded crawlerPaths list and selection"
|
|
31863
|
+
],
|
|
31864
|
+
doNotClaim: [
|
|
31865
|
+
"A user-agent match does not verify the crawler identity or source IP.",
|
|
31866
|
+
"The supplied file does not prove that every server or time period is represented.",
|
|
31867
|
+
"Path rankings are partial when path tracking reached its limit."
|
|
31868
|
+
],
|
|
31869
|
+
verify: "Check important crawler errors against the original log line and deployment or server configuration before changing the site.",
|
|
31870
|
+
related: [
|
|
31871
|
+
{
|
|
31872
|
+
id: "site-crawl",
|
|
31873
|
+
reason: "Checks the current site directly."
|
|
31874
|
+
},
|
|
31875
|
+
{
|
|
31876
|
+
id: "crawler-rules",
|
|
31877
|
+
reason: "Explains current crawl checks."
|
|
31878
|
+
},
|
|
31879
|
+
{
|
|
31880
|
+
id: "redirect-trace",
|
|
31881
|
+
reason: "Traces one failing or redirecting path."
|
|
31882
|
+
}
|
|
31883
|
+
]
|
|
31884
|
+
},
|
|
29757
31885
|
cannibalisation: {
|
|
29758
31886
|
readOrder: [
|
|
29759
31887
|
"dataStatus and both source.pageExposure and source.propertyDemand with validation counts and completeness",
|
|
@@ -31037,6 +33165,27 @@ const REPORT_GUIDANCE = {
|
|
|
31037
33165
|
avoidWhen: ["You need the crawler to discover pages for you.", "You only need a deep review of one page."],
|
|
31038
33166
|
outcome: "Consistent technical checks for each supplied URL."
|
|
31039
33167
|
},
|
|
33168
|
+
"bing-webmaster-overview": {
|
|
33169
|
+
name: "Bing Webmaster overview",
|
|
33170
|
+
description: "Review bounded Bing traffic trends, crawl changes, and query and page opportunities for one verified site.",
|
|
33171
|
+
useWhen: ["You want processed search and crawl insights observed by Bing.", "You need a second search-engine view beside Search Console."],
|
|
33172
|
+
avoidWhen: ["Bing Webmaster is not connected for this installation.", "You need URL-level backlink rows or proof that a page is indexed."],
|
|
33173
|
+
outcome: "Prioritized Bing findings with traffic comparisons, crawl changes, query and page opportunities, provider coverage, and caveats."
|
|
33174
|
+
},
|
|
33175
|
+
"link-evidence": {
|
|
33176
|
+
name: "Referring link evidence",
|
|
33177
|
+
description: "Review a bounded set of pages that link to the site from Bing Webmaster or a local export.",
|
|
33178
|
+
useWhen: ["You want concrete referring URLs and anchor text where available.", "You need one normalized view of Bing or an exported CSV, JSON, or JSONL file."],
|
|
33179
|
+
avoidWhen: ["You need a complete backlink index or a domain authority score.", "You want to find broken internal links during a crawl."],
|
|
33180
|
+
outcome: "A bounded, deduplicated link list with target-page counts and explicit source coverage."
|
|
33181
|
+
},
|
|
33182
|
+
"server-log-analysis": {
|
|
33183
|
+
name: "Server log crawler analysis",
|
|
33184
|
+
description: "Stream a local access log to see observed search and AI crawler requests, statuses, and paths.",
|
|
33185
|
+
useWhen: ["You need evidence of crawler requests that reached the web server.", "You want to find crawler-facing 4xx or 5xx responses in a local access log."],
|
|
33186
|
+
avoidWhen: ["You do not have a combined or JSONL access log.", "You need to prove crawler identity from a user-agent string alone."],
|
|
33187
|
+
outcome: "Bounded crawler, status, and path aggregates with file coverage and parser limits kept visible."
|
|
33188
|
+
},
|
|
31040
33189
|
cannibalisation: {
|
|
31041
33190
|
name: "Query and URL overlap",
|
|
31042
33191
|
description: "Find search queries that appear across several URLs and decide whether the overlap needs attention.",
|
|
@@ -31543,6 +33692,35 @@ function registerReportTools(server) {
|
|
|
31543
33692
|
registerSecondPageTool(server);
|
|
31544
33693
|
}
|
|
31545
33694
|
//#endregion
|
|
33695
|
+
//#region packages/mcp/src/server-log-tools.ts
|
|
33696
|
+
function registerServerLogTools(server) {
|
|
33697
|
+
server.registerTool("seo_server_log_analysis", {
|
|
33698
|
+
description: "Stream a local access log into bounded search and AI crawler evidence",
|
|
33699
|
+
inputSchema: {
|
|
33700
|
+
file: z$1.string().min(1).max(4096),
|
|
33701
|
+
format: z$1.enum(["combined", "jsonl"]).optional(),
|
|
33702
|
+
rowLimit: z$1.number().int().min(1).max(1e7).optional(),
|
|
33703
|
+
pathLimit: z$1.number().int().min(1).max(1e5).optional(),
|
|
33704
|
+
limit: z$1.number().int().min(1).max(500).optional()
|
|
33705
|
+
}
|
|
33706
|
+
}, async ({ file, format, rowLimit, pathLimit, limit }) => {
|
|
33707
|
+
try {
|
|
33708
|
+
const report = serverLogReport({
|
|
33709
|
+
evidence: await importServerLog({
|
|
33710
|
+
file,
|
|
33711
|
+
format,
|
|
33712
|
+
rowLimit,
|
|
33713
|
+
pathLimit
|
|
33714
|
+
}),
|
|
33715
|
+
limit
|
|
33716
|
+
});
|
|
33717
|
+
return toolSuccess(`${report.summary.crawlerRows} crawler requests were observed in ${report.summary.parsedRows} parsed log rows.`, report);
|
|
33718
|
+
} catch (error) {
|
|
33719
|
+
return toolError(error);
|
|
33720
|
+
}
|
|
33721
|
+
});
|
|
33722
|
+
}
|
|
33723
|
+
//#endregion
|
|
31546
33724
|
//#region packages/mcp/src/agent-output-budget.ts
|
|
31547
33725
|
const AGENT_STRUCTURED_OUTPUT_MAX_BYTES = 96 * 1024;
|
|
31548
33726
|
function jsonBytes(value) {
|
|
@@ -31939,6 +34117,16 @@ const reportGroups = [
|
|
|
31939
34117
|
"seo_redirect_trace"
|
|
31940
34118
|
]
|
|
31941
34119
|
},
|
|
34120
|
+
{
|
|
34121
|
+
category: "monitoring",
|
|
34122
|
+
register: registerProviderTools,
|
|
34123
|
+
names: ["seo_bing_webmaster_overview", "seo_link_evidence"]
|
|
34124
|
+
},
|
|
34125
|
+
{
|
|
34126
|
+
category: "monitoring",
|
|
34127
|
+
register: registerServerLogTools,
|
|
34128
|
+
names: ["seo_server_log_analysis"]
|
|
34129
|
+
},
|
|
31942
34130
|
{
|
|
31943
34131
|
category: "reporting",
|
|
31944
34132
|
register: registerReportTools,
|