seo 0.2.10 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -1
- package/dist/cli.js +212 -130
- package/dist/cli.js.map +1 -1
- package/dist/{dist-CBY8QOnv.js → dist-BB0yH7qK.js} +1 -1
- package/dist/{dist-w2XtKwjW.js → dist-Dwdogl4j.js} +85 -80
- package/dist/dist-Dwdogl4j.js.map +1 -0
- package/dist/index.d.ts +1984 -71
- package/dist/index.js +2900 -247
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +2647 -197
- package/dist/mcp.js.map +1 -1
- package/dist/{package-92W1Zc-o.js → package-B_swz2Xd.js} +2 -2
- package/dist/{package-92W1Zc-o.js.map → package-B_swz2Xd.js.map} +1 -1
- package/evals/bing-webmaster-overview.json +31 -0
- package/evals/link-evidence.json +29 -0
- package/evals/seo.json +16 -0
- package/evals/server-log-analysis.json +29 -0
- package/package.json +3 -3
- package/skills/seo/SKILL.md +56 -67
- package/dist/dist-w2XtKwjW.js.map +0 -1
package/dist/mcp.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { n as load, t as version } from "./package-
|
|
2
|
+
import { n as load, t as version } from "./package-B_swz2Xd.js";
|
|
3
3
|
import { Headers, fetch as fetch$1 } from "undici";
|
|
4
|
-
import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { chmodSync, createReadStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
5
5
|
import { arch, homedir, platform, totalmem } from "node:os";
|
|
6
|
-
import { dirname, join } from "node:path";
|
|
6
|
+
import { dirname, extname, join, resolve } from "node:path";
|
|
7
7
|
import { z } from "zod";
|
|
8
8
|
import lockfile from "proper-lockfile";
|
|
9
9
|
import { createHash, randomUUID, sign } from "node:crypto";
|
|
@@ -21,9 +21,9 @@ import { gunzipSync } from "node:zlib";
|
|
|
21
21
|
import { execFile } from "node:child_process";
|
|
22
22
|
import { fileURLToPath } from "node:url";
|
|
23
23
|
import { promisify } from "node:util";
|
|
24
|
+
import { readFile, stat } from "node:fs/promises";
|
|
24
25
|
import "pino";
|
|
25
26
|
import "pino-pretty";
|
|
26
|
-
import "node:fs/promises";
|
|
27
27
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
28
28
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
29
29
|
import * as z$1 from "zod/v4";
|
|
@@ -86,12 +86,12 @@ var SeoError = class extends Error {
|
|
|
86
86
|
this.retryable = errorMetadata[code].retryable;
|
|
87
87
|
}
|
|
88
88
|
};
|
|
89
|
-
function errorMessage$
|
|
89
|
+
function errorMessage$3(error) {
|
|
90
90
|
return error instanceof Error ? error.message : String(error);
|
|
91
91
|
}
|
|
92
92
|
function toSeoError(error) {
|
|
93
93
|
if (error instanceof SeoError) return error;
|
|
94
|
-
const message = errorMessage$
|
|
94
|
+
const message = errorMessage$3(error);
|
|
95
95
|
if (/not logged in/i.test(message)) return new SeoError("AUTH_REQUIRED", message);
|
|
96
96
|
if (/oauth client config.*missing|shared seo google app/i.test(message)) return new SeoError("AUTH_CONFIG_REQUIRED", message);
|
|
97
97
|
if (/invalid_grant|refresh token.*no longer valid/i.test(message)) return new SeoError("AUTH_EXPIRED", message);
|
|
@@ -148,6 +148,7 @@ function resolveSeoCliPaths(input = {}) {
|
|
|
148
148
|
configFile: join(configDir, "config.json"),
|
|
149
149
|
tokensFile: join(configDir, "tokens.json"),
|
|
150
150
|
oauthClientFile: join(configDir, "oauth-client.json"),
|
|
151
|
+
providerSecretsFile: join(configDir, "provider-secrets.json"),
|
|
151
152
|
telemetryStateFile: join(configDir, "telemetry.json"),
|
|
152
153
|
cacheDbFile: join(cacheDir, "cache.db")
|
|
153
154
|
};
|
|
@@ -227,6 +228,7 @@ const clientProfileSchema = z.object({
|
|
|
227
228
|
watchUrls: z.array(z.string()).default([]),
|
|
228
229
|
brandTerms: z.array(z.string()).default([]),
|
|
229
230
|
analytics: analyticsConnectionsSchema,
|
|
231
|
+
searchEngines: z.object({ bing: z.object({ siteUrl: z.string() }).optional() }).optional(),
|
|
230
232
|
reportDay: z.number().int().min(1).max(31).optional(),
|
|
231
233
|
technicalWeekday: z.number().int().min(0).max(7).optional(),
|
|
232
234
|
isDefault: z.boolean().optional(),
|
|
@@ -305,7 +307,7 @@ async function deleteKeyringPassword(service, account) {
|
|
|
305
307
|
}
|
|
306
308
|
//#endregion
|
|
307
309
|
//#region packages/core/dist/storage/config.js
|
|
308
|
-
const KEYRING_SERVICE = "seo";
|
|
310
|
+
const KEYRING_SERVICE$1 = "seo";
|
|
309
311
|
const PRIVATE_FILE_MODE = 384;
|
|
310
312
|
function tightenConfigPermissions(path) {
|
|
311
313
|
try {
|
|
@@ -336,7 +338,7 @@ function tokenMetadata(tokens) {
|
|
|
336
338
|
}
|
|
337
339
|
async function readKeyringTokens(tokens) {
|
|
338
340
|
const account = getKeyringAccount(tokens);
|
|
339
|
-
const [accessToken, refreshToken] = await Promise.all([getKeyringPassword(KEYRING_SERVICE, `${account}:access`), getKeyringPassword(KEYRING_SERVICE, `${account}:refresh`)]);
|
|
341
|
+
const [accessToken, refreshToken] = await Promise.all([getKeyringPassword(KEYRING_SERVICE$1, `${account}:access`), getKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`)]);
|
|
340
342
|
return {
|
|
341
343
|
accessToken: accessToken ?? void 0,
|
|
342
344
|
refreshToken: refreshToken ?? void 0
|
|
@@ -344,14 +346,14 @@ async function readKeyringTokens(tokens) {
|
|
|
344
346
|
}
|
|
345
347
|
async function writeKeyringTokens(tokens) {
|
|
346
348
|
const account = getKeyringAccount(tokens);
|
|
347
|
-
if (tokens.access_token) await setKeyringPassword(KEYRING_SERVICE, `${account}:access`, tokens.access_token);
|
|
348
|
-
else await deleteKeyringPassword(KEYRING_SERVICE, `${account}:access`);
|
|
349
|
-
if (tokens.refresh_token) await setKeyringPassword(KEYRING_SERVICE, `${account}:refresh`, tokens.refresh_token);
|
|
350
|
-
else await deleteKeyringPassword(KEYRING_SERVICE, `${account}:refresh`);
|
|
349
|
+
if (tokens.access_token) await setKeyringPassword(KEYRING_SERVICE$1, `${account}:access`, tokens.access_token);
|
|
350
|
+
else await deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:access`);
|
|
351
|
+
if (tokens.refresh_token) await setKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`, tokens.refresh_token);
|
|
352
|
+
else await deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`);
|
|
351
353
|
}
|
|
352
354
|
async function deleteKeyringTokens(tokens) {
|
|
353
355
|
const account = getKeyringAccount(tokens);
|
|
354
|
-
await Promise.all([deleteKeyringPassword(KEYRING_SERVICE, `${account}:access`), deleteKeyringPassword(KEYRING_SERVICE, `${account}:refresh`)]);
|
|
356
|
+
await Promise.all([deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:access`), deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`)]);
|
|
355
357
|
}
|
|
356
358
|
async function readTokens() {
|
|
357
359
|
ensureSeoCliDirs();
|
|
@@ -2132,6 +2134,43 @@ function extractCanonicalEvidence($, headers, baseUrl) {
|
|
|
2132
2134
|
return evidenceFromCandidates([...htmlCandidates($, baseUrl), ...headerCandidates(headers, baseUrl)]);
|
|
2133
2135
|
}
|
|
2134
2136
|
//#endregion
|
|
2137
|
+
//#region packages/core/dist/extract/content-sketch.js
|
|
2138
|
+
const SHINGLE_SIZE = 5;
|
|
2139
|
+
const MAX_HASHES = 32;
|
|
2140
|
+
function words(value) {
|
|
2141
|
+
return value.normalize("NFKC").toLocaleLowerCase("en").match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
2142
|
+
}
|
|
2143
|
+
function sampleIndexes(total, limit) {
|
|
2144
|
+
if (total <= limit) return Array.from({ length: total }, (_, index) => index);
|
|
2145
|
+
return Array.from({ length: limit }, (_, index) => Math.round(index * (total - 1) / (limit - 1)));
|
|
2146
|
+
}
|
|
2147
|
+
function digest(value) {
|
|
2148
|
+
return createHash("sha256").update(value).digest("hex").slice(0, 16);
|
|
2149
|
+
}
|
|
2150
|
+
function contentSketch(value) {
|
|
2151
|
+
const tokens = words(value);
|
|
2152
|
+
const hashes = sampleIndexes(Math.max(0, tokens.length - SHINGLE_SIZE + 1), MAX_HASHES).map((index) => digest(tokens.slice(index, index + SHINGLE_SIZE).join(" ")));
|
|
2153
|
+
return {
|
|
2154
|
+
version: 1,
|
|
2155
|
+
wordCount: tokens.length,
|
|
2156
|
+
shingleSize: SHINGLE_SIZE,
|
|
2157
|
+
sampledShingles: hashes.length,
|
|
2158
|
+
hashes
|
|
2159
|
+
};
|
|
2160
|
+
}
|
|
2161
|
+
function contentSketchCoverage(source, candidate) {
|
|
2162
|
+
if (!source || source.hashes.length < 4) return null;
|
|
2163
|
+
const targets = new Set(source.hashes);
|
|
2164
|
+
const matched = /* @__PURE__ */ new Set();
|
|
2165
|
+
const tokens = words(candidate);
|
|
2166
|
+
for (let index = 0; index <= tokens.length - source.shingleSize; index += 1) {
|
|
2167
|
+
const hash = digest(tokens.slice(index, index + source.shingleSize).join(" "));
|
|
2168
|
+
if (targets.has(hash)) matched.add(hash);
|
|
2169
|
+
if (matched.size === targets.size) break;
|
|
2170
|
+
}
|
|
2171
|
+
return Math.round(matched.size / targets.size * 1e3) / 1e3;
|
|
2172
|
+
}
|
|
2173
|
+
//#endregion
|
|
2135
2174
|
//#region packages/core/dist/extract/main-content.js
|
|
2136
2175
|
function readableError(error) {
|
|
2137
2176
|
return error instanceof Error ? error.message : String(error);
|
|
@@ -2749,6 +2788,48 @@ const SOCIAL_PROFILE_HOSTS = [
|
|
|
2749
2788
|
"x.com",
|
|
2750
2789
|
"youtube.com"
|
|
2751
2790
|
];
|
|
2791
|
+
const LOGIN_PATH_PATTERN = /(?:^|\/)(?:auth|login|log-in|signin|sign-in|sso)(?:\/|$)/iu;
|
|
2792
|
+
const LOGIN_TITLE_PATTERN = /\b(?:authenticate|log[ -]?in|sign[ -]?in)\b/iu;
|
|
2793
|
+
function softAuthenticationGate($, finalUrl, title) {
|
|
2794
|
+
const passwordForms = $("form").toArray().filter((form) => $(form).find("input[type=\"password\" i]").length > 0);
|
|
2795
|
+
if (passwordForms.length === 0) return void 0;
|
|
2796
|
+
const pagePath = new URL(finalUrl).pathname;
|
|
2797
|
+
const action = $(passwordForms[0]).attr("action");
|
|
2798
|
+
const actionUrl = action ? httpUrl$1(action, finalUrl) : void 0;
|
|
2799
|
+
const actionPath = actionUrl ? new URL(actionUrl).pathname : void 0;
|
|
2800
|
+
const indicators = [
|
|
2801
|
+
...LOGIN_PATH_PATTERN.test(pagePath) ? ["login-path"] : [],
|
|
2802
|
+
...LOGIN_TITLE_PATTERN.test(title ?? "") ? ["login-title"] : [],
|
|
2803
|
+
...LOGIN_PATH_PATTERN.test(actionPath ?? "") ? ["login-form-action"] : []
|
|
2804
|
+
];
|
|
2805
|
+
if (indicators.length === 0) return void 0;
|
|
2806
|
+
return {
|
|
2807
|
+
kind: "login-form",
|
|
2808
|
+
indicators,
|
|
2809
|
+
...actionPath ? { formActionPath: actionPath } : {}
|
|
2810
|
+
};
|
|
2811
|
+
}
|
|
2812
|
+
function tabbedContent($) {
|
|
2813
|
+
const groups = $("[role=\"tablist\" i]").length;
|
|
2814
|
+
const panels = $("[role=\"tabpanel\" i]").toArray();
|
|
2815
|
+
if (groups === 0 || panels.length === 0) return void 0;
|
|
2816
|
+
const panelSketches = panels.slice(0, 12).map((panel) => {
|
|
2817
|
+
const labelledBy = $(panel).attr("aria-labelledby");
|
|
2818
|
+
const labelledElement = labelledBy ? $("[id]").toArray().find((element) => $(element).attr("id") === labelledBy) : void 0;
|
|
2819
|
+
const label = safeText$1($(panel).attr("aria-label") ?? (labelledElement ? $(labelledElement).text() : void 0));
|
|
2820
|
+
return {
|
|
2821
|
+
...label ? { label: label.slice(0, 80) } : {},
|
|
2822
|
+
sketch: contentSketch($(panel).text())
|
|
2823
|
+
};
|
|
2824
|
+
});
|
|
2825
|
+
return {
|
|
2826
|
+
groups,
|
|
2827
|
+
panels: panels.length,
|
|
2828
|
+
retainedPanels: panelSketches.length,
|
|
2829
|
+
truncated: panels.length > panelSketches.length,
|
|
2830
|
+
panelSketches
|
|
2831
|
+
};
|
|
2832
|
+
}
|
|
2752
2833
|
function safeText$1(value) {
|
|
2753
2834
|
const trimmed = value?.replace(/\s+/g, " ").trim();
|
|
2754
2835
|
return trimmed ? trimmed : void 0;
|
|
@@ -2765,7 +2846,7 @@ function absoluteUrl$2(value, base) {
|
|
|
2765
2846
|
return;
|
|
2766
2847
|
}
|
|
2767
2848
|
}
|
|
2768
|
-
function httpUrl(value, base) {
|
|
2849
|
+
function httpUrl$1(value, base) {
|
|
2769
2850
|
const resolved = absoluteUrl$2(value, base);
|
|
2770
2851
|
if (!resolved) return void 0;
|
|
2771
2852
|
const protocol = new URL(resolved).protocol;
|
|
@@ -2917,14 +2998,14 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
2917
2998
|
let invalidHreflangCount = 0;
|
|
2918
2999
|
for (const element of $("link[rel~=\"alternate\"][hreflang][href]").toArray()) {
|
|
2919
3000
|
const language = $(element).attr("hreflang") ?? "";
|
|
2920
|
-
const href = httpUrl($(element).attr("href"), fetchResult.finalUrl);
|
|
3001
|
+
const href = httpUrl$1($(element).attr("href"), fetchResult.finalUrl);
|
|
2921
3002
|
if (language && href) hreflang.push({
|
|
2922
3003
|
hreflang: language,
|
|
2923
3004
|
href
|
|
2924
3005
|
});
|
|
2925
3006
|
else invalidHreflangCount += 1;
|
|
2926
3007
|
}
|
|
2927
|
-
const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
|
|
3008
|
+
const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl$1($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
|
|
2928
3009
|
const openGraph = Object.fromEntries($("meta[property^=\"og:\"]").toArray().map((element) => [$(element).attr("property") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
2929
3010
|
const twitter = Object.fromEntries($("meta[name^=\"twitter:\"]").toArray().map((element) => [$(element).attr("name") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
2930
3011
|
const structuredData = extractStructuredData($, fetchResult.finalUrl, { retainJsonLd: extractor !== "crawler" });
|
|
@@ -2975,10 +3056,11 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
2975
3056
|
html: sanitizedContentHtml($, fetchResult.html, fetchResult.finalUrl)
|
|
2976
3057
|
}, extractor, dependencies);
|
|
2977
3058
|
const { text, excerpt } = content;
|
|
3059
|
+
const title = safeText$1($("title").first().text());
|
|
2978
3060
|
return {
|
|
2979
3061
|
url: fetchResult.url,
|
|
2980
3062
|
finalUrl: fetchResult.finalUrl,
|
|
2981
|
-
title
|
|
3063
|
+
title,
|
|
2982
3064
|
metaDescription: safeText$1($("meta[name=\"description\"]").attr("content")),
|
|
2983
3065
|
metaRobots,
|
|
2984
3066
|
xRobotsTag: safeText$1(headerValue$4(fetchResult.headers, "x-robots-tag")),
|
|
@@ -2998,7 +3080,10 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
2998
3080
|
externalLinks: [...crawlerExternalLinks],
|
|
2999
3081
|
internalAnchorSamples,
|
|
3000
3082
|
externalAnchorSamples,
|
|
3001
|
-
socialProfileLinks: [...socialProfileLinks]
|
|
3083
|
+
socialProfileLinks: [...socialProfileLinks],
|
|
3084
|
+
contentSketch: contentSketch(text),
|
|
3085
|
+
softAuthenticationGate: softAuthenticationGate($, fetchResult.finalUrl, title),
|
|
3086
|
+
tabbedContent: tabbedContent($)
|
|
3002
3087
|
} } : {},
|
|
3003
3088
|
markdownAlternates,
|
|
3004
3089
|
hreflang,
|
|
@@ -4341,7 +4426,7 @@ function urlInspectionQuotaDate(now) {
|
|
|
4341
4426
|
function urlInspectionQuotaResetAt(now) {
|
|
4342
4427
|
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1));
|
|
4343
4428
|
}
|
|
4344
|
-
function boundedLimit(value) {
|
|
4429
|
+
function boundedLimit$1(value) {
|
|
4345
4430
|
const limit = value ?? 2e3;
|
|
4346
4431
|
if (!Number.isInteger(limit) || limit < 1 || limit > 2e3) throw new SeoError("INVALID_INPUT", "URL Inspection daily limit must be an integer from 1 to 2000.");
|
|
4347
4432
|
return limit;
|
|
@@ -4374,7 +4459,7 @@ function reserveUrlInspectionQuota(input) {
|
|
|
4374
4459
|
const database = input.database ?? getDb();
|
|
4375
4460
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
4376
4461
|
const nowMs = now.getTime();
|
|
4377
|
-
const requestedLimit = boundedLimit(input.limit);
|
|
4462
|
+
const requestedLimit = boundedLimit$1(input.limit);
|
|
4378
4463
|
const quotaDate = urlInspectionQuotaDate(now);
|
|
4379
4464
|
const resetAt = urlInspectionQuotaResetAt(now);
|
|
4380
4465
|
const result = database.transaction(() => {
|
|
@@ -6157,13 +6242,13 @@ function isLowActionabilityQuery(query) {
|
|
|
6157
6242
|
//#endregion
|
|
6158
6243
|
//#region packages/core/dist/analyze/page-opportunities-selection.js
|
|
6159
6244
|
const DEFAULT_MIN_IMPRESSIONS$3 = 10;
|
|
6160
|
-
const DEFAULT_LIMIT$
|
|
6161
|
-
const MAX_LIMIT$
|
|
6245
|
+
const DEFAULT_LIMIT$4 = 25;
|
|
6246
|
+
const MAX_LIMIT$4 = 100;
|
|
6162
6247
|
function integerInRange$2(value, fallback, minimum, maximum) {
|
|
6163
6248
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
6164
6249
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
6165
6250
|
}
|
|
6166
|
-
function isHttpUrl$
|
|
6251
|
+
function isHttpUrl$2(value) {
|
|
6167
6252
|
try {
|
|
6168
6253
|
const parsed = new URL(value);
|
|
6169
6254
|
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
@@ -6185,7 +6270,7 @@ function isBrandRow(input) {
|
|
|
6185
6270
|
function normalizePageOpportunityOptions(input) {
|
|
6186
6271
|
return {
|
|
6187
6272
|
minImpressions: integerInRange$2(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$3, 1, Number.MAX_SAFE_INTEGER),
|
|
6188
|
-
limit: integerInRange$2(input.limit, DEFAULT_LIMIT$
|
|
6273
|
+
limit: integerInRange$2(input.limit, DEFAULT_LIMIT$4, 1, MAX_LIMIT$4)
|
|
6189
6274
|
};
|
|
6190
6275
|
}
|
|
6191
6276
|
function selectTargetRows(input) {
|
|
@@ -6204,7 +6289,7 @@ function selectTargetRows(input) {
|
|
|
6204
6289
|
for (const row of input.rows) {
|
|
6205
6290
|
const query = row.keys[0]?.trim() ?? "";
|
|
6206
6291
|
const page = row.keys[1] ?? "";
|
|
6207
|
-
if (row.keys.length < 2 || !query || !isHttpUrl$
|
|
6292
|
+
if (row.keys.length < 2 || !query || !isHttpUrl$2(page) || !hasValidMetrics(row)) {
|
|
6208
6293
|
selection.invalidRows++;
|
|
6209
6294
|
continue;
|
|
6210
6295
|
}
|
|
@@ -6240,7 +6325,7 @@ function selectTargetRows(input) {
|
|
|
6240
6325
|
function selectBenchmarkRows$1(input) {
|
|
6241
6326
|
return input.rows.filter((row) => {
|
|
6242
6327
|
const query = row.keys[0]?.trim() ?? "";
|
|
6243
|
-
return row.keys.length >= 2 && isHttpUrl$
|
|
6328
|
+
return row.keys.length >= 2 && isHttpUrl$2(row.keys[1] ?? "") && hasValidMetrics(row) && row.position >= 1 && row.position <= 10 && Boolean(query) && !isLowActionabilityQuery(query) && !isBrandRow({
|
|
6244
6329
|
query,
|
|
6245
6330
|
site: input.site,
|
|
6246
6331
|
brandTerms: input.brandTerms,
|
|
@@ -6422,7 +6507,7 @@ const defaultDependencies$14 = {
|
|
|
6422
6507
|
extract: extractPage,
|
|
6423
6508
|
now: () => /* @__PURE__ */ new Date()
|
|
6424
6509
|
};
|
|
6425
|
-
function errorMessage$
|
|
6510
|
+
function errorMessage$2(error) {
|
|
6426
6511
|
return error instanceof Error ? error.message : String(error);
|
|
6427
6512
|
}
|
|
6428
6513
|
function unique$4(values) {
|
|
@@ -6443,7 +6528,7 @@ async function verifyPage(input) {
|
|
|
6443
6528
|
});
|
|
6444
6529
|
warnings.push(...fetched.warnings);
|
|
6445
6530
|
} catch (error) {
|
|
6446
|
-
const reason = `Page fetch failed: ${errorMessage$
|
|
6531
|
+
const reason = `Page fetch failed: ${errorMessage$2(error)}`;
|
|
6447
6532
|
return {
|
|
6448
6533
|
verification: {
|
|
6449
6534
|
status: "failed",
|
|
@@ -6465,7 +6550,7 @@ async function verifyPage(input) {
|
|
|
6465
6550
|
warnings: unique$4(warnings)
|
|
6466
6551
|
};
|
|
6467
6552
|
} catch (error) {
|
|
6468
|
-
const reason = `Page extraction failed: ${errorMessage$
|
|
6553
|
+
const reason = `Page extraction failed: ${errorMessage$2(error)}`;
|
|
6469
6554
|
return {
|
|
6470
6555
|
fetched,
|
|
6471
6556
|
verification: {
|
|
@@ -7103,6 +7188,8 @@ function markdownQuality(markdown, page) {
|
|
|
7103
7188
|
const codeFences = markdown.match(/^```/gmu)?.length ?? 0;
|
|
7104
7189
|
const wordCount = bodyWordCount(markdown);
|
|
7105
7190
|
const sourceWordCount = page.wordCount ?? 0;
|
|
7191
|
+
const evaluatedPanels = (page.tabbedContent?.panelSketches ?? []).map((panel) => contentSketchCoverage(panel.sketch, markdown)).filter((coverage) => coverage !== null);
|
|
7192
|
+
const retainedPanels = evaluatedPanels.filter((coverage) => coverage >= .6).length;
|
|
7106
7193
|
return {
|
|
7107
7194
|
frontmatterTitle: /^---[\s\S]*?^title:\s*.+$/mu.test(markdown),
|
|
7108
7195
|
h1Count: markdown.match(/^#\s+\S.+$/gmu)?.length ?? 0,
|
|
@@ -7119,7 +7206,15 @@ function markdownQuality(markdown, page) {
|
|
|
7119
7206
|
sourceWordCount,
|
|
7120
7207
|
wordRetentionRatio: sourceWordCount > 0 ? Math.round(wordCount / sourceWordCount * 1e3) / 1e3 : null,
|
|
7121
7208
|
introductoryCopyRetained: introProbe ? normalizedMarkdown.includes(introProbe) : null,
|
|
7122
|
-
navigationOnly: bodyWordCount(markdown) < 25
|
|
7209
|
+
navigationOnly: bodyWordCount(markdown) < 25,
|
|
7210
|
+
contentSketchCoverage: contentSketchCoverage(page.contentSketch, markdown),
|
|
7211
|
+
tabbedContent: {
|
|
7212
|
+
detectedPanels: page.tabbedContent?.panels ?? 0,
|
|
7213
|
+
evaluatedPanels: evaluatedPanels.length,
|
|
7214
|
+
retainedPanels,
|
|
7215
|
+
missingPanels: Math.max(0, evaluatedPanels.length - retainedPanels),
|
|
7216
|
+
complete: evaluatedPanels.length === 0 ? null : retainedPanels === evaluatedPanels.length
|
|
7217
|
+
}
|
|
7123
7218
|
};
|
|
7124
7219
|
}
|
|
7125
7220
|
function combinedSignal(timeoutMs, signal) {
|
|
@@ -7173,6 +7268,8 @@ async function fetchRepresentation(input) {
|
|
|
7173
7268
|
status: response.status,
|
|
7174
7269
|
contentType: response.headers.get("content-type") ?? void 0,
|
|
7175
7270
|
bytes: Buffer.byteLength(body),
|
|
7271
|
+
characters: body.length,
|
|
7272
|
+
estimatedTokens: Math.ceil(body.length / 4),
|
|
7176
7273
|
sha256: sha256(body),
|
|
7177
7274
|
canonicalUrl: link.find((entry) => entry.rel.includes("canonical"))?.url,
|
|
7178
7275
|
alternateUrl: link.find((entry) => entry.rel.includes("alternate") && entry.type === "text/markdown")?.url,
|
|
@@ -7231,11 +7328,15 @@ async function inspectMarkdownPage(input) {
|
|
|
7231
7328
|
if (explicit) observation.explicit = explicit.observation;
|
|
7232
7329
|
observation.negotiated = negotiated.observation;
|
|
7233
7330
|
observation.repeated = repeated.observation;
|
|
7234
|
-
|
|
7235
|
-
const
|
|
7236
|
-
|
|
7237
|
-
|
|
7238
|
-
|
|
7331
|
+
const isMarkdown = (result) => result?.observation.status !== void 0 && result.observation.status >= 200 && result.observation.status < 300 && /^\s*text\/markdown\b/iu.test(result.observation.contentType ?? "");
|
|
7332
|
+
const explicitMarkdown = isMarkdown(explicit) ? explicit : void 0;
|
|
7333
|
+
const negotiatedMarkdown = isMarkdown(negotiated) ? negotiated : void 0;
|
|
7334
|
+
const repeatedMarkdown = isMarkdown(repeated) ? repeated : void 0;
|
|
7335
|
+
observation.explicitMatchesNegotiated = explicitMarkdown?.observation.sha256 && negotiatedMarkdown?.observation.sha256 ? explicitMarkdown.observation.sha256 === negotiatedMarkdown.observation.sha256 : null;
|
|
7336
|
+
const primary = explicitMarkdown ?? negotiatedMarkdown;
|
|
7337
|
+
observation.repeatedHashStable = primary?.observation.sha256 && repeatedMarkdown?.observation.sha256 ? primary.observation.sha256 === repeatedMarkdown.observation.sha256 : null;
|
|
7338
|
+
observation.markdownCanonicalMatchesHtml = sameDocument(primary?.observation.canonicalUrl, input.page.finalUrl);
|
|
7339
|
+
if (primary?.body) observation.quality = markdownQuality(primary.body, input.page);
|
|
7239
7340
|
return observation;
|
|
7240
7341
|
}
|
|
7241
7342
|
async function fetchText(input) {
|
|
@@ -7479,17 +7580,16 @@ async function inspectRouteManifest(input) {
|
|
|
7479
7580
|
const declared = Array.isArray(parsed.pages) ? parsed.pages : [];
|
|
7480
7581
|
const declaredHtmlRoutes = declared.map((page) => page.htmlPath).filter((path) => typeof path === "string").sort();
|
|
7481
7582
|
const declaredMarkdownRoutes = declared.map((page) => page.markdownPath).filter((path) => typeof path === "string").sort();
|
|
7482
|
-
const crawledHtmlRoutes = new Set(input.pages.map((page) => new URL(page.finalUrl).pathname.replace(/\/$/u, "") || "/"));
|
|
7483
|
-
const observedMarkdownRoutes = new Set(input.observations.flatMap((page) => page.advertisedUrls.map((value) => new URL(value).pathname)));
|
|
7484
7583
|
return {
|
|
7485
7584
|
url,
|
|
7486
7585
|
status: result.response.status,
|
|
7487
7586
|
valid: Array.isArray(parsed.pages),
|
|
7488
7587
|
declaredHtmlRoutes,
|
|
7489
7588
|
declaredMarkdownRoutes,
|
|
7490
|
-
missingHtmlRoutes:
|
|
7491
|
-
missingMarkdownRoutes:
|
|
7492
|
-
orphanMarkdownRoutes: [
|
|
7589
|
+
missingHtmlRoutes: [],
|
|
7590
|
+
missingMarkdownRoutes: [],
|
|
7591
|
+
orphanMarkdownRoutes: [],
|
|
7592
|
+
comparisonStatus: "partial"
|
|
7493
7593
|
};
|
|
7494
7594
|
} catch (error) {
|
|
7495
7595
|
return {
|
|
@@ -7500,6 +7600,7 @@ async function inspectRouteManifest(input) {
|
|
|
7500
7600
|
missingHtmlRoutes: [],
|
|
7501
7601
|
missingMarkdownRoutes: [],
|
|
7502
7602
|
orphanMarkdownRoutes: [],
|
|
7603
|
+
comparisonStatus: "not-applicable",
|
|
7503
7604
|
error: safeError(error)
|
|
7504
7605
|
};
|
|
7505
7606
|
}
|
|
@@ -7543,9 +7644,7 @@ async function collectAgentDiscovery(input) {
|
|
|
7543
7644
|
}),
|
|
7544
7645
|
inspectRouteManifest({
|
|
7545
7646
|
...input,
|
|
7546
|
-
origin
|
|
7547
|
-
pages,
|
|
7548
|
-
observations
|
|
7647
|
+
origin
|
|
7549
7648
|
}),
|
|
7550
7649
|
inspectAgentEndpoints({
|
|
7551
7650
|
...input,
|
|
@@ -7650,20 +7749,13 @@ async function collectAgentDiscovery(input) {
|
|
|
7650
7749
|
};
|
|
7651
7750
|
}
|
|
7652
7751
|
//#endregion
|
|
7653
|
-
//#region packages/core/dist/analyze/crawler/agent-readiness.js
|
|
7654
|
-
function check$
|
|
7752
|
+
//#region packages/core/dist/analyze/crawler/agent-readiness-markdown.js
|
|
7753
|
+
function check$2(section, input) {
|
|
7655
7754
|
return {
|
|
7656
7755
|
section,
|
|
7657
7756
|
...input
|
|
7658
7757
|
};
|
|
7659
7758
|
}
|
|
7660
|
-
function section$1(id, title, checks) {
|
|
7661
|
-
return {
|
|
7662
|
-
id,
|
|
7663
|
-
title,
|
|
7664
|
-
checks
|
|
7665
|
-
};
|
|
7666
|
-
}
|
|
7667
7759
|
function failedMarkdownUrls(discovery) {
|
|
7668
7760
|
return discovery.markdownAlternates.pages.filter((page) => {
|
|
7669
7761
|
const negotiatedOk = page.negotiated?.status === 200 && /^\s*text\/markdown\b/iu.test(page.negotiated.contentType ?? "") && page.negotiated.varyAccept;
|
|
@@ -7674,38 +7766,12 @@ function failedMarkdownUrls(discovery) {
|
|
|
7674
7766
|
function qualityFailureUrls(discovery) {
|
|
7675
7767
|
return discovery.markdownAlternates.pages.filter((page) => {
|
|
7676
7768
|
const quality = page.quality;
|
|
7677
|
-
if (!quality) return
|
|
7678
|
-
return !quality.frontmatterTitle || quality.h1Count !== 1 || !quality.codeFenceBalanced || quality.rawHtmlTags > 0 || quality.rawSvgTags > 0 || quality.rawScriptTags > 0 || quality.rawStyleTags > 0 || quality.navigationOnly || quality.repeatedLines > 0 || quality.wordRetentionRatio !== null && quality.wordRetentionRatio < .4;
|
|
7769
|
+
if (!quality) return false;
|
|
7770
|
+
return !quality.frontmatterTitle || quality.h1Count !== 1 || !quality.codeFenceBalanced || quality.rawHtmlTags > 0 || quality.rawSvgTags > 0 || quality.rawScriptTags > 0 || quality.rawStyleTags > 0 || quality.navigationOnly || quality.repeatedLines > 0 || quality.contentSketchCoverage !== null && quality.contentSketchCoverage !== void 0 && quality.contentSketchCoverage < .6 || quality.tabbedContent?.complete === false || quality.wordRetentionRatio !== null && quality.wordRetentionRatio < .4;
|
|
7679
7771
|
}).map((page) => page.htmlUrl);
|
|
7680
7772
|
}
|
|
7681
|
-
function
|
|
7682
|
-
return
|
|
7683
|
-
"api",
|
|
7684
|
-
"application",
|
|
7685
|
-
"commerce"
|
|
7686
|
-
].map((profile) => check$1("profile", {
|
|
7687
|
-
id: `profile-${profile}`,
|
|
7688
|
-
status: "notApplicable",
|
|
7689
|
-
title: `${profile[0]?.toUpperCase()}${profile.slice(1)} checks are not applicable`,
|
|
7690
|
-
plainEnglish: discovery.profileApplicability[profile].reason,
|
|
7691
|
-
action: "Choose this profile only when the site exposes that capability publicly.",
|
|
7692
|
-
evidence: {
|
|
7693
|
-
profile,
|
|
7694
|
-
selectedProfile: discovery.profile
|
|
7695
|
-
}
|
|
7696
|
-
}));
|
|
7697
|
-
}
|
|
7698
|
-
function summariseChecks(checks) {
|
|
7699
|
-
const count = (status) => checks.filter((item) => item.status === status).length;
|
|
7700
|
-
return {
|
|
7701
|
-
checks: checks.length,
|
|
7702
|
-
passed: count("pass"),
|
|
7703
|
-
warnings: count("warning"),
|
|
7704
|
-
failed: count("fail"),
|
|
7705
|
-
unknown: count("unknown"),
|
|
7706
|
-
information: count("info"),
|
|
7707
|
-
notApplicable: count("notApplicable")
|
|
7708
|
-
};
|
|
7773
|
+
function confirmedMarkdown(page) {
|
|
7774
|
+
for (const response of [page.explicit, page.negotiated]) if (response?.status !== void 0 && response.status >= 200 && response.status < 300 && /^\s*text\/markdown\b/iu.test(response.contentType ?? "")) return response;
|
|
7709
7775
|
}
|
|
7710
7776
|
function markdownChecks(discovery) {
|
|
7711
7777
|
const markdown = discovery.markdownAlternates;
|
|
@@ -7715,12 +7781,28 @@ function markdownChecks(discovery) {
|
|
|
7715
7781
|
const explicitPages = markdown.pages.filter((page) => page.htmlAlternateUnique).length;
|
|
7716
7782
|
const coverageComplete = markdown.eligibleHtmlPages > 0 && markdown.evaluatedPages === markdown.eligibleHtmlPages;
|
|
7717
7783
|
const negotiationComplete = markdown.eligibleHtmlPages > 0 && negotiatedPages === markdown.eligibleHtmlPages && markdown.pages.every((page) => page.negotiated?.varyAccept) && markdown.pages.every((page) => !page.htmlAlternateUnique || page.explicitMatchesNegotiated === true);
|
|
7718
|
-
const stable = markdown.
|
|
7784
|
+
const stable = markdown.evaluatedPages > 0 && markdown.stableResponses === markdown.evaluatedPages;
|
|
7719
7785
|
const tokenHeaders = markdown.pages.filter((page) => {
|
|
7720
|
-
return (page
|
|
7786
|
+
return confirmedMarkdown(page)?.markdownTokens !== void 0 && (!page.explicit || !page.negotiated || page.explicit.markdownTokens === page.negotiated.markdownTokens);
|
|
7721
7787
|
}).length;
|
|
7788
|
+
const sizedPages = markdown.pages.filter((page) => {
|
|
7789
|
+
return confirmedMarkdown(page)?.characters !== void 0;
|
|
7790
|
+
});
|
|
7791
|
+
const largePages = sizedPages.filter((page) => {
|
|
7792
|
+
return (confirmedMarkdown(page)?.characters ?? 0) > 5e4;
|
|
7793
|
+
});
|
|
7794
|
+
const veryLargePages = sizedPages.filter((page) => {
|
|
7795
|
+
return (confirmedMarkdown(page)?.characters ?? 0) > 1e5;
|
|
7796
|
+
});
|
|
7797
|
+
const maxCharacters = Math.max(0, ...sizedPages.map((page) => confirmedMarkdown(page)?.characters ?? 0));
|
|
7798
|
+
const maxEstimatedTokens = Math.max(0, ...sizedPages.map((page) => confirmedMarkdown(page)?.estimatedTokens ?? 0));
|
|
7799
|
+
const parityPages = markdown.pages.filter((page) => page.quality?.contentSketchCoverage !== null && page.quality?.contentSketchCoverage !== void 0);
|
|
7800
|
+
const parityFailures = parityPages.filter((page) => (page.quality?.contentSketchCoverage ?? 0) < .6);
|
|
7801
|
+
const tabbedPages = markdown.pages.filter((page) => (page.quality?.tabbedContent?.detectedPanels ?? 0) > 0);
|
|
7802
|
+
const tabFailures = tabbedPages.filter((page) => page.quality?.tabbedContent?.complete === false);
|
|
7803
|
+
const tabUnknown = tabbedPages.filter((page) => page.quality?.tabbedContent?.complete === null);
|
|
7722
7804
|
return [
|
|
7723
|
-
check$
|
|
7805
|
+
check$2("representations", {
|
|
7724
7806
|
id: "markdown-coverage",
|
|
7725
7807
|
status: markdown.eligibleHtmlPages === 0 ? "unknown" : coverageComplete ? "pass" : "fail",
|
|
7726
7808
|
title: coverageComplete ? "Every successful HTML page has a Markdown representation" : "Markdown representation coverage is incomplete",
|
|
@@ -7734,17 +7816,62 @@ function markdownChecks(discovery) {
|
|
|
7734
7816
|
},
|
|
7735
7817
|
urls: failedUrls.slice(0, 25)
|
|
7736
7818
|
}),
|
|
7737
|
-
check$
|
|
7819
|
+
check$2("representations", {
|
|
7738
7820
|
id: "markdown-token-estimates",
|
|
7739
|
-
status: markdown.evaluatedPages === 0 ? "unknown" : tokenHeaders === markdown.evaluatedPages ? "pass" : "
|
|
7821
|
+
status: markdown.evaluatedPages === 0 ? "unknown" : tokenHeaders === markdown.evaluatedPages ? "pass" : "info",
|
|
7740
7822
|
title: tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes a token estimate" : "Some Markdown responses are missing a stable token estimate",
|
|
7741
7823
|
plainEnglish: `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned X-Markdown-Tokens. Paired explicit and negotiated responses agreed where both existed. The value is an estimate, not an exact model-specific token count.`,
|
|
7742
|
-
action:
|
|
7824
|
+
action: tokenHeaders === markdown.evaluatedPages ? "Keep the estimate stable when both explicit and negotiated responses exist." : "No change is required. X-Markdown-Tokens is optional; local size estimates are reported separately.",
|
|
7743
7825
|
urls: markdown.pages.filter((page) => {
|
|
7744
|
-
return (page
|
|
7826
|
+
return confirmedMarkdown(page)?.markdownTokens === void 0 || page.explicit !== void 0 && page.negotiated !== void 0 && page.explicit.markdownTokens !== page.negotiated.markdownTokens;
|
|
7745
7827
|
}).map((page) => page.htmlUrl).slice(0, 25)
|
|
7746
7828
|
}),
|
|
7747
|
-
check$
|
|
7829
|
+
check$2("representations", {
|
|
7830
|
+
id: "markdown-size",
|
|
7831
|
+
status: sizedPages.length === 0 ? "unknown" : largePages.length === 0 ? "pass" : "warning",
|
|
7832
|
+
title: sizedPages.length === 0 ? "Markdown size was not measured" : largePages.length === 0 ? "Markdown representations stay within the review threshold" : "Some Markdown representations are large enough to review",
|
|
7833
|
+
plainEnglish: `${sizedPages.length} Markdown responses were measured locally. ${largePages.length} exceeded 50,000 characters and ${veryLargePages.length} exceeded 100,000 characters. These are processing-risk thresholds, not universal model limits.`,
|
|
7834
|
+
action: largePages.length === 0 ? "Keep important content early and retain stable section structure as pages grow." : "Split genuinely separate topics, keep navigation and repeated chrome out of Markdown, and put the most useful content early. Do not remove necessary detail just to hit a threshold.",
|
|
7835
|
+
evidence: {
|
|
7836
|
+
measuredPages: sizedPages.length,
|
|
7837
|
+
reviewThresholdCharacters: 5e4,
|
|
7838
|
+
highRiskThresholdCharacters: 1e5,
|
|
7839
|
+
largePages: largePages.length,
|
|
7840
|
+
veryLargePages: veryLargePages.length,
|
|
7841
|
+
maxCharacters,
|
|
7842
|
+
maxEstimatedTokens,
|
|
7843
|
+
heuristic: true
|
|
7844
|
+
},
|
|
7845
|
+
urls: largePages.map((page) => page.htmlUrl).slice(0, 25)
|
|
7846
|
+
}),
|
|
7847
|
+
check$2("representations", {
|
|
7848
|
+
id: "markdown-content-parity",
|
|
7849
|
+
status: parityPages.length === 0 ? "unknown" : parityFailures.length === 0 ? "pass" : "warning",
|
|
7850
|
+
title: parityPages.length === 0 ? "HTML and Markdown content parity was not measured" : parityFailures.length === 0 ? "Markdown retains sampled content from the HTML document" : "Some Markdown responses may omit important HTML content",
|
|
7851
|
+
plainEnglish: `${parityPages.length} pages were compared using bounded hashes sampled across the main HTML content. ${parityFailures.length} retained less than 60% of that sample. This detects likely omissions without storing full page bodies.`,
|
|
7852
|
+
action: parityFailures.length === 0 ? "Keep the shared conversion path covered as templates and components change." : "Compare the affected HTML and Markdown around the missing sections, then fix the shared converter or component serialization.",
|
|
7853
|
+
evidence: {
|
|
7854
|
+
evaluatedPages: parityPages.length,
|
|
7855
|
+
failedPages: parityFailures.length,
|
|
7856
|
+
minimumCoverage: .6,
|
|
7857
|
+
sampling: "bounded-content-shingles"
|
|
7858
|
+
},
|
|
7859
|
+
urls: parityFailures.map((page) => page.htmlUrl).slice(0, 25)
|
|
7860
|
+
}),
|
|
7861
|
+
check$2("representations", {
|
|
7862
|
+
id: "markdown-tab-serialization",
|
|
7863
|
+
status: tabbedPages.length === 0 ? "notApplicable" : tabFailures.length > 0 ? "warning" : tabUnknown.length > 0 ? "unknown" : "pass",
|
|
7864
|
+
title: tabbedPages.length === 0 ? "No accessible tabbed content was detected" : tabFailures.length > 0 ? "Some tab panels are missing from Markdown" : tabUnknown.length > 0 ? "Some tab panels were too small to compare reliably" : "Markdown retains detected tab panels",
|
|
7865
|
+
plainEnglish: tabbedPages.length === 0 ? "The crawled HTML did not contain ARIA tablists with tab panels, so this check does not apply." : `${tabbedPages.length} pages contained accessible tab panels. ${tabFailures.length} did not retain every evaluated panel in Markdown, and ${tabUnknown.length} had no panel large enough for a reliable comparison.`,
|
|
7866
|
+
action: tabFailures.length === 0 ? tabUnknown.length > 0 ? "Review the affected page directly if the short tab labels or values are important to the document." : "Keep every panel serialized in document order even when only one tab is visible by default." : "Serialize all tab panels into the Markdown document with headings instead of exporting only the initially active panel.",
|
|
7867
|
+
evidence: {
|
|
7868
|
+
tabbedPages: tabbedPages.length,
|
|
7869
|
+
failedPages: tabFailures.length,
|
|
7870
|
+
unknownPages: tabUnknown.length
|
|
7871
|
+
},
|
|
7872
|
+
urls: [...tabFailures, ...tabUnknown].map((page) => page.htmlUrl).slice(0, 25)
|
|
7873
|
+
}),
|
|
7874
|
+
check$2("representations", {
|
|
7748
7875
|
id: "markdown-negotiation",
|
|
7749
7876
|
status: markdown.eligibleHtmlPages === 0 ? "unknown" : negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "pass" : "fail",
|
|
7750
7877
|
title: negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "Markdown content negotiation works across the site" : "Markdown content negotiation needs attention",
|
|
@@ -7758,16 +7885,16 @@ function markdownChecks(discovery) {
|
|
|
7758
7885
|
},
|
|
7759
7886
|
urls: failedUrls.slice(0, 25)
|
|
7760
7887
|
}),
|
|
7761
|
-
check$
|
|
7888
|
+
check$2("representations", {
|
|
7762
7889
|
id: "markdown-determinism",
|
|
7763
|
-
status: markdown.
|
|
7890
|
+
status: markdown.evaluatedPages === 0 ? "unknown" : stable ? "pass" : "fail",
|
|
7764
7891
|
title: stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
|
|
7765
|
-
plainEnglish: `${markdown.stableResponses} of ${markdown.
|
|
7892
|
+
plainEnglish: `${markdown.stableResponses} of ${markdown.evaluatedPages} repeated Markdown responses kept the same SHA-256 digest.`,
|
|
7766
7893
|
action: "Remove timestamps, random output, or other runtime rewriting that changes the same Markdown response between requests.",
|
|
7767
7894
|
evidence: { stableResponses: markdown.stableResponses },
|
|
7768
7895
|
urls: failedUrls.slice(0, 25)
|
|
7769
7896
|
}),
|
|
7770
|
-
check$
|
|
7897
|
+
check$2("representations", {
|
|
7771
7898
|
id: "markdown-quality",
|
|
7772
7899
|
status: markdown.evaluatedPages === 0 ? "unknown" : qualityUrls.length === 0 ? "pass" : "warning",
|
|
7773
7900
|
title: qualityUrls.length === 0 ? "Markdown keeps the useful document structure cleanly" : "Some Markdown alternatives need an extraction review",
|
|
@@ -7777,6 +7904,50 @@ function markdownChecks(discovery) {
|
|
|
7777
7904
|
})
|
|
7778
7905
|
];
|
|
7779
7906
|
}
|
|
7907
|
+
//#endregion
|
|
7908
|
+
//#region packages/core/dist/analyze/crawler/agent-readiness.js
|
|
7909
|
+
function check$1(section, input) {
|
|
7910
|
+
return {
|
|
7911
|
+
section,
|
|
7912
|
+
...input
|
|
7913
|
+
};
|
|
7914
|
+
}
|
|
7915
|
+
function section$2(id, title, checks) {
|
|
7916
|
+
return {
|
|
7917
|
+
id,
|
|
7918
|
+
title,
|
|
7919
|
+
checks
|
|
7920
|
+
};
|
|
7921
|
+
}
|
|
7922
|
+
function profileChecks(discovery) {
|
|
7923
|
+
return [
|
|
7924
|
+
"api",
|
|
7925
|
+
"application",
|
|
7926
|
+
"commerce"
|
|
7927
|
+
].map((profile) => check$1("profile", {
|
|
7928
|
+
id: `profile-${profile}`,
|
|
7929
|
+
status: "notApplicable",
|
|
7930
|
+
title: `${profile[0]?.toUpperCase()}${profile.slice(1)} checks are not applicable`,
|
|
7931
|
+
plainEnglish: discovery.profileApplicability[profile].reason,
|
|
7932
|
+
action: "Choose this profile only when the site exposes that capability publicly.",
|
|
7933
|
+
evidence: {
|
|
7934
|
+
profile,
|
|
7935
|
+
selectedProfile: discovery.profile
|
|
7936
|
+
}
|
|
7937
|
+
}));
|
|
7938
|
+
}
|
|
7939
|
+
function summariseChecks(checks) {
|
|
7940
|
+
const count = (status) => checks.filter((item) => item.status === status).length;
|
|
7941
|
+
return {
|
|
7942
|
+
checks: checks.length,
|
|
7943
|
+
passed: count("pass"),
|
|
7944
|
+
warnings: count("warning"),
|
|
7945
|
+
failed: count("fail"),
|
|
7946
|
+
unknown: count("unknown"),
|
|
7947
|
+
information: count("info"),
|
|
7948
|
+
notApplicable: count("notApplicable")
|
|
7949
|
+
};
|
|
7950
|
+
}
|
|
7780
7951
|
function contentSignalsCheck(discovery, robotsContentSignals) {
|
|
7781
7952
|
const signals = discovery.contentSignals;
|
|
7782
7953
|
const robotsDeclared = robotsContentSignals ?? [];
|
|
@@ -7877,8 +8048,8 @@ function discoveryChecks(discovery, robotsContentSignals) {
|
|
|
7877
8048
|
check$1("discovery", {
|
|
7878
8049
|
id: "route-manifest",
|
|
7879
8050
|
status: !discovery.routeManifest.valid ? "info" : discovery.routeManifest.missingHtmlRoutes.length === 0 && discovery.routeManifest.missingMarkdownRoutes.length === 0 && discovery.routeManifest.orphanMarkdownRoutes.length === 0 ? "pass" : "warning",
|
|
7880
|
-
title: discovery.routeManifest.valid ? "The public route manifest
|
|
7881
|
-
plainEnglish: discovery.routeManifest.valid ? `The manifest declared ${discovery.routeManifest.declaredHtmlRoutes.length} HTML routes and ${discovery.routeManifest.declaredMarkdownRoutes.length} Markdown routes.
|
|
8051
|
+
title: discovery.routeManifest.valid ? "The public route manifest is valid" : "No public route manifest was available",
|
|
8052
|
+
plainEnglish: discovery.routeManifest.valid ? `The manifest declared ${discovery.routeManifest.declaredHtmlRoutes.length} HTML routes and ${discovery.routeManifest.declaredMarkdownRoutes.length} Markdown routes. This bounded crawl did not treat unvisited routes as missing.` : "A public route manifest is optional. Without one, this crawl cannot prove that no orphan Markdown files were deployed.",
|
|
7882
8053
|
action: "Generate one deterministic route inventory from the build when exact HTML and Markdown parity matters.",
|
|
7883
8054
|
evidence: discovery.routeManifest
|
|
7884
8055
|
})
|
|
@@ -8009,25 +8180,42 @@ function accessChecks(report, discovery) {
|
|
|
8009
8180
|
const blocked = bots.filter((bot) => bot.allowed === false);
|
|
8010
8181
|
const robotsKnown = report.ai?.robotsTxt?.availability === "available" || report.ai?.robotsTxt?.availability === "absent";
|
|
8011
8182
|
const protocol = discovery.protocolVariants;
|
|
8012
|
-
|
|
8013
|
-
|
|
8014
|
-
|
|
8015
|
-
|
|
8016
|
-
|
|
8017
|
-
|
|
8018
|
-
|
|
8019
|
-
|
|
8020
|
-
|
|
8021
|
-
|
|
8022
|
-
|
|
8023
|
-
|
|
8024
|
-
|
|
8025
|
-
|
|
8026
|
-
|
|
8027
|
-
|
|
8028
|
-
|
|
8029
|
-
|
|
8030
|
-
|
|
8183
|
+
const authenticationGates = report.issues.filter((issue) => issue.ruleId === "soft_authentication_gate");
|
|
8184
|
+
const clientRenderedPages = report.issues.filter((issue) => issue.ruleId === "client_rendered_content");
|
|
8185
|
+
return [
|
|
8186
|
+
check$1("access", {
|
|
8187
|
+
id: "crawler-access",
|
|
8188
|
+
status: !robotsKnown ? "unknown" : blocked.length > 0 ? "warning" : "pass",
|
|
8189
|
+
title: blocked.length ? "robots.txt blocks selected crawler tokens" : "No selected crawler token is blocked at the start URL",
|
|
8190
|
+
plainEnglish: robotsKnown ? `${blocked.length} of ${bots.length} selected crawler tokens were blocked by the start URL robots.txt policy.` : "The robots.txt response was not stable enough to make a crawler-access claim.",
|
|
8191
|
+
action: "Keep blocks intentional. A crawler token being allowed here does not prove that the service will crawl, index, or cite the site.",
|
|
8192
|
+
evidence: {
|
|
8193
|
+
availability: report.ai?.robotsTxt?.availability,
|
|
8194
|
+
blocked,
|
|
8195
|
+
botAccess: bots
|
|
8196
|
+
}
|
|
8197
|
+
}),
|
|
8198
|
+
check$1("access", {
|
|
8199
|
+
id: "public-content-access",
|
|
8200
|
+
status: authenticationGates.length > 0 || clientRenderedPages.length > 0 ? "warning" : "pass",
|
|
8201
|
+
title: authenticationGates.length > 0 || clientRenderedPages.length > 0 ? "Some public content may be difficult for constrained clients to reach" : "No login gate or empty initial document was detected",
|
|
8202
|
+
plainEnglish: `${authenticationGates.length} crawled pages looked like login gates and ${clientRenderedPages.length} relied on JavaScript to replace an empty or near-empty initial document. These are general site-health observations reused here because they affect agents that do not maintain sessions or run JavaScript.`,
|
|
8203
|
+
action: authenticationGates.length > 0 || clientRenderedPages.length > 0 ? "Make intended public content available without a session and include its important document content in the initial HTML or a complete server-generated representation." : "Keep important public content available without a session or client-side rendering requirement.",
|
|
8204
|
+
evidence: {
|
|
8205
|
+
authenticationGates: authenticationGates.length,
|
|
8206
|
+
clientRenderedPages: clientRenderedPages.length
|
|
8207
|
+
},
|
|
8208
|
+
urls: [...authenticationGates, ...clientRenderedPages].map((issue) => issue.url).slice(0, 25)
|
|
8209
|
+
}),
|
|
8210
|
+
check$1("access", {
|
|
8211
|
+
id: "protocol-canonicalization",
|
|
8212
|
+
status: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "pass" : protocol.http.permanentRedirectToHttps === null ? "unknown" : "warning",
|
|
8213
|
+
title: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "HTTP redirects permanently to HTTPS and HTTPS sends HSTS" : "Protocol canonicalization needs attention or could not be proven",
|
|
8214
|
+
plainEnglish: `The HTTP variant ${protocol.http.permanentRedirectToHttps === true ? "redirected permanently to HTTPS" : "did not produce a confirmed permanent HTTPS redirect"}. The HTTPS start page ${protocol.hstsOnStartPage ? "sent HSTS" : "did not provide confirmed HSTS evidence"}.`,
|
|
8215
|
+
action: "Use one permanent HTTP-to-HTTPS redirect and send HSTS only after HTTPS is stable across the site.",
|
|
8216
|
+
evidence: protocol
|
|
8217
|
+
})
|
|
8218
|
+
];
|
|
8031
8219
|
}
|
|
8032
8220
|
function identityChecks(report) {
|
|
8033
8221
|
const types = new Set(report.pages.flatMap((page) => page.schemaTypes ?? []));
|
|
@@ -8095,19 +8283,19 @@ function agentReadiness(report) {
|
|
|
8095
8283
|
headline: "No agent-discovery evidence is stored in this crawl. Run a fresh focused check.",
|
|
8096
8284
|
profileApplicability: profiles,
|
|
8097
8285
|
summary: summariseChecks(checks),
|
|
8098
|
-
sections: [section$
|
|
8286
|
+
sections: [section$2("collection", "Evidence collection", [missing]), section$2("profile", "Profile scope", scopeChecks)],
|
|
8099
8287
|
checks,
|
|
8100
8288
|
topActions: [missing],
|
|
8101
8289
|
caveats: ["Not evaluated does not mean absent or broken.", "This report does not measure rankings, AI mentions, citations, or selection."]
|
|
8102
8290
|
};
|
|
8103
8291
|
}
|
|
8104
8292
|
const sections = [
|
|
8105
|
-
section$
|
|
8106
|
-
section$
|
|
8107
|
-
section$
|
|
8108
|
-
section$
|
|
8109
|
-
section$
|
|
8110
|
-
section$
|
|
8293
|
+
section$2("profile", "Profile scope", profileChecks(discovery)),
|
|
8294
|
+
section$2("access", "Crawler and protocol access", accessChecks(report, discovery)),
|
|
8295
|
+
section$2("representations", "HTML and Markdown representations", markdownChecks(discovery)),
|
|
8296
|
+
section$2("discovery", "Agent discovery files", discoveryChecks(discovery, report.ai?.robotsTxt?.contentSignals)),
|
|
8297
|
+
section$2("endpoints", "Agent endpoints and auth discovery", endpointChecks(discovery)),
|
|
8298
|
+
section$2("identity", "Identity evidence", identityChecks(report))
|
|
8111
8299
|
];
|
|
8112
8300
|
const checks = sections.flatMap((item) => item.checks);
|
|
8113
8301
|
const actionable = checks.filter((item) => [
|
|
@@ -8294,7 +8482,7 @@ function status(score, maxScore) {
|
|
|
8294
8482
|
if (ratio >= .5) return "warning";
|
|
8295
8483
|
return "fail";
|
|
8296
8484
|
}
|
|
8297
|
-
function section(id, title, checks) {
|
|
8485
|
+
function section$1(id, title, checks) {
|
|
8298
8486
|
return {
|
|
8299
8487
|
id,
|
|
8300
8488
|
title,
|
|
@@ -8352,7 +8540,7 @@ function aiReadiness(report) {
|
|
|
8352
8540
|
const blockedSnippets = restrictedSnippets.filter(({ control }) => control.status === "blocked");
|
|
8353
8541
|
const limitedSnippets = restrictedSnippets.filter(({ control }) => control.status === "limited");
|
|
8354
8542
|
const sections = [
|
|
8355
|
-
section("agent-access", "Agent access", [
|
|
8543
|
+
section$1("agent-access", "Agent access", [
|
|
8356
8544
|
check({
|
|
8357
8545
|
id: "robots-ai-bots",
|
|
8358
8546
|
section: "agent-access",
|
|
@@ -8395,7 +8583,7 @@ function aiReadiness(report) {
|
|
|
8395
8583
|
evidence: { resources: report.ai?.agentResources ?? [] }
|
|
8396
8584
|
})
|
|
8397
8585
|
]),
|
|
8398
|
-
section("machine-readable", "Machine-readable signals", [check({
|
|
8586
|
+
section$1("machine-readable", "Machine-readable signals", [check({
|
|
8399
8587
|
id: "llms-txt",
|
|
8400
8588
|
section: "machine-readable",
|
|
8401
8589
|
maxScore: 0,
|
|
@@ -8420,7 +8608,7 @@ function aiReadiness(report) {
|
|
|
8420
8608
|
action: invalidJsonLdPages.length ? "Fix invalid JSON-LD before relying on its structured-data claims." : "Use structured data only where it accurately represents the visible page and a supported use case.",
|
|
8421
8609
|
urls: invalidJsonLdPages.slice(0, 10).map((page) => page.finalUrl)
|
|
8422
8610
|
})]),
|
|
8423
|
-
section("content-clarity", "Content clarity", [
|
|
8611
|
+
section$1("content-clarity", "Content clarity", [
|
|
8424
8612
|
check({
|
|
8425
8613
|
id: "answerable-content",
|
|
8426
8614
|
section: "content-clarity",
|
|
@@ -8470,7 +8658,7 @@ function aiReadiness(report) {
|
|
|
8470
8658
|
action: "Set the html lang attribute so crawlers and assistive tech understand the page language."
|
|
8471
8659
|
})
|
|
8472
8660
|
]),
|
|
8473
|
-
section("entity-signals", "Entity signals", [check({
|
|
8661
|
+
section$1("entity-signals", "Entity signals", [check({
|
|
8474
8662
|
id: "entity-schema",
|
|
8475
8663
|
section: "entity-signals",
|
|
8476
8664
|
maxScore: 0,
|
|
@@ -8489,7 +8677,7 @@ function aiReadiness(report) {
|
|
|
8489
8677
|
plainEnglish: "Author and date signals help agents judge freshness and accountability.",
|
|
8490
8678
|
action: "Add visible authors, reviewed dates, and updated dates to content where trust and freshness matter."
|
|
8491
8679
|
})]),
|
|
8492
|
-
section("technical-ux", "Technical UX", [
|
|
8680
|
+
section$1("technical-ux", "Technical UX", [
|
|
8493
8681
|
check({
|
|
8494
8682
|
id: "snippet-controls",
|
|
8495
8683
|
section: "technical-ux",
|
|
@@ -8527,7 +8715,7 @@ function aiReadiness(report) {
|
|
|
8527
8715
|
action: "Fix 4xx/5xx pages, accidental noindex, and blocked internal URLs before expanding AI discovery work."
|
|
8528
8716
|
})
|
|
8529
8717
|
]),
|
|
8530
|
-
section("agent-content", "Agent content observations", [
|
|
8718
|
+
section$1("agent-content", "Agent content observations", [
|
|
8531
8719
|
check({
|
|
8532
8720
|
id: "markdown-alternates",
|
|
8533
8721
|
section: "agent-content",
|
|
@@ -8567,7 +8755,7 @@ function aiReadiness(report) {
|
|
|
8567
8755
|
evidence: agentDiscovery?.profileApplicability
|
|
8568
8756
|
})
|
|
8569
8757
|
]),
|
|
8570
|
-
section("crawl-completeness", "Crawl completeness", [check({
|
|
8758
|
+
section$1("crawl-completeness", "Crawl completeness", [check({
|
|
8571
8759
|
id: "crawl-depth",
|
|
8572
8760
|
section: "crawl-completeness",
|
|
8573
8761
|
maxScore: 8,
|
|
@@ -9556,6 +9744,48 @@ const RULE_DEFINITIONS = [
|
|
|
9556
9744
|
suggestedCommands: ["seo crawl --sitemap-url <url> --health --json"]
|
|
9557
9745
|
}
|
|
9558
9746
|
},
|
|
9747
|
+
{
|
|
9748
|
+
id: "soft_authentication_gate",
|
|
9749
|
+
title: "Page appears to be an authentication gate",
|
|
9750
|
+
category: "response",
|
|
9751
|
+
defaultSeverity: "medium",
|
|
9752
|
+
whyItMatters: "The URL returned a successful HTML response, but the visible document looks like a login gate rather than the intended public page. Users and crawlers may receive a 200 response without the content they expected.",
|
|
9753
|
+
howToFix: "Confirm whether authentication is intentional. Public URLs should return their public content without a login form. Protected URLs should use a clear 401 or 403 response where appropriate and should not appear in public sitemaps or internal crawl paths.",
|
|
9754
|
+
impactIfIgnored: "A login page can be mistaken for the intended content, producing misleading crawl results and weak search or agent access.",
|
|
9755
|
+
howToVerify: "Fetch the affected URL without a session and confirm it returns the intended public page, or a deliberate authentication status for protected content.",
|
|
9756
|
+
agentHints: {
|
|
9757
|
+
evidenceFields: ["page.softAuthenticationGate", "page.finalUrl"],
|
|
9758
|
+
suggestedCommands: ["seo crawl <url> --max-pages 1 --json"]
|
|
9759
|
+
}
|
|
9760
|
+
},
|
|
9761
|
+
{
|
|
9762
|
+
id: "client_rendered_content",
|
|
9763
|
+
title: "Important content requires JavaScript rendering",
|
|
9764
|
+
category: "response",
|
|
9765
|
+
defaultSeverity: "medium",
|
|
9766
|
+
whyItMatters: "The raw HTML contained very little content and the useful document appeared only after JavaScript rendering. Some crawlers, agents, previews, and constrained clients do not run the page JavaScript.",
|
|
9767
|
+
howToFix: "Render the important page content into the initial HTML, or publish a complete server-generated representation such as negotiated Markdown. Keep navigation, headings, canonicals, and indexability signals available without client execution.",
|
|
9768
|
+
impactIfIgnored: "Clients that do not execute JavaScript may see an empty shell or miss important content and links.",
|
|
9769
|
+
howToVerify: "Re-run the crawl and confirm the raw and rendered content observations contain the same important document content.",
|
|
9770
|
+
agentHints: {
|
|
9771
|
+
evidenceFields: ["page.fetchDiagnostics.rendering.documentDifference", "page.usedJs"],
|
|
9772
|
+
suggestedCommands: ["seo crawl <url> --json"]
|
|
9773
|
+
}
|
|
9774
|
+
},
|
|
9775
|
+
{
|
|
9776
|
+
id: "soft_404",
|
|
9777
|
+
title: "Unknown URLs return a successful response",
|
|
9778
|
+
category: "response",
|
|
9779
|
+
defaultSeverity: "high",
|
|
9780
|
+
whyItMatters: "Known-nonexistent URLs returned a successful response or redirected to another successful page. Search engines and users cannot reliably distinguish missing content from a real page.",
|
|
9781
|
+
howToFix: "Return 404 or 410 for missing URLs. If a replacement exists, redirect only the specific retired URL to its closest relevant replacement rather than sending every unknown path to a generic page.",
|
|
9782
|
+
impactIfIgnored: "Missing URLs can remain in crawl queues, waste crawl attention, and create confusing search or monitoring evidence.",
|
|
9783
|
+
howToVerify: "Request several known-nonexistent paths and confirm each returns 404 or 410 without redirecting to a generic successful page.",
|
|
9784
|
+
agentHints: {
|
|
9785
|
+
evidenceFields: ["report.siteChecks.soft404"],
|
|
9786
|
+
suggestedCommands: ["seo crawl <url> --json"]
|
|
9787
|
+
}
|
|
9788
|
+
},
|
|
9559
9789
|
{
|
|
9560
9790
|
id: "client_error",
|
|
9561
9791
|
title: "Client error",
|
|
@@ -10045,6 +10275,8 @@ const RULE_RECOMMENDATIONS = {
|
|
|
10045
10275
|
x_robots_noindex: "review",
|
|
10046
10276
|
robots_blocked: "review",
|
|
10047
10277
|
crawler_access_blocked: "review",
|
|
10278
|
+
soft_authentication_gate: "review",
|
|
10279
|
+
client_rendered_content: "review",
|
|
10048
10280
|
orphan_page: "review",
|
|
10049
10281
|
redirected_url: "review",
|
|
10050
10282
|
slow_response: "review",
|
|
@@ -10135,6 +10367,28 @@ const WEAK_VALUABLE_INLINKS = 1;
|
|
|
10135
10367
|
const QUERY_COVERAGE_MIN_IMPRESSIONS = 50;
|
|
10136
10368
|
const QUERY_COVERAGE_MIN = .6;
|
|
10137
10369
|
const OVERSIZED_IMAGE_CANDIDATE_LIMIT = 2e3;
|
|
10370
|
+
const EMPTY_INITIAL_HTML_CHARACTERS = 200;
|
|
10371
|
+
const SUBSTANTIVE_RENDERED_CHARACTERS = 500;
|
|
10372
|
+
function clientRenderedContentEvidence(page) {
|
|
10373
|
+
const rendering = page.fetchDiagnostics?.rendering;
|
|
10374
|
+
const difference = rendering?.documentDifference;
|
|
10375
|
+
if (rendering?.status !== "rendered" || !difference?.changed.includes("content")) return;
|
|
10376
|
+
const rawCharacters = difference.raw.content.characters;
|
|
10377
|
+
const renderedCharacters = difference.rendered.content.characters;
|
|
10378
|
+
if (rawCharacters >= EMPTY_INITIAL_HTML_CHARACTERS || renderedCharacters < SUBSTANTIVE_RENDERED_CHARACTERS || renderedCharacters < rawCharacters * 3) return;
|
|
10379
|
+
return {
|
|
10380
|
+
rawCharacters,
|
|
10381
|
+
rawWords: difference.raw.content.wordCount,
|
|
10382
|
+
renderedCharacters,
|
|
10383
|
+
renderedWords: difference.rendered.content.wordCount,
|
|
10384
|
+
changed: difference.changed,
|
|
10385
|
+
thresholds: {
|
|
10386
|
+
emptyInitialHtmlCharacters: EMPTY_INITIAL_HTML_CHARACTERS,
|
|
10387
|
+
substantiveRenderedCharacters: SUBSTANTIVE_RENDERED_CHARACTERS,
|
|
10388
|
+
minimumGrowthRatio: 3
|
|
10389
|
+
}
|
|
10390
|
+
};
|
|
10391
|
+
}
|
|
10138
10392
|
function issue(ruleId, page, detail, evidence) {
|
|
10139
10393
|
const rule = explainRule(ruleId);
|
|
10140
10394
|
if (!rule) throw new Error(`Missing rule guidance for ${ruleId}`);
|
|
@@ -10349,6 +10603,9 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
10349
10603
|
}));
|
|
10350
10604
|
if (page.auditScope === "status") continue;
|
|
10351
10605
|
if (redirected) continue;
|
|
10606
|
+
if (page.softAuthenticationGate) issues.push(issue("soft_authentication_gate", page, "A password form and login-specific page signals were observed.", page.softAuthenticationGate));
|
|
10607
|
+
const renderedContent = clientRenderedContentEvidence(page);
|
|
10608
|
+
if (renderedContent) issues.push(issue("client_rendered_content", page, "The raw response was an empty or near-empty shell.", renderedContent));
|
|
10352
10609
|
if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(issue("large_html", page, `${Math.round((page.sizeBytes ?? 0) / 1024)} KB`, {
|
|
10353
10610
|
sizeBytes: page.sizeBytes,
|
|
10354
10611
|
thresholdBytes: LARGE_HTML_BYTES
|
|
@@ -10953,7 +11210,7 @@ function createCrawlReport(input) {
|
|
|
10953
11210
|
const requests = sortRequests(sanitizeTenantValue(input.requests ?? []));
|
|
10954
11211
|
const requestEvidenceStatus = input.requestEvidenceStatus ?? (input.requests ? "available" : "unavailable");
|
|
10955
11212
|
if (requestEvidenceStatus === "unavailable" && requests.length > 0) throw new Error("Unavailable request evidence cannot include request observations.");
|
|
10956
|
-
const safeIssues = sanitizeIssues(input.issues ?? [...auditCrawlRequests(requests), ...auditCrawlPages(safePagesWithLinks, { startUrl: config.url })]);
|
|
11213
|
+
const safeIssues = sanitizeIssues([...input.issues ?? [...auditCrawlRequests(requests), ...auditCrawlPages(safePagesWithLinks, { startUrl: config.url })], ...input.additionalIssues ?? []]);
|
|
10957
11214
|
const pages = safePagesWithLinks;
|
|
10958
11215
|
return {
|
|
10959
11216
|
id: input.id ?? crawlRunId(),
|
|
@@ -10982,6 +11239,7 @@ function createCrawlReport(input) {
|
|
|
10982
11239
|
...input.ai ? { ai: sanitizeTenantValue(input.ai) } : {},
|
|
10983
11240
|
...input.sitemapDiscovery ? { sitemapDiscovery: sanitizeTenantValue(input.sitemapDiscovery) } : {},
|
|
10984
11241
|
...input.externalLinkVerification ? { externalLinkVerification: sanitizeTenantValue(input.externalLinkVerification) } : {},
|
|
11242
|
+
...input.siteChecks ? { siteChecks: sanitizeTenantValue(input.siteChecks) } : {},
|
|
10985
11243
|
...input.agentDiscovery ? { agentDiscovery: sanitizeTenantValue(input.agentDiscovery) } : {},
|
|
10986
11244
|
warnings: sanitizeMessages(input.warnings).sort(compareText$13),
|
|
10987
11245
|
caveats: sanitizeMessages(input.caveats).sort(compareText$13)
|
|
@@ -11542,6 +11800,75 @@ function observationFromPage(requestedUrl, page) {
|
|
|
11542
11800
|
};
|
|
11543
11801
|
}
|
|
11544
11802
|
//#endregion
|
|
11803
|
+
//#region packages/core/dist/analyze/crawler/site-checks.js
|
|
11804
|
+
const PROBE_COUNT = 2;
|
|
11805
|
+
function probeUrls(startUrl) {
|
|
11806
|
+
const origin = new URL(startUrl).origin;
|
|
11807
|
+
const seed = createHash("sha256").update(origin).digest("hex").slice(0, 12);
|
|
11808
|
+
return Array.from({ length: PROBE_COUNT }, (_, index) => new URL(`/.well-known/seo-audit/not-found-${seed}-${index + 1}`, origin).toString());
|
|
11809
|
+
}
|
|
11810
|
+
function probeObservation(url, result) {
|
|
11811
|
+
const request = result.request;
|
|
11812
|
+
if (!request) return {
|
|
11813
|
+
url,
|
|
11814
|
+
error: result.warning ?? "No response evidence."
|
|
11815
|
+
};
|
|
11816
|
+
if (request.outcome === "response") return {
|
|
11817
|
+
url,
|
|
11818
|
+
status: request.status,
|
|
11819
|
+
finalUrl: request.finalUrl,
|
|
11820
|
+
redirected: request.finalUrl !== url,
|
|
11821
|
+
accessBlocked: Boolean(request.accessBlock)
|
|
11822
|
+
};
|
|
11823
|
+
return {
|
|
11824
|
+
url,
|
|
11825
|
+
error: request.outcome === "failure" ? request.error : `Probe skipped: ${request.reason}`
|
|
11826
|
+
};
|
|
11827
|
+
}
|
|
11828
|
+
async function collectCrawlSiteChecks(input) {
|
|
11829
|
+
const probes = [];
|
|
11830
|
+
for (const url of probeUrls(input.startUrl)) {
|
|
11831
|
+
if (input.signal?.aborted) break;
|
|
11832
|
+
const result = await input.fetchStatusPage(url, {
|
|
11833
|
+
js: "off",
|
|
11834
|
+
refresh: true,
|
|
11835
|
+
timeoutMs: input.timeoutMs,
|
|
11836
|
+
signal: input.signal,
|
|
11837
|
+
writeCache: false,
|
|
11838
|
+
respectRobots: input.respectRobots
|
|
11839
|
+
});
|
|
11840
|
+
probes.push(probeObservation(url, result));
|
|
11841
|
+
}
|
|
11842
|
+
const conclusive = probes.filter((probe) => probe.status !== void 0 && !probe.accessBlocked);
|
|
11843
|
+
const successful = conclusive.filter((probe) => (probe.status ?? 0) >= 200 && (probe.status ?? 0) < 400);
|
|
11844
|
+
const correctMissing = conclusive.filter((probe) => probe.status === 404 || probe.status === 410);
|
|
11845
|
+
const status = conclusive.length !== PROBE_COUNT ? "unknown" : successful.length === PROBE_COUNT ? "warning" : correctMissing.length === PROBE_COUNT ? "pass" : "unknown";
|
|
11846
|
+
const checks = { soft404: {
|
|
11847
|
+
status,
|
|
11848
|
+
probes,
|
|
11849
|
+
probeLimit: PROBE_COUNT,
|
|
11850
|
+
complete: probes.length === PROBE_COUNT
|
|
11851
|
+
} };
|
|
11852
|
+
if (status !== "warning") return {
|
|
11853
|
+
checks,
|
|
11854
|
+
issues: []
|
|
11855
|
+
};
|
|
11856
|
+
const rule = explainRule("soft_404");
|
|
11857
|
+
if (!rule) throw new Error("Missing rule guidance for soft_404");
|
|
11858
|
+
return {
|
|
11859
|
+
checks,
|
|
11860
|
+
issues: [{
|
|
11861
|
+
ruleId: "soft_404",
|
|
11862
|
+
title: rule.title,
|
|
11863
|
+
category: rule.category,
|
|
11864
|
+
severity: rule.defaultSeverity,
|
|
11865
|
+
url: new URL("/", input.startUrl).toString(),
|
|
11866
|
+
detail: `${successful.length} known-nonexistent paths returned a successful response or redirect.`,
|
|
11867
|
+
evidence: checks.soft404
|
|
11868
|
+
}]
|
|
11869
|
+
};
|
|
11870
|
+
}
|
|
11871
|
+
//#endregion
|
|
11545
11872
|
//#region packages/core/dist/analyze/crawler/site-crawl-ai-signals.js
|
|
11546
11873
|
const MAX_AUXILIARY_RESPONSE_BYTES = 2 * 1024 * 1024;
|
|
11547
11874
|
const AI_BOT_USER_AGENTS = [
|
|
@@ -12261,6 +12588,9 @@ async function crawlOne(url, opts = {}) {
|
|
|
12261
12588
|
mainContentHash: hashText(extracted.contentText),
|
|
12262
12589
|
textRatio: fetched.html.length ? Math.min(1, extracted.contentText.length / fetched.html.length) : void 0,
|
|
12263
12590
|
contentSample: truncate$1(extracted.contentText, 300),
|
|
12591
|
+
contentSketch: crawlerEvidence.contentSketch,
|
|
12592
|
+
softAuthenticationGate: crawlerEvidence.softAuthenticationGate,
|
|
12593
|
+
tabbedContent: crawlerEvidence.tabbedContent,
|
|
12264
12594
|
lang: extracted.lang,
|
|
12265
12595
|
hasViewport: extracted.hasViewport,
|
|
12266
12596
|
isHttps: new URL(extracted.finalUrl).protocol === "https:",
|
|
@@ -13545,6 +13875,13 @@ async function crawlSite(input, dependencies) {
|
|
|
13545
13875
|
signal,
|
|
13546
13876
|
concurrency: Math.min(config.concurrency, 4)
|
|
13547
13877
|
}) : void 0;
|
|
13878
|
+
const siteCheckResult = !healthStrategy && !cancelled && !stoppedForMemory ? await collectCrawlSiteChecks({
|
|
13879
|
+
startUrl: config.url,
|
|
13880
|
+
timeoutMs: config.timeoutMs,
|
|
13881
|
+
fetchStatusPage: deps.fetchStatusPage,
|
|
13882
|
+
respectRobots: config.respectRobots,
|
|
13883
|
+
signal
|
|
13884
|
+
}) : void 0;
|
|
13548
13885
|
for (const page of pages) {
|
|
13549
13886
|
if (!page.geo || !llmsTxt) continue;
|
|
13550
13887
|
page.geo = {
|
|
@@ -13570,6 +13907,10 @@ async function crawlSite(input, dependencies) {
|
|
|
13570
13907
|
} } : {},
|
|
13571
13908
|
...sitemapDiscovery ? { sitemapDiscovery } : {},
|
|
13572
13909
|
...externalLinkVerification ? { externalLinkVerification } : {},
|
|
13910
|
+
...siteCheckResult ? {
|
|
13911
|
+
siteChecks: siteCheckResult.checks,
|
|
13912
|
+
additionalIssues: siteCheckResult.issues
|
|
13913
|
+
} : {},
|
|
13573
13914
|
...agentDiscovery ? { agentDiscovery } : {},
|
|
13574
13915
|
status: reportStatus,
|
|
13575
13916
|
warnings,
|
|
@@ -14450,6 +14791,8 @@ const agentRepresentationResponseSchema = z.object({
|
|
|
14450
14791
|
status: z.number().int().optional(),
|
|
14451
14792
|
contentType: z.string().optional(),
|
|
14452
14793
|
bytes: z.number().int().nonnegative().optional(),
|
|
14794
|
+
characters: z.number().int().nonnegative().optional(),
|
|
14795
|
+
estimatedTokens: z.number().int().nonnegative().optional(),
|
|
14453
14796
|
sha256: z.string().optional(),
|
|
14454
14797
|
canonicalUrl: z.string().url().optional(),
|
|
14455
14798
|
alternateUrl: z.string().url().optional(),
|
|
@@ -14507,7 +14850,15 @@ const agentDiscoverySchema = z.object({
|
|
|
14507
14850
|
sourceWordCount: z.number().int().nonnegative(),
|
|
14508
14851
|
wordRetentionRatio: z.number().nonnegative().nullable(),
|
|
14509
14852
|
introductoryCopyRetained: z.boolean().nullable(),
|
|
14510
|
-
navigationOnly: z.boolean()
|
|
14853
|
+
navigationOnly: z.boolean(),
|
|
14854
|
+
contentSketchCoverage: z.number().min(0).max(1).nullable().optional(),
|
|
14855
|
+
tabbedContent: z.object({
|
|
14856
|
+
detectedPanels: z.number().int().nonnegative(),
|
|
14857
|
+
evaluatedPanels: z.number().int().nonnegative(),
|
|
14858
|
+
retainedPanels: z.number().int().nonnegative(),
|
|
14859
|
+
missingPanels: z.number().int().nonnegative(),
|
|
14860
|
+
complete: z.boolean().nullable()
|
|
14861
|
+
}).optional()
|
|
14511
14862
|
}).optional()
|
|
14512
14863
|
}))
|
|
14513
14864
|
}),
|
|
@@ -14526,6 +14877,11 @@ const agentDiscoverySchema = z.object({
|
|
|
14526
14877
|
missingHtmlRoutes: z.array(z.string()),
|
|
14527
14878
|
missingMarkdownRoutes: z.array(z.string()),
|
|
14528
14879
|
orphanMarkdownRoutes: z.array(z.string()),
|
|
14880
|
+
comparisonStatus: z.enum([
|
|
14881
|
+
"complete",
|
|
14882
|
+
"partial",
|
|
14883
|
+
"not-applicable"
|
|
14884
|
+
]).optional(),
|
|
14529
14885
|
error: z.string().optional()
|
|
14530
14886
|
}),
|
|
14531
14887
|
agentSkills: z.object({
|
|
@@ -14631,6 +14987,25 @@ const agentDiscoverySchema = z.object({
|
|
|
14631
14987
|
warnings: z.array(z.string())
|
|
14632
14988
|
});
|
|
14633
14989
|
//#endregion
|
|
14990
|
+
//#region packages/core/dist/analyze/crawler/site-checks-schema.js
|
|
14991
|
+
const crawlSiteChecksSchema = z.object({ soft404: z.object({
|
|
14992
|
+
status: z.enum([
|
|
14993
|
+
"pass",
|
|
14994
|
+
"warning",
|
|
14995
|
+
"unknown"
|
|
14996
|
+
]),
|
|
14997
|
+
probes: z.array(z.object({
|
|
14998
|
+
url: z.string().url(),
|
|
14999
|
+
status: z.number().int().optional(),
|
|
15000
|
+
finalUrl: z.string().url().optional(),
|
|
15001
|
+
redirected: z.boolean().optional(),
|
|
15002
|
+
accessBlocked: z.boolean().optional(),
|
|
15003
|
+
error: z.string().optional()
|
|
15004
|
+
})),
|
|
15005
|
+
probeLimit: z.number().int().positive(),
|
|
15006
|
+
complete: z.boolean()
|
|
15007
|
+
}) });
|
|
15008
|
+
//#endregion
|
|
14634
15009
|
//#region packages/core/dist/analyze/crawler/schemas.js
|
|
14635
15010
|
const crawlerRuleSeveritySchema = z.enum([
|
|
14636
15011
|
"low",
|
|
@@ -15060,6 +15435,34 @@ const crawlPageSnapshotSchema = z.object({
|
|
|
15060
15435
|
mainContentHash: z.string().optional(),
|
|
15061
15436
|
textRatio: z.number().min(0).max(1).optional(),
|
|
15062
15437
|
contentSample: z.string().optional(),
|
|
15438
|
+
contentSketch: z.object({
|
|
15439
|
+
version: z.literal(1),
|
|
15440
|
+
wordCount: z.number().int().nonnegative(),
|
|
15441
|
+
shingleSize: z.number().int().positive(),
|
|
15442
|
+
sampledShingles: z.number().int().nonnegative(),
|
|
15443
|
+
hashes: z.array(z.string())
|
|
15444
|
+
}).optional(),
|
|
15445
|
+
softAuthenticationGate: z.object({
|
|
15446
|
+
kind: z.literal("login-form"),
|
|
15447
|
+
indicators: z.array(z.string()),
|
|
15448
|
+
formActionPath: z.string().optional()
|
|
15449
|
+
}).optional(),
|
|
15450
|
+
tabbedContent: z.object({
|
|
15451
|
+
groups: z.number().int().positive(),
|
|
15452
|
+
panels: z.number().int().positive(),
|
|
15453
|
+
retainedPanels: z.number().int().nonnegative(),
|
|
15454
|
+
truncated: z.boolean(),
|
|
15455
|
+
panelSketches: z.array(z.object({
|
|
15456
|
+
label: z.string().optional(),
|
|
15457
|
+
sketch: z.object({
|
|
15458
|
+
version: z.literal(1),
|
|
15459
|
+
wordCount: z.number().int().nonnegative(),
|
|
15460
|
+
shingleSize: z.number().int().positive(),
|
|
15461
|
+
sampledShingles: z.number().int().nonnegative(),
|
|
15462
|
+
hashes: z.array(z.string())
|
|
15463
|
+
})
|
|
15464
|
+
}))
|
|
15465
|
+
}).optional(),
|
|
15063
15466
|
lang: z.string().optional(),
|
|
15064
15467
|
hasViewport: z.boolean().optional(),
|
|
15065
15468
|
isHttps: z.boolean().optional(),
|
|
@@ -15551,6 +15954,7 @@ const crawlReportBaseSchema = z.object({
|
|
|
15551
15954
|
ai: crawlAiSignalsSchema.optional(),
|
|
15552
15955
|
sitemapDiscovery: crawlSitemapDiscoverySchema.optional(),
|
|
15553
15956
|
externalLinkVerification: crawlExternalLinkVerificationSchema.optional(),
|
|
15957
|
+
siteChecks: crawlSiteChecksSchema.optional(),
|
|
15554
15958
|
agentDiscovery: agentDiscoverySchema.optional(),
|
|
15555
15959
|
warnings: z.array(z.string()),
|
|
15556
15960
|
caveats: z.array(z.string())
|
|
@@ -15580,7 +15984,7 @@ function compareText$10(left, right) {
|
|
|
15580
15984
|
}
|
|
15581
15985
|
return leftPoints.length - rightPoints.length;
|
|
15582
15986
|
}
|
|
15583
|
-
function boundedInteger$
|
|
15987
|
+
function boundedInteger$6(value, fallback, minimum, maximum) {
|
|
15584
15988
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
15585
15989
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
15586
15990
|
}
|
|
@@ -15657,8 +16061,8 @@ function initialSelection(sourceRows) {
|
|
|
15657
16061
|
};
|
|
15658
16062
|
}
|
|
15659
16063
|
function analyzeCtrUnderperformersFromRows(input) {
|
|
15660
|
-
const minImpressions = boundedInteger$
|
|
15661
|
-
const limit = boundedInteger$
|
|
16064
|
+
const minImpressions = boundedInteger$6(input.minImpressions, 200, 1, CTR_MAX_MIN_IMPRESSIONS);
|
|
16065
|
+
const limit = boundedInteger$6(input.limit, 25, 1, 100);
|
|
15662
16066
|
const selection = initialSelection(input.rows.length);
|
|
15663
16067
|
const benchmarkRows = aggregateCtrRows(input.rows, selection).filter((row) => {
|
|
15664
16068
|
const query = row.keys[0] ?? "";
|
|
@@ -16157,7 +16561,7 @@ function analyzeSegmentRows(input) {
|
|
|
16157
16561
|
}
|
|
16158
16562
|
//#endregion
|
|
16159
16563
|
//#region packages/core/dist/analyze/segment-impact-input.js
|
|
16160
|
-
const DAY_MS$
|
|
16564
|
+
const DAY_MS$2 = 864e5;
|
|
16161
16565
|
const MAX_WINDOW_DAYS = 240;
|
|
16162
16566
|
function subtractDays$1(value, days) {
|
|
16163
16567
|
const date = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -16165,7 +16569,7 @@ function subtractDays$1(value, days) {
|
|
|
16165
16569
|
return date.toISOString().slice(0, 10);
|
|
16166
16570
|
}
|
|
16167
16571
|
function segmentRangeDays(range) {
|
|
16168
|
-
return Math.floor((Date.parse(`${range.endDate}T00:00:00.000Z`) - Date.parse(`${range.startDate}T00:00:00.000Z`)) / DAY_MS$
|
|
16572
|
+
return Math.floor((Date.parse(`${range.endDate}T00:00:00.000Z`) - Date.parse(`${range.startDate}T00:00:00.000Z`)) / DAY_MS$2) + 1;
|
|
16169
16573
|
}
|
|
16170
16574
|
function segmentComparisonRange(after, compareDays) {
|
|
16171
16575
|
const endDate = subtractDays$1(after.startDate, 1);
|
|
@@ -16745,13 +17149,13 @@ const MINIMUM_PAGE_SHARE = .1;
|
|
|
16745
17149
|
const MINIMUM_PAGE_IMPRESSIONS = 10;
|
|
16746
17150
|
const MAXIMUM_DOMINANT_SHARE = .8;
|
|
16747
17151
|
const MAX_SUPPRESSIONS = 100;
|
|
16748
|
-
function boundedInteger$
|
|
17152
|
+
function boundedInteger$5(value, fallback, minimum, maximum) {
|
|
16749
17153
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
16750
17154
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
16751
17155
|
}
|
|
16752
17156
|
function analyzeCannibalRows(input) {
|
|
16753
|
-
const minImpressions = boundedInteger$
|
|
16754
|
-
const limit = boundedInteger$
|
|
17157
|
+
const minImpressions = boundedInteger$5(input.minImpressions, 50, 0, 1e9);
|
|
17158
|
+
const limit = boundedInteger$5(input.limit, 25, 1, 100);
|
|
16755
17159
|
const selection = {
|
|
16756
17160
|
sourceRows: input.rows.length,
|
|
16757
17161
|
invalidRows: 0,
|
|
@@ -17531,7 +17935,7 @@ function groupQuickWins(items) {
|
|
|
17531
17935
|
}
|
|
17532
17936
|
//#endregion
|
|
17533
17937
|
//#region packages/core/dist/analyze/site-diagnostics/quick-wins-analysis-primitives.js
|
|
17534
|
-
function boundedInteger$
|
|
17938
|
+
function boundedInteger$4(value, fallback, minimum, maximum) {
|
|
17535
17939
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
17536
17940
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
17537
17941
|
}
|
|
@@ -17652,12 +18056,12 @@ function roundedSum(values) {
|
|
|
17652
18056
|
//#region packages/core/dist/analyze/site-diagnostics/quick-wins-analysis.js
|
|
17653
18057
|
const DEFAULT_MIN_IMPRESSIONS$2 = 200;
|
|
17654
18058
|
const MAX_MIN_IMPRESSIONS$2 = 1e9;
|
|
17655
|
-
const DEFAULT_LIMIT$
|
|
17656
|
-
const MAX_LIMIT$
|
|
18059
|
+
const DEFAULT_LIMIT$3 = 25;
|
|
18060
|
+
const MAX_LIMIT$3 = 100;
|
|
17657
18061
|
const FALLBACK_SOURCE = "builtin_position_ctr_curve_v1";
|
|
17658
18062
|
function analyzeQuickWinsFromRows(input) {
|
|
17659
|
-
const minImpressions = boundedInteger$
|
|
17660
|
-
const limit = boundedInteger$
|
|
18063
|
+
const minImpressions = boundedInteger$4(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$2, 0, MAX_MIN_IMPRESSIONS$2);
|
|
18064
|
+
const limit = boundedInteger$4(input.limit, DEFAULT_LIMIT$3, 1, MAX_LIMIT$3);
|
|
17661
18065
|
const selected = selectBenchmarkRows(input);
|
|
17662
18066
|
const benchmarkContext = createCtrBenchmarkContext(selected.rows, {
|
|
17663
18067
|
samplePopulation: "all_qualified_url_samples",
|
|
@@ -17998,7 +18402,7 @@ function compareText$5(left, right) {
|
|
|
17998
18402
|
}
|
|
17999
18403
|
return leftPoints.length - rightPoints.length;
|
|
18000
18404
|
}
|
|
18001
|
-
function isHttpUrl(value) {
|
|
18405
|
+
function isHttpUrl$1(value) {
|
|
18002
18406
|
try {
|
|
18003
18407
|
const parsed = new URL(value);
|
|
18004
18408
|
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
@@ -18007,7 +18411,7 @@ function isHttpUrl(value) {
|
|
|
18007
18411
|
}
|
|
18008
18412
|
}
|
|
18009
18413
|
function isValidStrikingDistanceRow(row, query, url) {
|
|
18010
|
-
return Boolean(query) && isHttpUrl(url) && Number.isFinite(row.clicks) && Number.isFinite(row.impressions) && Number.isFinite(row.ctr) && Number.isFinite(row.position) && row.clicks >= 0 && row.impressions > 0 && row.clicks <= row.impressions && row.ctr >= 0 && row.ctr <= 1 && row.position > 0;
|
|
18414
|
+
return Boolean(query) && isHttpUrl$1(url) && Number.isFinite(row.clicks) && Number.isFinite(row.impressions) && Number.isFinite(row.ctr) && Number.isFinite(row.position) && row.clicks >= 0 && row.impressions > 0 && row.clicks <= row.impressions && row.ctr >= 0 && row.ctr <= 1 && row.position > 0;
|
|
18011
18415
|
}
|
|
18012
18416
|
function isStrikingDistanceBrandRow(input) {
|
|
18013
18417
|
return shouldExcludeBrandQuery({
|
|
@@ -18097,8 +18501,8 @@ function groupStrikingDistanceItems(items) {
|
|
|
18097
18501
|
//#region packages/core/dist/analyze/striking-distance-analysis.js
|
|
18098
18502
|
const DEFAULT_MIN_IMPRESSIONS$1 = 100;
|
|
18099
18503
|
const MAX_MIN_IMPRESSIONS$1 = 1e9;
|
|
18100
|
-
const DEFAULT_LIMIT$
|
|
18101
|
-
const MAX_LIMIT$
|
|
18504
|
+
const DEFAULT_LIMIT$2 = 25;
|
|
18505
|
+
const MAX_LIMIT$2 = 100;
|
|
18102
18506
|
function integerInRange$1(value, fallback, minimum, maximum) {
|
|
18103
18507
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
18104
18508
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
@@ -18140,7 +18544,7 @@ function selectRows(input) {
|
|
|
18140
18544
|
}
|
|
18141
18545
|
function analyzeStrikingDistanceRows(input) {
|
|
18142
18546
|
const minImpressions = integerInRange$1(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$1, 0, MAX_MIN_IMPRESSIONS$1);
|
|
18143
|
-
const limit = integerInRange$1(input.limit, DEFAULT_LIMIT$
|
|
18547
|
+
const limit = integerInRange$1(input.limit, DEFAULT_LIMIT$2, 1, MAX_LIMIT$2);
|
|
18144
18548
|
const selected = selectRows({
|
|
18145
18549
|
...input,
|
|
18146
18550
|
minImpressions
|
|
@@ -18214,7 +18618,7 @@ const defaultDependencies$7 = {
|
|
|
18214
18618
|
verifyContent: verifyQueryContent,
|
|
18215
18619
|
now: () => /* @__PURE__ */ new Date()
|
|
18216
18620
|
};
|
|
18217
|
-
function boundedInteger$
|
|
18621
|
+
function boundedInteger$3(value, fallback, minimum, maximum) {
|
|
18218
18622
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
18219
18623
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
18220
18624
|
}
|
|
@@ -18278,7 +18682,7 @@ async function strikingDistance(input, dependencies = defaultDependencies$7) {
|
|
|
18278
18682
|
brandTerms: input.brandTerms,
|
|
18279
18683
|
includeBrand: input.includeBrand
|
|
18280
18684
|
});
|
|
18281
|
-
const verifyLimit = boundedInteger$
|
|
18685
|
+
const verifyLimit = boundedInteger$3(input.verifyLimit, DEFAULT_VERIFY_LIMIT, 0, analysis.limit);
|
|
18282
18686
|
const verificationRequested = input.verifyContent === true || input.verifyContent !== false && input.verifyLimit !== void 0;
|
|
18283
18687
|
const attempted = verificationRequested ? Math.min(verifyLimit, analysis.items.length) : 0;
|
|
18284
18688
|
const items = await Promise.all(analysis.items.map(async (item, index) => {
|
|
@@ -18467,7 +18871,7 @@ function standardDeviation(values) {
|
|
|
18467
18871
|
function toFixedNumber(value, digits = 3) {
|
|
18468
18872
|
return Number(value.toFixed(digits));
|
|
18469
18873
|
}
|
|
18470
|
-
function percentChange(baseline, comparison) {
|
|
18874
|
+
function percentChange$1(baseline, comparison) {
|
|
18471
18875
|
if (!baseline) return comparison ? null : 0;
|
|
18472
18876
|
return (comparison - baseline) / baseline * 100;
|
|
18473
18877
|
}
|
|
@@ -18653,7 +19057,7 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
18653
19057
|
const outsideFlatBaseline = zScore === null && comparisonMean !== baselineMean;
|
|
18654
19058
|
const significant = outsideFlatBaseline || zScore !== null && Math.abs(zScore) >= 2;
|
|
18655
19059
|
const direction = significant ? comparisonMean < baselineMean ? "drop" : "spike" : "normal";
|
|
18656
|
-
const relativeChange = percentChange(baselineMean, comparisonMean);
|
|
19060
|
+
const relativeChange = percentChange$1(baselineMean, comparisonMean);
|
|
18657
19061
|
return {
|
|
18658
19062
|
site: input.site,
|
|
18659
19063
|
metric,
|
|
@@ -19712,7 +20116,7 @@ function dateShift(date, days) {
|
|
|
19712
20116
|
}
|
|
19713
20117
|
//#endregion
|
|
19714
20118
|
//#region packages/core/dist/analyze/experiments/measurement/window.js
|
|
19715
|
-
const DAY_MS = 864e5;
|
|
20119
|
+
const DAY_MS$1 = 864e5;
|
|
19716
20120
|
function exactDate(value, label) {
|
|
19717
20121
|
const pattern = /^\d{4}-\d{2}-\d{2}$/;
|
|
19718
20122
|
const parsed = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -19727,7 +20131,7 @@ function boundedDays(value, fallback, label) {
|
|
|
19727
20131
|
function calendarDays(startDate, endDate) {
|
|
19728
20132
|
const start = (/* @__PURE__ */ new Date(`${startDate}T00:00:00.000Z`)).getTime();
|
|
19729
20133
|
const end = (/* @__PURE__ */ new Date(`${endDate}T00:00:00.000Z`)).getTime();
|
|
19730
|
-
return Math.floor((end - start) / DAY_MS) + 1;
|
|
20134
|
+
return Math.floor((end - start) / DAY_MS$1) + 1;
|
|
19731
20135
|
}
|
|
19732
20136
|
function subtractCalendarMonths(value, months) {
|
|
19733
20137
|
const source = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -20273,7 +20677,7 @@ function internalLinksVerdict(input) {
|
|
|
20273
20677
|
function uniqueInternalLinksWarnings(warnings) {
|
|
20274
20678
|
return [...new Map(warnings.map((warning) => [`${warning.stage}\u0000${warning.url}\u0000${warning.code}\u0000${warning.message}`, warning])).values()];
|
|
20275
20679
|
}
|
|
20276
|
-
function errorMessage(error) {
|
|
20680
|
+
function errorMessage$1(error) {
|
|
20277
20681
|
return error instanceof Error ? error.message : String(error);
|
|
20278
20682
|
}
|
|
20279
20683
|
function acceptedAlias(site, value) {
|
|
@@ -20323,7 +20727,7 @@ async function verifyInternalLinkTarget(input) {
|
|
|
20323
20727
|
stage: "target-fetch",
|
|
20324
20728
|
url: input.targetUrl,
|
|
20325
20729
|
code: "fetch-failed",
|
|
20326
|
-
message: errorMessage(error)
|
|
20730
|
+
message: errorMessage$1(error)
|
|
20327
20731
|
}]
|
|
20328
20732
|
};
|
|
20329
20733
|
}
|
|
@@ -20344,7 +20748,7 @@ async function verifyInternalLinkTarget(input) {
|
|
|
20344
20748
|
stage: "target-extract",
|
|
20345
20749
|
url: input.targetUrl,
|
|
20346
20750
|
code: "extract-failed",
|
|
20347
|
-
message: errorMessage(error)
|
|
20751
|
+
message: errorMessage$1(error)
|
|
20348
20752
|
}]
|
|
20349
20753
|
};
|
|
20350
20754
|
}
|
|
@@ -20407,7 +20811,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
20407
20811
|
stage: "source-fetch",
|
|
20408
20812
|
url: input.candidate.sourceUrl,
|
|
20409
20813
|
code: "fetch-failed",
|
|
20410
|
-
message: errorMessage(error)
|
|
20814
|
+
message: errorMessage$1(error)
|
|
20411
20815
|
}]
|
|
20412
20816
|
};
|
|
20413
20817
|
}
|
|
@@ -20425,7 +20829,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
20425
20829
|
stage: "source-fetch",
|
|
20426
20830
|
url: input.candidate.sourceUrl,
|
|
20427
20831
|
code: "fetch-failed",
|
|
20428
|
-
message: errorMessage(error)
|
|
20832
|
+
message: errorMessage$1(error)
|
|
20429
20833
|
}]
|
|
20430
20834
|
};
|
|
20431
20835
|
}
|
|
@@ -20439,7 +20843,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
20439
20843
|
stage: "source-extract",
|
|
20440
20844
|
url: input.candidate.sourceUrl,
|
|
20441
20845
|
code: "extract-failed",
|
|
20442
|
-
message: errorMessage(error)
|
|
20846
|
+
message: errorMessage$1(error)
|
|
20443
20847
|
}]
|
|
20444
20848
|
};
|
|
20445
20849
|
}
|
|
@@ -21247,7 +21651,7 @@ function compareText$3(left, right) {
|
|
|
21247
21651
|
}
|
|
21248
21652
|
return leftPoints.length - rightPoints.length;
|
|
21249
21653
|
}
|
|
21250
|
-
function boundedInteger$
|
|
21654
|
+
function boundedInteger$2(value, fallback, maximum) {
|
|
21251
21655
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
21252
21656
|
return Math.min(maximum, Math.max(1, Math.trunc(value)));
|
|
21253
21657
|
}
|
|
@@ -21440,10 +21844,10 @@ function caveats(input) {
|
|
|
21440
21844
|
return items;
|
|
21441
21845
|
}
|
|
21442
21846
|
function analyzeIndexCoverageSignals(input) {
|
|
21443
|
-
const itemLimit = boundedInteger$
|
|
21444
|
-
const clusterLimit = boundedInteger$
|
|
21445
|
-
const clusterSamples = boundedInteger$
|
|
21446
|
-
const minUrls = boundedInteger$
|
|
21847
|
+
const itemLimit = boundedInteger$2(input.limits?.itemsPerSection, DEFAULT_ITEMS_PER_SECTION, MAX_ITEMS_PER_SECTION);
|
|
21848
|
+
const clusterLimit = boundedInteger$2(input.limits?.templateClusters, DEFAULT_TEMPLATE_LIMIT, MAX_TEMPLATE_LIMIT);
|
|
21849
|
+
const clusterSamples = boundedInteger$2(input.limits?.templateSamples, DEFAULT_TEMPLATE_SAMPLES, MAX_TEMPLATE_SAMPLES);
|
|
21850
|
+
const minUrls = boundedInteger$2(input.templateThresholds?.minUrls, DEFAULT_TEMPLATE_MIN_URLS, Number.MAX_SAFE_INTEGER);
|
|
21447
21851
|
const requestedMinShare = input.templateThresholds?.minShare;
|
|
21448
21852
|
const minShare = Number.isFinite(requestedMinShare) ? Math.min(1, Math.max(0, requestedMinShare ?? DEFAULT_TEMPLATE_MIN_SHARE)) : DEFAULT_TEMPLATE_MIN_SHARE;
|
|
21449
21853
|
const crawl = crawlInput(input.crawl);
|
|
@@ -25130,16 +25534,16 @@ function benchmarkDetails(benchmark) {
|
|
|
25130
25534
|
positiveUrlSamples: benchmark.positiveUrlSamples
|
|
25131
25535
|
};
|
|
25132
25536
|
}
|
|
25133
|
-
function boundedInteger(input) {
|
|
25537
|
+
function boundedInteger$1(input) {
|
|
25134
25538
|
const value = Number.isFinite(input.value) ? input.value ?? 0 : input.fallback;
|
|
25135
25539
|
return Math.min(input.maximum ?? Number.POSITIVE_INFINITY, Math.max(1, Math.floor(value)));
|
|
25136
25540
|
}
|
|
25137
25541
|
function analyzeQueryClustersFromRows(input) {
|
|
25138
|
-
const minImpressions = boundedInteger({
|
|
25542
|
+
const minImpressions = boundedInteger$1({
|
|
25139
25543
|
value: input.minImpressions,
|
|
25140
25544
|
fallback: 25
|
|
25141
25545
|
});
|
|
25142
|
-
const limit = boundedInteger({
|
|
25546
|
+
const limit = boundedInteger$1({
|
|
25143
25547
|
value: input.limit,
|
|
25144
25548
|
fallback: 25,
|
|
25145
25549
|
maximum: 100
|
|
@@ -25781,8 +26185,8 @@ function reportPresentation(report) {
|
|
|
25781
26185
|
//#region packages/core/dist/analyze/second-page-analysis.js
|
|
25782
26186
|
const DEFAULT_MIN_IMPRESSIONS = 50;
|
|
25783
26187
|
const MAX_MIN_IMPRESSIONS = 1e9;
|
|
25784
|
-
const DEFAULT_LIMIT = 10;
|
|
25785
|
-
const MAX_LIMIT = 100;
|
|
26188
|
+
const DEFAULT_LIMIT$1 = 10;
|
|
26189
|
+
const MAX_LIMIT$1 = 100;
|
|
25786
26190
|
function integerInRange(value, fallback, minimum, maximum) {
|
|
25787
26191
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
25788
26192
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
@@ -25869,7 +26273,7 @@ function compareItems(left, right) {
|
|
|
25869
26273
|
}
|
|
25870
26274
|
function analyzeSecondPageRows(input) {
|
|
25871
26275
|
const minImpressions = integerInRange(input.minImpressions, DEFAULT_MIN_IMPRESSIONS, 0, MAX_MIN_IMPRESSIONS);
|
|
25872
|
-
const limit = integerInRange(input.limit, DEFAULT_LIMIT, 1, MAX_LIMIT);
|
|
26276
|
+
const limit = integerInRange(input.limit, DEFAULT_LIMIT$1, 1, MAX_LIMIT$1);
|
|
25873
26277
|
const selection = {
|
|
25874
26278
|
sourceRows: input.rows.length,
|
|
25875
26279
|
invalidRows: 0,
|
|
@@ -27267,28 +27671,884 @@ async function updatePostmortemWorkflow(input, dependencies = {}) {
|
|
|
27267
27671
|
});
|
|
27268
27672
|
}
|
|
27269
27673
|
//#endregion
|
|
27270
|
-
//#region packages/core/dist/
|
|
27271
|
-
const
|
|
27272
|
-
|
|
27273
|
-
|
|
27274
|
-
|
|
27275
|
-
|
|
27276
|
-
|
|
27277
|
-
|
|
27278
|
-
|
|
27279
|
-
|
|
27280
|
-
|
|
27281
|
-
|
|
27282
|
-
|
|
27283
|
-
|
|
27284
|
-
|
|
27285
|
-
|
|
27286
|
-
|
|
27287
|
-
|
|
27288
|
-
|
|
27289
|
-
|
|
27290
|
-
|
|
27291
|
-
|
|
27674
|
+
//#region packages/core/dist/bing/analysis.js
|
|
27675
|
+
const DAY_MS = 864e5;
|
|
27676
|
+
const DEFAULT_COMPARISON_DAYS = 28;
|
|
27677
|
+
const WEEKLY_PERIODS = 4;
|
|
27678
|
+
const OPPORTUNITY_MIN_IMPRESSIONS = 20;
|
|
27679
|
+
const OPPORTUNITY_MIN_POSITION = 4;
|
|
27680
|
+
const OPPORTUNITY_MAX_POSITION = 20;
|
|
27681
|
+
const RESULT_LIMIT = 10;
|
|
27682
|
+
function dateOffset(date, days) {
|
|
27683
|
+
return new Date(Date.parse(`${date}T00:00:00Z`) + days * DAY_MS).toISOString().slice(0, 10);
|
|
27684
|
+
}
|
|
27685
|
+
function percentChange(previous, current) {
|
|
27686
|
+
if (previous === 0) return null;
|
|
27687
|
+
return round((current - previous) / previous * 100);
|
|
27688
|
+
}
|
|
27689
|
+
function round(value, digits = 3) {
|
|
27690
|
+
const factor = 10 ** digits;
|
|
27691
|
+
return Math.round(value * factor) / factor;
|
|
27692
|
+
}
|
|
27693
|
+
function trafficPeriod(byDate, startDate, endDate, expectedDays) {
|
|
27694
|
+
let clicks = 0;
|
|
27695
|
+
let impressions = 0;
|
|
27696
|
+
let observedDays = 0;
|
|
27697
|
+
for (const [date, row] of byDate) {
|
|
27698
|
+
if (date < startDate || date > endDate) continue;
|
|
27699
|
+
observedDays += 1;
|
|
27700
|
+
clicks += row.clicks;
|
|
27701
|
+
impressions += row.impressions;
|
|
27702
|
+
}
|
|
27703
|
+
return {
|
|
27704
|
+
startDate,
|
|
27705
|
+
endDate,
|
|
27706
|
+
expectedDays,
|
|
27707
|
+
observedDays,
|
|
27708
|
+
missingDays: Math.max(0, expectedDays - observedDays),
|
|
27709
|
+
clicks,
|
|
27710
|
+
impressions,
|
|
27711
|
+
ctr: impressions ? round(clicks / impressions * 100) : null
|
|
27712
|
+
};
|
|
27713
|
+
}
|
|
27714
|
+
function analyzeBingTraffic(rows, comparisonDays = DEFAULT_COMPARISON_DAYS) {
|
|
27715
|
+
if (!rows.length) return void 0;
|
|
27716
|
+
const byDate = /* @__PURE__ */ new Map();
|
|
27717
|
+
for (const row of rows) {
|
|
27718
|
+
const existing = byDate.get(row.date) ?? {
|
|
27719
|
+
clicks: 0,
|
|
27720
|
+
impressions: 0
|
|
27721
|
+
};
|
|
27722
|
+
existing.clicks += row.clicks;
|
|
27723
|
+
existing.impressions += row.impressions;
|
|
27724
|
+
byDate.set(row.date, existing);
|
|
27725
|
+
}
|
|
27726
|
+
const latestDate = [...byDate.keys()].sort().at(-1);
|
|
27727
|
+
if (!latestDate) return void 0;
|
|
27728
|
+
const currentStart = dateOffset(latestDate, -(comparisonDays - 1));
|
|
27729
|
+
const previousEnd = dateOffset(currentStart, -1);
|
|
27730
|
+
const previousStart = dateOffset(previousEnd, -(comparisonDays - 1));
|
|
27731
|
+
const current = trafficPeriod(byDate, currentStart, latestDate, comparisonDays);
|
|
27732
|
+
const previous = trafficPeriod(byDate, previousStart, previousEnd, comparisonDays);
|
|
27733
|
+
return {
|
|
27734
|
+
comparisonDays,
|
|
27735
|
+
current,
|
|
27736
|
+
previous,
|
|
27737
|
+
changes: {
|
|
27738
|
+
clicks: current.clicks - previous.clicks,
|
|
27739
|
+
clicksPercent: percentChange(previous.clicks, current.clicks),
|
|
27740
|
+
impressions: current.impressions - previous.impressions,
|
|
27741
|
+
impressionsPercent: percentChange(previous.impressions, current.impressions),
|
|
27742
|
+
ctrPercentagePoints: current.ctr === null || previous.ctr === null ? null : round(current.ctr - previous.ctr)
|
|
27743
|
+
}
|
|
27744
|
+
};
|
|
27745
|
+
}
|
|
27746
|
+
const CRAWL_FIELDS = [
|
|
27747
|
+
"crawledPages",
|
|
27748
|
+
"inIndex",
|
|
27749
|
+
"inLinks",
|
|
27750
|
+
"crawlErrors",
|
|
27751
|
+
"code4xx",
|
|
27752
|
+
"code5xx",
|
|
27753
|
+
"blockedByRobotsTxt",
|
|
27754
|
+
"connectionTimeout",
|
|
27755
|
+
"dnsFailures",
|
|
27756
|
+
"containsMalware"
|
|
27757
|
+
];
|
|
27758
|
+
function analyzeBingCrawl(rows, comparisonDays = DEFAULT_COMPARISON_DAYS) {
|
|
27759
|
+
const ordered = [...rows].sort((a, b) => a.date.localeCompare(b.date, "en"));
|
|
27760
|
+
const current = ordered.at(-1);
|
|
27761
|
+
if (!current) return void 0;
|
|
27762
|
+
const targetDate = dateOffset(current.date, -comparisonDays);
|
|
27763
|
+
const previous = ordered.filter((row) => row.date <= targetDate).at(-1);
|
|
27764
|
+
const changes = {};
|
|
27765
|
+
if (!previous) return {
|
|
27766
|
+
comparisonDays,
|
|
27767
|
+
targetDate,
|
|
27768
|
+
current,
|
|
27769
|
+
previous: void 0,
|
|
27770
|
+
changes
|
|
27771
|
+
};
|
|
27772
|
+
for (const field of CRAWL_FIELDS) {
|
|
27773
|
+
const currentValue = current[field];
|
|
27774
|
+
const previousValue = previous[field];
|
|
27775
|
+
if (typeof currentValue !== "number" || typeof previousValue !== "number") continue;
|
|
27776
|
+
changes[field] = {
|
|
27777
|
+
previous: previousValue,
|
|
27778
|
+
current: currentValue,
|
|
27779
|
+
absolute: currentValue - previousValue,
|
|
27780
|
+
percent: percentChange(previousValue, currentValue)
|
|
27781
|
+
};
|
|
27782
|
+
}
|
|
27783
|
+
return {
|
|
27784
|
+
comparisonDays,
|
|
27785
|
+
targetDate,
|
|
27786
|
+
current,
|
|
27787
|
+
previous,
|
|
27788
|
+
changes
|
|
27789
|
+
};
|
|
27790
|
+
}
|
|
27791
|
+
function aggregateDimensions(rows, dates) {
|
|
27792
|
+
const values = /* @__PURE__ */ new Map();
|
|
27793
|
+
for (const row of rows) {
|
|
27794
|
+
if (!dates.has(row.date)) continue;
|
|
27795
|
+
const item = values.get(row.value) ?? {
|
|
27796
|
+
clicks: 0,
|
|
27797
|
+
impressions: 0,
|
|
27798
|
+
clickPositionTotal: 0,
|
|
27799
|
+
clickPositionWeight: 0,
|
|
27800
|
+
impressionPositionTotal: 0,
|
|
27801
|
+
impressionPositionWeight: 0,
|
|
27802
|
+
observedDates: /* @__PURE__ */ new Set()
|
|
27803
|
+
};
|
|
27804
|
+
item.clicks += row.clicks;
|
|
27805
|
+
item.impressions += row.impressions;
|
|
27806
|
+
item.observedDates.add(row.date);
|
|
27807
|
+
if (row.avgClickPosition !== void 0 && row.clicks > 0) {
|
|
27808
|
+
const weight = row.clicks;
|
|
27809
|
+
item.clickPositionTotal += row.avgClickPosition * weight;
|
|
27810
|
+
item.clickPositionWeight += weight;
|
|
27811
|
+
}
|
|
27812
|
+
if (row.avgImpressionPosition !== void 0 && row.impressions > 0) {
|
|
27813
|
+
const weight = row.impressions;
|
|
27814
|
+
item.impressionPositionTotal += row.avgImpressionPosition * weight;
|
|
27815
|
+
item.impressionPositionWeight += weight;
|
|
27816
|
+
}
|
|
27817
|
+
values.set(row.value, item);
|
|
27818
|
+
}
|
|
27819
|
+
return new Map([...values].map(([value, item]) => [value, {
|
|
27820
|
+
value,
|
|
27821
|
+
observedWeeks: item.observedDates.size,
|
|
27822
|
+
clicks: item.clicks,
|
|
27823
|
+
impressions: item.impressions,
|
|
27824
|
+
avgClickPosition: item.clickPositionWeight ? round(item.clickPositionTotal / item.clickPositionWeight) : null,
|
|
27825
|
+
avgImpressionPosition: item.impressionPositionWeight ? round(item.impressionPositionTotal / item.impressionPositionWeight) : null
|
|
27826
|
+
}]));
|
|
27827
|
+
}
|
|
27828
|
+
function compareValue(a, b) {
|
|
27829
|
+
return a.value.localeCompare(b.value, "en");
|
|
27830
|
+
}
|
|
27831
|
+
function analyzeBingDimensions(rows, kind) {
|
|
27832
|
+
const dates = [...new Set(rows.map((row) => row.date))].sort().reverse();
|
|
27833
|
+
const currentDates = dates.slice(0, WEEKLY_PERIODS);
|
|
27834
|
+
const previousDates = dates.slice(WEEKLY_PERIODS, WEEKLY_PERIODS * 2);
|
|
27835
|
+
const current = aggregateDimensions(rows, new Set(currentDates));
|
|
27836
|
+
const previous = aggregateDimensions(rows, new Set(previousDates));
|
|
27837
|
+
const matchedValues = [...current.keys()].filter((value) => previous.has(value));
|
|
27838
|
+
const comparableValues = currentDates.length === WEEKLY_PERIODS && previousDates.length === WEEKLY_PERIODS ? matchedValues.filter((value) => {
|
|
27839
|
+
const currentValue = current.get(value);
|
|
27840
|
+
const previousValue = previous.get(value);
|
|
27841
|
+
return currentValue?.observedWeeks === WEEKLY_PERIODS && previousValue?.observedWeeks === WEEKLY_PERIODS;
|
|
27842
|
+
}) : [];
|
|
27843
|
+
const movements = comparableValues.map((value) => {
|
|
27844
|
+
const currentValue = current.get(value);
|
|
27845
|
+
const previousValue = previous.get(value);
|
|
27846
|
+
return {
|
|
27847
|
+
value,
|
|
27848
|
+
current: currentValue,
|
|
27849
|
+
previous: previousValue,
|
|
27850
|
+
changes: {
|
|
27851
|
+
clicks: currentValue.clicks - previousValue.clicks,
|
|
27852
|
+
clicksPercent: percentChange(previousValue.clicks, currentValue.clicks),
|
|
27853
|
+
impressions: currentValue.impressions - previousValue.impressions,
|
|
27854
|
+
impressionsPercent: percentChange(previousValue.impressions, currentValue.impressions)
|
|
27855
|
+
}
|
|
27856
|
+
};
|
|
27857
|
+
}).sort((a, b) => Math.abs(b.changes.clicks) - Math.abs(a.changes.clicks) || Math.abs(b.changes.impressions) - Math.abs(a.changes.impressions) || compareValue(a, b)).slice(0, RESULT_LIMIT);
|
|
27858
|
+
const opportunities = [...current.values()].filter((row) => row.impressions >= OPPORTUNITY_MIN_IMPRESSIONS && row.avgImpressionPosition !== null && row.avgImpressionPosition >= OPPORTUNITY_MIN_POSITION && row.avgImpressionPosition <= OPPORTUNITY_MAX_POSITION).sort((a, b) => b.impressions - a.impressions || b.clicks - a.clicks || compareValue(a, b)).slice(0, RESULT_LIMIT);
|
|
27859
|
+
return {
|
|
27860
|
+
kind,
|
|
27861
|
+
sourceSemantics: "Bing weekly top-list evidence. Missing dimensions are unknown, not zero. Movements require observation in every week of both periods.",
|
|
27862
|
+
periods: {
|
|
27863
|
+
weeksPerPeriod: WEEKLY_PERIODS,
|
|
27864
|
+
currentDates,
|
|
27865
|
+
previousDates
|
|
27866
|
+
},
|
|
27867
|
+
coverage: {
|
|
27868
|
+
sourceRows: rows.length,
|
|
27869
|
+
currentDimensions: current.size,
|
|
27870
|
+
previousDimensions: previous.size,
|
|
27871
|
+
matchedDimensions: matchedValues.length,
|
|
27872
|
+
comparableDimensions: comparableValues.length,
|
|
27873
|
+
incompleteMatchedDimensions: matchedValues.length - comparableValues.length,
|
|
27874
|
+
currentOnlyDimensions: [...current.keys()].filter((value) => !previous.has(value)).length,
|
|
27875
|
+
previousOnlyDimensions: [...previous.keys()].filter((value) => !current.has(value)).length
|
|
27876
|
+
},
|
|
27877
|
+
thresholds: {
|
|
27878
|
+
heuristic: true,
|
|
27879
|
+
opportunityMinImpressions: OPPORTUNITY_MIN_IMPRESSIONS,
|
|
27880
|
+
opportunityPositionRange: [OPPORTUNITY_MIN_POSITION, OPPORTUNITY_MAX_POSITION],
|
|
27881
|
+
resultLimit: RESULT_LIMIT
|
|
27882
|
+
},
|
|
27883
|
+
movements,
|
|
27884
|
+
opportunities
|
|
27885
|
+
};
|
|
27886
|
+
}
|
|
27887
|
+
//#endregion
|
|
27888
|
+
//#region packages/core/dist/bing/client.js
|
|
27889
|
+
const DEFAULT_BASE_URL = "https://ssl.bing.com/webmaster/api.svc/json";
|
|
27890
|
+
const MAX_RESPONSE_BYTES$1 = 2e6;
|
|
27891
|
+
const MAX_STAT_ROWS = 400;
|
|
27892
|
+
const MAX_DIMENSION_ROWS = 8e3;
|
|
27893
|
+
const MAX_LINK_ROWS = 200;
|
|
27894
|
+
function bingDate(value) {
|
|
27895
|
+
if (typeof value !== "string") return void 0;
|
|
27896
|
+
const legacy = /^\/Date\((-?\d+)(?:[+-]\d{4})?\)\/$/.exec(value);
|
|
27897
|
+
const date = legacy ? new Date(Number(legacy[1])) : new Date(value);
|
|
27898
|
+
return Number.isNaN(date.getTime()) ? void 0 : date.toISOString().slice(0, 10);
|
|
27899
|
+
}
|
|
27900
|
+
function nonnegativeNumber(value) {
|
|
27901
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
|
|
27902
|
+
}
|
|
27903
|
+
function positionNumber(value) {
|
|
27904
|
+
if (value === void 0 || value === null || value === -1) return { valid: true };
|
|
27905
|
+
const number = nonnegativeNumber(value);
|
|
27906
|
+
return number === void 0 ? { valid: false } : {
|
|
27907
|
+
valid: true,
|
|
27908
|
+
value: number
|
|
27909
|
+
};
|
|
27910
|
+
}
|
|
27911
|
+
function isHttpUrl(value) {
|
|
27912
|
+
if (typeof value !== "string" || value.length > 2e3) return false;
|
|
27913
|
+
try {
|
|
27914
|
+
const url = new URL(value);
|
|
27915
|
+
return url.protocol === "http:" || url.protocol === "https:";
|
|
27916
|
+
} catch {
|
|
27917
|
+
return false;
|
|
27918
|
+
}
|
|
27919
|
+
}
|
|
27920
|
+
function boundedRows(rows, invalidRows, limit = MAX_STAT_ROWS) {
|
|
27921
|
+
const capped = rows.length > limit;
|
|
27922
|
+
return {
|
|
27923
|
+
rows: capped ? rows.slice(-limit) : rows,
|
|
27924
|
+
invalidRows,
|
|
27925
|
+
capped,
|
|
27926
|
+
returnedRows: rows.length
|
|
27927
|
+
};
|
|
27928
|
+
}
|
|
27929
|
+
function dimensionValue(value, kind) {
|
|
27930
|
+
if (typeof value !== "string") return void 0;
|
|
27931
|
+
const normalized = value.trim();
|
|
27932
|
+
if (!normalized || normalized.length > 2e3) return void 0;
|
|
27933
|
+
if (kind === "page" && !isHttpUrl(normalized)) return void 0;
|
|
27934
|
+
return normalized;
|
|
27935
|
+
}
|
|
27936
|
+
function errorMessage(body) {
|
|
27937
|
+
try {
|
|
27938
|
+
const parsed = JSON.parse(body);
|
|
27939
|
+
return typeof parsed.Message === "string" ? parsed.Message : void 0;
|
|
27940
|
+
} catch {
|
|
27941
|
+
return;
|
|
27942
|
+
}
|
|
27943
|
+
}
|
|
27944
|
+
var BingWebmasterClient = class {
|
|
27945
|
+
credentials;
|
|
27946
|
+
authentication;
|
|
27947
|
+
baseUrl;
|
|
27948
|
+
fetchImpl;
|
|
27949
|
+
timeoutMs;
|
|
27950
|
+
constructor(credentials, options = {}) {
|
|
27951
|
+
this.credentials = credentials;
|
|
27952
|
+
if (!(credentials.apiKey?.trim() || credentials.accessToken?.trim())) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster credentials are empty.");
|
|
27953
|
+
this.authentication = credentials.apiKey ? "api-key" : "oauth";
|
|
27954
|
+
this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, "");
|
|
27955
|
+
this.fetchImpl = options.fetchImpl ?? publicHttpFetch;
|
|
27956
|
+
this.timeoutMs = options.timeoutMs ?? 3e4;
|
|
27957
|
+
}
|
|
27958
|
+
async request(method, params = {}) {
|
|
27959
|
+
const url = new URL(`${this.baseUrl}/${method}`);
|
|
27960
|
+
for (const [key, value] of Object.entries(params)) url.searchParams.set(key, value);
|
|
27961
|
+
if (this.credentials.apiKey) url.searchParams.set("apikey", this.credentials.apiKey);
|
|
27962
|
+
let response;
|
|
27963
|
+
try {
|
|
27964
|
+
response = await this.fetchImpl(url.toString(), {
|
|
27965
|
+
headers: {
|
|
27966
|
+
accept: "application/json",
|
|
27967
|
+
...this.credentials.accessToken ? { authorization: `Bearer ${this.credentials.accessToken}` } : {}
|
|
27968
|
+
},
|
|
27969
|
+
signal: AbortSignal.timeout(this.timeoutMs)
|
|
27970
|
+
});
|
|
27971
|
+
} catch (error) {
|
|
27972
|
+
if (error instanceof SeoError) throw error;
|
|
27973
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster did not respond: ${error instanceof Error ? error.message : String(error)}`);
|
|
27974
|
+
}
|
|
27975
|
+
const body = await readBoundedResponseText(response, MAX_RESPONSE_BYTES$1, "Bing Webmaster response");
|
|
27976
|
+
if (!response.ok) {
|
|
27977
|
+
const message = errorMessage(body);
|
|
27978
|
+
if (response.status === 401 || /invalidapikey|unauthorized/i.test(message ?? "")) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster rejected the saved credential. Run `seo providers bing connect` again.");
|
|
27979
|
+
if (response.status === 403) throw new SeoError("ACCESS_DENIED", "Bing Webmaster denied access to this site. Check that it is verified for the connected account.");
|
|
27980
|
+
if (response.status === 429) throw new SeoError("RATE_LIMITED", "Bing Webmaster rate limited this request. Try again later.");
|
|
27981
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster request failed with HTTP ${response.status}${message ? ` (${message})` : ""}.`);
|
|
27982
|
+
}
|
|
27983
|
+
try {
|
|
27984
|
+
return JSON.parse(body).d;
|
|
27985
|
+
} catch {
|
|
27986
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid JSON.");
|
|
27987
|
+
}
|
|
27988
|
+
}
|
|
27989
|
+
async listSites() {
|
|
27990
|
+
const data = await this.request("GetUserSites");
|
|
27991
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned an invalid site list.");
|
|
27992
|
+
const sites = [];
|
|
27993
|
+
let invalidRows = 0;
|
|
27994
|
+
for (const item of data) {
|
|
27995
|
+
if (!item || typeof item !== "object") {
|
|
27996
|
+
invalidRows += 1;
|
|
27997
|
+
continue;
|
|
27998
|
+
}
|
|
27999
|
+
const row = item;
|
|
28000
|
+
if (typeof row.Url !== "string" || typeof row.IsVerified !== "boolean") {
|
|
28001
|
+
invalidRows += 1;
|
|
28002
|
+
continue;
|
|
28003
|
+
}
|
|
28004
|
+
sites.push({
|
|
28005
|
+
url: row.Url,
|
|
28006
|
+
isVerified: row.IsVerified
|
|
28007
|
+
});
|
|
28008
|
+
}
|
|
28009
|
+
sites.sort((a, b) => a.url.localeCompare(b.url, "en"));
|
|
28010
|
+
return {
|
|
28011
|
+
sites,
|
|
28012
|
+
invalidRows
|
|
28013
|
+
};
|
|
28014
|
+
}
|
|
28015
|
+
async getTraffic(siteUrl) {
|
|
28016
|
+
const data = await this.request("GetRankAndTrafficStats", { siteUrl });
|
|
28017
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid traffic statistics.");
|
|
28018
|
+
const rows = [];
|
|
28019
|
+
let invalidRows = 0;
|
|
28020
|
+
for (const item of data) {
|
|
28021
|
+
if (!item || typeof item !== "object") {
|
|
28022
|
+
invalidRows += 1;
|
|
28023
|
+
continue;
|
|
28024
|
+
}
|
|
28025
|
+
const row = item;
|
|
28026
|
+
const date = bingDate(row.Date);
|
|
28027
|
+
const clicks = nonnegativeNumber(row.Clicks);
|
|
28028
|
+
const impressions = nonnegativeNumber(row.Impressions);
|
|
28029
|
+
if (!date || clicks === void 0 || impressions === void 0) {
|
|
28030
|
+
invalidRows += 1;
|
|
28031
|
+
continue;
|
|
28032
|
+
}
|
|
28033
|
+
rows.push({
|
|
28034
|
+
date,
|
|
28035
|
+
clicks,
|
|
28036
|
+
impressions
|
|
28037
|
+
});
|
|
28038
|
+
}
|
|
28039
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en") || a.clicks - b.clicks || a.impressions - b.impressions);
|
|
28040
|
+
return boundedRows(rows, invalidRows);
|
|
28041
|
+
}
|
|
28042
|
+
async getCrawlStats(siteUrl) {
|
|
28043
|
+
const data = await this.request("GetCrawlStats", { siteUrl });
|
|
28044
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid crawl statistics.");
|
|
28045
|
+
const fieldMap = {
|
|
28046
|
+
AllOtherCodes: "allOtherCodes",
|
|
28047
|
+
BlockedByRobotsTxt: "blockedByRobotsTxt",
|
|
28048
|
+
Code2xx: "code2xx",
|
|
28049
|
+
Code301: "code301",
|
|
28050
|
+
Code302: "code302",
|
|
28051
|
+
Code4xx: "code4xx",
|
|
28052
|
+
Code5xx: "code5xx",
|
|
28053
|
+
ConnectionTimeout: "connectionTimeout",
|
|
28054
|
+
ContainsMalware: "containsMalware",
|
|
28055
|
+
CrawledPages: "crawledPages",
|
|
28056
|
+
CrawlErrors: "crawlErrors",
|
|
28057
|
+
DnsFailures: "dnsFailures",
|
|
28058
|
+
InIndex: "inIndex",
|
|
28059
|
+
InLinks: "inLinks"
|
|
28060
|
+
};
|
|
28061
|
+
const rows = [];
|
|
28062
|
+
let invalidRows = 0;
|
|
28063
|
+
for (const item of data) {
|
|
28064
|
+
if (!item || typeof item !== "object") {
|
|
28065
|
+
invalidRows += 1;
|
|
28066
|
+
continue;
|
|
28067
|
+
}
|
|
28068
|
+
const source = item;
|
|
28069
|
+
const date = bingDate(source.Date);
|
|
28070
|
+
if (!date) {
|
|
28071
|
+
invalidRows += 1;
|
|
28072
|
+
continue;
|
|
28073
|
+
}
|
|
28074
|
+
const row = { date };
|
|
28075
|
+
let invalid = false;
|
|
28076
|
+
for (const [providerField, field] of Object.entries(fieldMap)) {
|
|
28077
|
+
const value = source[providerField];
|
|
28078
|
+
if (value === void 0 || value === null) continue;
|
|
28079
|
+
const number = nonnegativeNumber(value);
|
|
28080
|
+
if (number === void 0) {
|
|
28081
|
+
invalid = true;
|
|
28082
|
+
break;
|
|
28083
|
+
}
|
|
28084
|
+
row[field] = number;
|
|
28085
|
+
}
|
|
28086
|
+
if (invalid) {
|
|
28087
|
+
invalidRows += 1;
|
|
28088
|
+
continue;
|
|
28089
|
+
}
|
|
28090
|
+
rows.push(row);
|
|
28091
|
+
}
|
|
28092
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en"));
|
|
28093
|
+
return boundedRows(rows, invalidRows);
|
|
28094
|
+
}
|
|
28095
|
+
async getQueryStats(siteUrl) {
|
|
28096
|
+
return this.getDimensionStats("GetQueryStats", siteUrl, "query");
|
|
28097
|
+
}
|
|
28098
|
+
async getPageStats(siteUrl) {
|
|
28099
|
+
return this.getDimensionStats("GetPageStats", siteUrl, "page");
|
|
28100
|
+
}
|
|
28101
|
+
async getDimensionStats(method, siteUrl, kind) {
|
|
28102
|
+
const data = await this.request(method, { siteUrl });
|
|
28103
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster returned invalid ${kind} statistics.`);
|
|
28104
|
+
const rows = [];
|
|
28105
|
+
let invalidRows = 0;
|
|
28106
|
+
for (const item of data) {
|
|
28107
|
+
if (!item || typeof item !== "object") {
|
|
28108
|
+
invalidRows += 1;
|
|
28109
|
+
continue;
|
|
28110
|
+
}
|
|
28111
|
+
const source = item;
|
|
28112
|
+
const date = bingDate(source.Date);
|
|
28113
|
+
const value = dimensionValue(source.Query, kind);
|
|
28114
|
+
const clicks = nonnegativeNumber(source.Clicks);
|
|
28115
|
+
const impressions = nonnegativeNumber(source.Impressions);
|
|
28116
|
+
const clickPosition = positionNumber(source.AvgClickPosition);
|
|
28117
|
+
const impressionPosition = positionNumber(source.AvgImpressionPosition);
|
|
28118
|
+
if (!date || !value || clicks === void 0 || impressions === void 0 || !clickPosition.valid || !impressionPosition.valid) {
|
|
28119
|
+
invalidRows += 1;
|
|
28120
|
+
continue;
|
|
28121
|
+
}
|
|
28122
|
+
rows.push({
|
|
28123
|
+
date,
|
|
28124
|
+
value,
|
|
28125
|
+
clicks,
|
|
28126
|
+
impressions,
|
|
28127
|
+
avgClickPosition: clickPosition.value,
|
|
28128
|
+
avgImpressionPosition: impressionPosition.value
|
|
28129
|
+
});
|
|
28130
|
+
}
|
|
28131
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en") || a.value.localeCompare(b.value, "en"));
|
|
28132
|
+
return boundedRows(rows, invalidRows, MAX_DIMENSION_ROWS);
|
|
28133
|
+
}
|
|
28134
|
+
async getLinkCounts(siteUrl, page) {
|
|
28135
|
+
const data = await this.request("GetLinkCounts", {
|
|
28136
|
+
siteUrl,
|
|
28137
|
+
page: String(page)
|
|
28138
|
+
});
|
|
28139
|
+
if (!data || typeof data !== "object") throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link counts.");
|
|
28140
|
+
const response = data;
|
|
28141
|
+
if (!Array.isArray(response.Links)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link counts.");
|
|
28142
|
+
const rows = [];
|
|
28143
|
+
let invalidRows = 0;
|
|
28144
|
+
for (const item of response.Links) {
|
|
28145
|
+
if (!item || typeof item !== "object") {
|
|
28146
|
+
invalidRows += 1;
|
|
28147
|
+
continue;
|
|
28148
|
+
}
|
|
28149
|
+
const row = item;
|
|
28150
|
+
const count = nonnegativeNumber(row.Count);
|
|
28151
|
+
if (!isHttpUrl(row.Url) || count === void 0) {
|
|
28152
|
+
invalidRows += 1;
|
|
28153
|
+
continue;
|
|
28154
|
+
}
|
|
28155
|
+
rows.push({
|
|
28156
|
+
url: row.Url,
|
|
28157
|
+
count
|
|
28158
|
+
});
|
|
28159
|
+
}
|
|
28160
|
+
rows.sort((a, b) => b.count - a.count || a.url.localeCompare(b.url, "en"));
|
|
28161
|
+
return linkPage(rows, response.TotalPages, invalidRows);
|
|
28162
|
+
}
|
|
28163
|
+
async getUrlLinks(siteUrl, link, page) {
|
|
28164
|
+
const data = await this.request("GetUrlLinks", {
|
|
28165
|
+
siteUrl,
|
|
28166
|
+
link,
|
|
28167
|
+
page: String(page)
|
|
28168
|
+
});
|
|
28169
|
+
if (!data || typeof data !== "object") throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid referring links.");
|
|
28170
|
+
const response = data;
|
|
28171
|
+
if (!Array.isArray(response.Details)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid referring links.");
|
|
28172
|
+
const rows = [];
|
|
28173
|
+
let invalidRows = 0;
|
|
28174
|
+
for (const item of response.Details) {
|
|
28175
|
+
if (!item || typeof item !== "object") {
|
|
28176
|
+
invalidRows += 1;
|
|
28177
|
+
continue;
|
|
28178
|
+
}
|
|
28179
|
+
const row = item;
|
|
28180
|
+
if (!isHttpUrl(row.Url)) {
|
|
28181
|
+
invalidRows += 1;
|
|
28182
|
+
continue;
|
|
28183
|
+
}
|
|
28184
|
+
rows.push({
|
|
28185
|
+
url: row.Url,
|
|
28186
|
+
anchorText: typeof row.AnchorText === "string" && row.AnchorText.trim() ? row.AnchorText.trim() : void 0
|
|
28187
|
+
});
|
|
28188
|
+
}
|
|
28189
|
+
rows.sort((a, b) => a.url.localeCompare(b.url, "en") || (a.anchorText ?? "").localeCompare(b.anchorText ?? "", "en"));
|
|
28190
|
+
return linkPage(rows, response.TotalPages, invalidRows);
|
|
28191
|
+
}
|
|
28192
|
+
};
|
|
28193
|
+
function linkPage(rows, totalPagesValue, invalidRows) {
|
|
28194
|
+
const totalPages = nonnegativeNumber(totalPagesValue);
|
|
28195
|
+
if (totalPages === void 0 || !Number.isInteger(totalPages)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link pagination.");
|
|
28196
|
+
const capped = rows.length > MAX_LINK_ROWS;
|
|
28197
|
+
return {
|
|
28198
|
+
rows: capped ? rows.slice(0, MAX_LINK_ROWS) : rows,
|
|
28199
|
+
totalPages,
|
|
28200
|
+
invalidRows,
|
|
28201
|
+
capped,
|
|
28202
|
+
returnedRows: rows.length
|
|
28203
|
+
};
|
|
28204
|
+
}
|
|
28205
|
+
//#endregion
|
|
28206
|
+
//#region packages/core/dist/storage/provider-secrets.js
|
|
28207
|
+
const KEYRING_SERVICE = "seo";
|
|
28208
|
+
function account(name) {
|
|
28209
|
+
return `provider:${name}`;
|
|
28210
|
+
}
|
|
28211
|
+
function readFile$1() {
|
|
28212
|
+
return readJsonFile(getSeoCliPaths().providerSecretsFile) ?? {
|
|
28213
|
+
version: 1,
|
|
28214
|
+
secrets: {}
|
|
28215
|
+
};
|
|
28216
|
+
}
|
|
28217
|
+
async function readProviderSecret(input) {
|
|
28218
|
+
const environment = input.env ?? process.env;
|
|
28219
|
+
const environmentValue = input.envVar ? environment[input.envVar]?.trim() : void 0;
|
|
28220
|
+
if (environmentValue) return {
|
|
28221
|
+
value: environmentValue,
|
|
28222
|
+
source: "environment"
|
|
28223
|
+
};
|
|
28224
|
+
if (readConfig().security.useKeychain) try {
|
|
28225
|
+
const value = await getKeyringPassword(KEYRING_SERVICE, account(input.name));
|
|
28226
|
+
if (value) return {
|
|
28227
|
+
value,
|
|
28228
|
+
source: "keychain"
|
|
28229
|
+
};
|
|
28230
|
+
} catch {}
|
|
28231
|
+
const value = readFile$1().secrets[input.name];
|
|
28232
|
+
return value ? {
|
|
28233
|
+
value,
|
|
28234
|
+
source: "file"
|
|
28235
|
+
} : void 0;
|
|
28236
|
+
}
|
|
28237
|
+
//#endregion
|
|
28238
|
+
//#region packages/core/dist/bing/credentials.js
|
|
28239
|
+
const BING_API_KEY_SECRET = "bing-api-key";
|
|
28240
|
+
const BING_API_KEY_ENV = "SEO_BING_API_KEY";
|
|
28241
|
+
async function createBingWebmasterClient(input = {}) {
|
|
28242
|
+
const secret = await readProviderSecret({
|
|
28243
|
+
name: BING_API_KEY_SECRET,
|
|
28244
|
+
envVar: BING_API_KEY_ENV,
|
|
28245
|
+
env: input.env
|
|
28246
|
+
});
|
|
28247
|
+
if (!secret) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster is not connected. Run `seo providers bing connect`, or set SEO_BING_API_KEY for this process.");
|
|
28248
|
+
return {
|
|
28249
|
+
client: new BingWebmasterClient({ apiKey: secret.value }),
|
|
28250
|
+
credentialSource: secret.source
|
|
28251
|
+
};
|
|
28252
|
+
}
|
|
28253
|
+
//#endregion
|
|
28254
|
+
//#region packages/core/dist/bing/overview.js
|
|
28255
|
+
const DAILY_OUTPUT_ROWS = 14;
|
|
28256
|
+
function message(error) {
|
|
28257
|
+
return error instanceof Error ? error.message : String(error);
|
|
28258
|
+
}
|
|
28259
|
+
function range(rows) {
|
|
28260
|
+
return rows.length ? {
|
|
28261
|
+
startDate: rows[0]?.date ?? "",
|
|
28262
|
+
endDate: rows.at(-1)?.date ?? ""
|
|
28263
|
+
} : void 0;
|
|
28264
|
+
}
|
|
28265
|
+
function dailyOutput(data) {
|
|
28266
|
+
const rows = data.rows.slice(-14);
|
|
28267
|
+
return {
|
|
28268
|
+
...data,
|
|
28269
|
+
rows,
|
|
28270
|
+
retainedRows: data.rows.length,
|
|
28271
|
+
outputSelection: {
|
|
28272
|
+
strategy: "most-recent",
|
|
28273
|
+
availableRows: data.rows.length,
|
|
28274
|
+
returnedRows: rows.length,
|
|
28275
|
+
omittedRows: Math.max(0, data.rows.length - rows.length)
|
|
28276
|
+
}
|
|
28277
|
+
};
|
|
28278
|
+
}
|
|
28279
|
+
async function section(work, options) {
|
|
28280
|
+
try {
|
|
28281
|
+
const data = await work();
|
|
28282
|
+
const reasons = [
|
|
28283
|
+
...data.invalidRows ? [`${data.invalidRows} invalid provider rows were excluded.`] : [],
|
|
28284
|
+
...data.capped ? [`The provider response exceeded the ${options.rowLimit}-row limit.`] : [],
|
|
28285
|
+
...options.sourceWarning ? [options.sourceWarning] : []
|
|
28286
|
+
];
|
|
28287
|
+
return reasons.length ? {
|
|
28288
|
+
status: "partial",
|
|
28289
|
+
data,
|
|
28290
|
+
warning: reasons.join(" ")
|
|
28291
|
+
} : {
|
|
28292
|
+
status: "complete",
|
|
28293
|
+
data
|
|
28294
|
+
};
|
|
28295
|
+
} catch (error) {
|
|
28296
|
+
if (error instanceof SeoError && (error.code === "AUTH_REQUIRED" || error.code === "ACCESS_DENIED")) throw error;
|
|
28297
|
+
return {
|
|
28298
|
+
status: "unavailable",
|
|
28299
|
+
warning: message(error)
|
|
28300
|
+
};
|
|
28301
|
+
}
|
|
28302
|
+
}
|
|
28303
|
+
function changeFinding(input) {
|
|
28304
|
+
return {
|
|
28305
|
+
code: input.code,
|
|
28306
|
+
severity: "warning",
|
|
28307
|
+
title: input.title,
|
|
28308
|
+
evidence: {
|
|
28309
|
+
current: input.current,
|
|
28310
|
+
previous: input.previous,
|
|
28311
|
+
absoluteChange: input.absolute,
|
|
28312
|
+
percentChange: input.percent,
|
|
28313
|
+
currentDate: input.dates.current,
|
|
28314
|
+
previousDate: input.dates.previous
|
|
28315
|
+
},
|
|
28316
|
+
interpretation: input.interpretation,
|
|
28317
|
+
verification: input.verification
|
|
28318
|
+
};
|
|
28319
|
+
}
|
|
28320
|
+
function buildFindings(input) {
|
|
28321
|
+
const findings = [];
|
|
28322
|
+
const traffic = input.traffic;
|
|
28323
|
+
if (traffic && traffic.current.observedDays === traffic.current.expectedDays && traffic.previous.observedDays === traffic.previous.expectedDays && (traffic.changes.clicksPercent !== null && Math.abs(traffic.changes.clicksPercent) >= 10 || traffic.changes.impressionsPercent !== null && Math.abs(traffic.changes.impressionsPercent) >= 10)) {
|
|
28324
|
+
const metric = traffic.changes.clicksPercent !== null && Math.abs(traffic.changes.clicksPercent) >= 10 ? "clicks" : "impressions";
|
|
28325
|
+
const declined = traffic.changes[metric] < 0;
|
|
28326
|
+
findings.push({
|
|
28327
|
+
code: `bing_${metric}_${declined ? "declined" : "increased"}`,
|
|
28328
|
+
severity: declined ? "warning" : "info",
|
|
28329
|
+
title: `Bing ${metric} ${declined ? "declined" : "increased"} in the latest complete period`,
|
|
28330
|
+
evidence: {
|
|
28331
|
+
currentClicks: traffic.current.clicks,
|
|
28332
|
+
previousClicks: traffic.previous.clicks,
|
|
28333
|
+
clicksChange: traffic.changes.clicks,
|
|
28334
|
+
clicksPercentChange: traffic.changes.clicksPercent,
|
|
28335
|
+
currentImpressions: traffic.current.impressions,
|
|
28336
|
+
previousImpressions: traffic.previous.impressions,
|
|
28337
|
+
impressionsChange: traffic.changes.impressions,
|
|
28338
|
+
impressionsPercentChange: traffic.changes.impressionsPercent,
|
|
28339
|
+
ctrPercentagePointChange: traffic.changes.ctrPercentagePoints,
|
|
28340
|
+
currentRange: {
|
|
28341
|
+
startDate: traffic.current.startDate,
|
|
28342
|
+
endDate: traffic.current.endDate
|
|
28343
|
+
},
|
|
28344
|
+
previousRange: {
|
|
28345
|
+
startDate: traffic.previous.startDate,
|
|
28346
|
+
endDate: traffic.previous.endDate
|
|
28347
|
+
}
|
|
28348
|
+
},
|
|
28349
|
+
interpretation: "This is a Bing-observed traffic change. It does not establish the cause or describe Google traffic.",
|
|
28350
|
+
verification: "Review the query and page evidence for the same dates, then compare current pages and crawl health before changing content."
|
|
28351
|
+
});
|
|
28352
|
+
}
|
|
28353
|
+
const crawl = input.crawl;
|
|
28354
|
+
if (crawl?.previous) {
|
|
28355
|
+
const dates = {
|
|
28356
|
+
current: crawl.current.date,
|
|
28357
|
+
previous: crawl.previous.date
|
|
28358
|
+
};
|
|
28359
|
+
const errors = crawl.changes.crawlErrors;
|
|
28360
|
+
if (errors && errors.absolute >= Math.max(10, Math.ceil(errors.previous * .5))) findings.push(changeFinding({
|
|
28361
|
+
code: "bing_crawl_errors_increased",
|
|
28362
|
+
title: "Bing crawl errors increased",
|
|
28363
|
+
...errors,
|
|
28364
|
+
dates,
|
|
28365
|
+
interpretation: "Bing reported more crawl errors in its latest daily snapshot. The count may include URLs outside the current sitemap.",
|
|
28366
|
+
verification: "Run the fast sitemap health check, inspect Bing crawl issue samples, then audit any affected URLs directly."
|
|
28367
|
+
}));
|
|
28368
|
+
const code4xx = crawl.changes.code4xx;
|
|
28369
|
+
if (code4xx && code4xx.absolute >= Math.max(10, Math.ceil(code4xx.previous * .5))) findings.push(changeFinding({
|
|
28370
|
+
code: "bing_4xx_increased",
|
|
28371
|
+
title: "Bing-observed 4xx responses increased",
|
|
28372
|
+
...code4xx,
|
|
28373
|
+
dates,
|
|
28374
|
+
interpretation: "Bing encountered more 4xx responses. Intentional removals and obsolete discovered URLs can contribute to this count.",
|
|
28375
|
+
verification: "Inspect Bing crawl issue URLs and recover or redirect only URLs that should still exist or retain search value."
|
|
28376
|
+
}));
|
|
28377
|
+
const code5xx = crawl.changes.code5xx;
|
|
28378
|
+
if (code5xx && code5xx.current > 0 && code5xx.absolute > 0) findings.push(changeFinding({
|
|
28379
|
+
code: "bing_5xx_increased",
|
|
28380
|
+
title: "Bing-observed server errors increased",
|
|
28381
|
+
...code5xx,
|
|
28382
|
+
dates,
|
|
28383
|
+
interpretation: "Bing encountered more server errors in the latest snapshot. Even a small increase deserves a live check.",
|
|
28384
|
+
verification: "Check current server logs and run targeted URL audits before treating the provider count as an active outage."
|
|
28385
|
+
}));
|
|
28386
|
+
}
|
|
28387
|
+
for (const [kind, analysis] of [["query", input.queries], ["page", input.pages]]) {
|
|
28388
|
+
if (!analysis?.opportunities.length) continue;
|
|
28389
|
+
findings.push({
|
|
28390
|
+
code: `bing_${kind}_opportunities`,
|
|
28391
|
+
severity: "review",
|
|
28392
|
+
title: `${analysis.opportunities.length} Bing ${kind} opportunities need review`,
|
|
28393
|
+
evidence: {
|
|
28394
|
+
retainedOpportunities: analysis.opportunities.length,
|
|
28395
|
+
currentDimensions: analysis.coverage.currentDimensions,
|
|
28396
|
+
matchedDimensions: analysis.coverage.matchedDimensions,
|
|
28397
|
+
comparableDimensions: analysis.coverage.comparableDimensions,
|
|
28398
|
+
thresholds: analysis.thresholds
|
|
28399
|
+
},
|
|
28400
|
+
interpretation: "These are high-impression top-list entries currently observed around positions 4 to 20. The threshold is a prioritization heuristic, not a search-engine rule.",
|
|
28401
|
+
verification: kind === "query" ? "Check intent and the ranking page before changing copy or creating content." : "Audit the selected page and its matching queries before changing internal links, titles, or content."
|
|
28402
|
+
});
|
|
28403
|
+
}
|
|
28404
|
+
return findings;
|
|
28405
|
+
}
|
|
28406
|
+
async function bingWebmasterOverview(input) {
|
|
28407
|
+
if (!input.site.trim()) throw new SeoError("INVALID_INPUT", "Pass a Bing Webmaster site URL.");
|
|
28408
|
+
const resolved = input.client ? {
|
|
28409
|
+
client: input.client,
|
|
28410
|
+
credentialSource: input.credentialSource ?? "environment"
|
|
28411
|
+
} : await createBingWebmasterClient();
|
|
28412
|
+
const observedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
28413
|
+
const topListWarning = "Bing returns weekly top-list rows, so dimensions missing from a period remain unknown.";
|
|
28414
|
+
const [traffic, crawl, queries, pages] = await Promise.all([
|
|
28415
|
+
section(() => resolved.client.getTraffic(input.site), { rowLimit: 400 }),
|
|
28416
|
+
section(() => resolved.client.getCrawlStats(input.site), { rowLimit: 400 }),
|
|
28417
|
+
section(() => resolved.client.getQueryStats(input.site), {
|
|
28418
|
+
rowLimit: 8e3,
|
|
28419
|
+
sourceWarning: topListWarning
|
|
28420
|
+
}),
|
|
28421
|
+
section(() => resolved.client.getPageStats(input.site), {
|
|
28422
|
+
rowLimit: 8e3,
|
|
28423
|
+
sourceWarning: topListWarning
|
|
28424
|
+
})
|
|
28425
|
+
]);
|
|
28426
|
+
const statuses = [
|
|
28427
|
+
traffic.status,
|
|
28428
|
+
crawl.status,
|
|
28429
|
+
queries.status,
|
|
28430
|
+
pages.status
|
|
28431
|
+
];
|
|
28432
|
+
const dataStatus = statuses.every((status) => status === "complete") ? "complete" : statuses.every((status) => status === "unavailable") ? "unavailable" : "partial";
|
|
28433
|
+
const trafficAnalysis = traffic.status === "unavailable" ? void 0 : analyzeBingTraffic(traffic.data.rows);
|
|
28434
|
+
const crawlAnalysis = crawl.status === "unavailable" ? void 0 : analyzeBingCrawl(crawl.data.rows);
|
|
28435
|
+
const queryAnalysis = queries.status === "unavailable" ? void 0 : analyzeBingDimensions(queries.data.rows, "query");
|
|
28436
|
+
const pageAnalysis = pages.status === "unavailable" ? void 0 : analyzeBingDimensions(pages.data.rows, "page");
|
|
28437
|
+
const findings = buildFindings({
|
|
28438
|
+
traffic: trafficAnalysis,
|
|
28439
|
+
crawl: crawlAnalysis,
|
|
28440
|
+
queries: queryAnalysis,
|
|
28441
|
+
pages: pageAnalysis
|
|
28442
|
+
});
|
|
28443
|
+
return {
|
|
28444
|
+
schemaVersion: 2,
|
|
28445
|
+
site: input.site,
|
|
28446
|
+
generatedAt: observedAt,
|
|
28447
|
+
dataStatus,
|
|
28448
|
+
summary: {
|
|
28449
|
+
findings: findings.length,
|
|
28450
|
+
warnings: findings.filter((finding) => finding.severity === "warning").length,
|
|
28451
|
+
reviewItems: findings.filter((finding) => finding.severity === "review").length
|
|
28452
|
+
},
|
|
28453
|
+
findings,
|
|
28454
|
+
outputBudget: {
|
|
28455
|
+
maxFindings: 6,
|
|
28456
|
+
maxDailyRowsPerSection: DAILY_OUTPUT_ROWS,
|
|
28457
|
+
maxDailyRowsTotal: DAILY_OUTPUT_ROWS * 2,
|
|
28458
|
+
maxDimensionItemsPerList: 10,
|
|
28459
|
+
maxDimensionItemsTotal: 40
|
|
28460
|
+
},
|
|
28461
|
+
provenance: {
|
|
28462
|
+
provider: "bing-webmaster",
|
|
28463
|
+
authentication: resolved.client.authentication,
|
|
28464
|
+
credentialSource: resolved.credentialSource,
|
|
28465
|
+
observedAt,
|
|
28466
|
+
cached: false,
|
|
28467
|
+
rowLimits: {
|
|
28468
|
+
traffic: 400,
|
|
28469
|
+
crawl: 400,
|
|
28470
|
+
queries: 8e3,
|
|
28471
|
+
pages: 8e3
|
|
28472
|
+
},
|
|
28473
|
+
methods: [
|
|
28474
|
+
"GetRankAndTrafficStats",
|
|
28475
|
+
"GetCrawlStats",
|
|
28476
|
+
"GetQueryStats",
|
|
28477
|
+
"GetPageStats"
|
|
28478
|
+
]
|
|
28479
|
+
},
|
|
28480
|
+
traffic: traffic.status === "unavailable" ? traffic : {
|
|
28481
|
+
...traffic,
|
|
28482
|
+
data: {
|
|
28483
|
+
...dailyOutput(traffic.data),
|
|
28484
|
+
range: range(traffic.data.rows),
|
|
28485
|
+
clicks: traffic.data.rows.reduce((total, row) => total + row.clicks, 0),
|
|
28486
|
+
impressions: traffic.data.rows.reduce((total, row) => total + row.impressions, 0),
|
|
28487
|
+
analysis: trafficAnalysis
|
|
28488
|
+
}
|
|
28489
|
+
},
|
|
28490
|
+
crawl: crawl.status === "unavailable" ? crawl : {
|
|
28491
|
+
...crawl,
|
|
28492
|
+
data: {
|
|
28493
|
+
...dailyOutput(crawl.data),
|
|
28494
|
+
range: range(crawl.data.rows),
|
|
28495
|
+
latest: crawl.data.rows.at(-1),
|
|
28496
|
+
analysis: crawlAnalysis
|
|
28497
|
+
}
|
|
28498
|
+
},
|
|
28499
|
+
queries: queries.status === "unavailable" ? queries : {
|
|
28500
|
+
...queries,
|
|
28501
|
+
data: {
|
|
28502
|
+
returnedRows: queries.data.returnedRows,
|
|
28503
|
+
retainedRows: queries.data.rows.length,
|
|
28504
|
+
invalidRows: queries.data.invalidRows,
|
|
28505
|
+
capped: queries.data.capped,
|
|
28506
|
+
range: range(queries.data.rows),
|
|
28507
|
+
analysis: queryAnalysis
|
|
28508
|
+
}
|
|
28509
|
+
},
|
|
28510
|
+
pages: pages.status === "unavailable" ? pages : {
|
|
28511
|
+
...pages,
|
|
28512
|
+
data: {
|
|
28513
|
+
returnedRows: pages.data.returnedRows,
|
|
28514
|
+
retainedRows: pages.data.rows.length,
|
|
28515
|
+
invalidRows: pages.data.invalidRows,
|
|
28516
|
+
capped: pages.data.capped,
|
|
28517
|
+
range: range(pages.data.rows),
|
|
28518
|
+
analysis: pageAnalysis
|
|
28519
|
+
}
|
|
28520
|
+
},
|
|
28521
|
+
caveats: [
|
|
28522
|
+
"Bing reports its own observed search and crawl evidence. It is not a complete view of every search engine.",
|
|
28523
|
+
"The inIndex field is provider-reported crawl statistics, not independent proof that a URL is indexed.",
|
|
28524
|
+
"Query and page movements include only dimensions observed in every weekly top list in both periods. Missing entries are not treated as zero.",
|
|
28525
|
+
"Opportunity thresholds prioritize review. They are heuristics, not search-engine requirements."
|
|
28526
|
+
]
|
|
28527
|
+
};
|
|
28528
|
+
}
|
|
28529
|
+
//#endregion
|
|
28530
|
+
//#region packages/core/dist/doctor.js
|
|
28531
|
+
const REQUIRED_SCOPES = ["https://www.googleapis.com/auth/webmasters.readonly", "https://www.googleapis.com/auth/analytics.readonly"];
|
|
28532
|
+
async function runDoctor(dependencies = {}) {
|
|
28533
|
+
const paths = getSeoCliPaths();
|
|
28534
|
+
const config = readConfig();
|
|
28535
|
+
const authMode = getAuthModeStatus();
|
|
28536
|
+
const serviceAccount = getServiceAccountStatus();
|
|
28537
|
+
const tokens = await readTokens();
|
|
28538
|
+
const oauthClient = readOauthClient();
|
|
28539
|
+
const tokenClientConfigured = tokens ? Boolean(getClientConfig(tokens.client_source)) : void 0;
|
|
28540
|
+
const checks = [];
|
|
28541
|
+
try {
|
|
28542
|
+
const database = (dependencies.checkDatabase ?? checkDatabaseReadiness)();
|
|
28543
|
+
checks.push({
|
|
28544
|
+
id: "local-database",
|
|
28545
|
+
label: "Local database",
|
|
28546
|
+
status: "pass",
|
|
28547
|
+
detail: database.dbPath
|
|
28548
|
+
});
|
|
28549
|
+
} catch (error) {
|
|
28550
|
+
checks.push({
|
|
28551
|
+
id: "local-database",
|
|
27292
28552
|
label: "Local database",
|
|
27293
28553
|
status: "fail",
|
|
27294
28554
|
detail: error instanceof Error ? error.message : "The local database could not be opened.",
|
|
@@ -27343,9 +28603,504 @@ async function runDoctor(dependencies = {}) {
|
|
|
27343
28603
|
detail: config.analytics.google.defaultPropertyId ?? "No saved default. Human CLI commands will prompt; agents can pass --property."
|
|
27344
28604
|
});
|
|
27345
28605
|
return {
|
|
27346
|
-
ok: checks.every((check) => check.status !== "fail"),
|
|
28606
|
+
ok: checks.every((check) => check.status !== "fail"),
|
|
28607
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
28608
|
+
checks
|
|
28609
|
+
};
|
|
28610
|
+
}
|
|
28611
|
+
//#endregion
|
|
28612
|
+
//#region packages/core/dist/links/normalize.js
|
|
28613
|
+
const FIELD_ALIASES = {
|
|
28614
|
+
sourceUrl: [
|
|
28615
|
+
"sourceurl",
|
|
28616
|
+
"source",
|
|
28617
|
+
"referringurl",
|
|
28618
|
+
"referringpage",
|
|
28619
|
+
"fromurl"
|
|
28620
|
+
],
|
|
28621
|
+
targetUrl: [
|
|
28622
|
+
"targeturl",
|
|
28623
|
+
"target",
|
|
28624
|
+
"destinationurl",
|
|
28625
|
+
"destination",
|
|
28626
|
+
"tourl"
|
|
28627
|
+
],
|
|
28628
|
+
anchorText: ["anchortext", "anchor"],
|
|
28629
|
+
firstSeenAt: ["firstseenat", "firstseen"],
|
|
28630
|
+
lastSeenAt: ["lastseenat", "lastseen"],
|
|
28631
|
+
nofollow: ["nofollow", "isnofollow"]
|
|
28632
|
+
};
|
|
28633
|
+
function key(value) {
|
|
28634
|
+
return value.toLowerCase().replaceAll(/[^a-z0-9]/g, "");
|
|
28635
|
+
}
|
|
28636
|
+
function valueFor(row, field) {
|
|
28637
|
+
const aliases = new Set(FIELD_ALIASES[field]);
|
|
28638
|
+
for (const [name, value] of Object.entries(row)) if (aliases.has(key(name))) return value;
|
|
28639
|
+
}
|
|
28640
|
+
function httpUrl(value) {
|
|
28641
|
+
if (typeof value !== "string" || !value.trim() || value.trim().length > 2e3) return;
|
|
28642
|
+
try {
|
|
28643
|
+
const url = new URL(value.trim());
|
|
28644
|
+
return url.protocol === "http:" || url.protocol === "https:" ? url : void 0;
|
|
28645
|
+
} catch {
|
|
28646
|
+
return;
|
|
28647
|
+
}
|
|
28648
|
+
}
|
|
28649
|
+
function optionalString(value) {
|
|
28650
|
+
return typeof value === "string" && value.trim() ? value.trim().slice(0, 1e3) : void 0;
|
|
28651
|
+
}
|
|
28652
|
+
function optionalBoolean(value) {
|
|
28653
|
+
if (typeof value === "boolean") return value;
|
|
28654
|
+
if (typeof value !== "string") return void 0;
|
|
28655
|
+
if (/^(true|yes|1)$/i.test(value.trim())) return true;
|
|
28656
|
+
if (/^(false|no|0)$/i.test(value.trim())) return false;
|
|
28657
|
+
}
|
|
28658
|
+
function normalizeLinkEvidenceRow(raw) {
|
|
28659
|
+
const source = httpUrl(valueFor(raw, "sourceUrl"));
|
|
28660
|
+
const target = httpUrl(valueFor(raw, "targetUrl"));
|
|
28661
|
+
if (!source || !target) return void 0;
|
|
28662
|
+
return {
|
|
28663
|
+
sourceUrl: source.toString(),
|
|
28664
|
+
sourceDomain: source.hostname.toLowerCase(),
|
|
28665
|
+
targetUrl: target.toString(),
|
|
28666
|
+
anchorText: optionalString(valueFor(raw, "anchorText")),
|
|
28667
|
+
firstSeenAt: optionalString(valueFor(raw, "firstSeenAt")),
|
|
28668
|
+
lastSeenAt: optionalString(valueFor(raw, "lastSeenAt")),
|
|
28669
|
+
nofollow: optionalBoolean(valueFor(raw, "nofollow"))
|
|
28670
|
+
};
|
|
28671
|
+
}
|
|
28672
|
+
function linkEvidenceKey(row) {
|
|
28673
|
+
return `${row.sourceUrl}\u0000${row.targetUrl}\u0000${row.anchorText ?? ""}`;
|
|
28674
|
+
}
|
|
28675
|
+
function compareLinkEvidence(a, b) {
|
|
28676
|
+
return a.targetUrl.localeCompare(b.targetUrl, "en") || a.sourceUrl.localeCompare(b.sourceUrl, "en") || (a.anchorText ?? "").localeCompare(b.anchorText ?? "", "en");
|
|
28677
|
+
}
|
|
28678
|
+
//#endregion
|
|
28679
|
+
//#region packages/core/dist/links/bing.js
|
|
28680
|
+
const MAX_COUNT_PAGES = 5;
|
|
28681
|
+
const DEFAULT_TARGET_LIMIT = 20;
|
|
28682
|
+
const MAX_TARGET_LIMIT = 50;
|
|
28683
|
+
const DEFAULT_DETAIL_PAGES = 1;
|
|
28684
|
+
const MAX_DETAIL_PAGES = 3;
|
|
28685
|
+
const DEFAULT_ROW_LIMIT$2 = 500;
|
|
28686
|
+
const MAX_ROW_LIMIT$2 = 1e3;
|
|
28687
|
+
const MAX_CONCURRENCY = 3;
|
|
28688
|
+
function integer(value, fallback, max, name) {
|
|
28689
|
+
const resolved = value ?? fallback;
|
|
28690
|
+
if (!Number.isInteger(resolved) || resolved < 1 || resolved > max) throw new SeoError("INVALID_INPUT", `${name} must be between 1 and ${max}.`);
|
|
28691
|
+
return resolved;
|
|
28692
|
+
}
|
|
28693
|
+
async function mapConcurrent(values, concurrency, work) {
|
|
28694
|
+
const output = new Array(values.length);
|
|
28695
|
+
let next = 0;
|
|
28696
|
+
async function worker() {
|
|
28697
|
+
while (next < values.length) {
|
|
28698
|
+
const index = next;
|
|
28699
|
+
next += 1;
|
|
28700
|
+
const value = values[index];
|
|
28701
|
+
if (value !== void 0) output[index] = await work(value);
|
|
28702
|
+
}
|
|
28703
|
+
}
|
|
28704
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, () => worker()));
|
|
28705
|
+
return output;
|
|
28706
|
+
}
|
|
28707
|
+
async function targetDetails(input) {
|
|
28708
|
+
const rows = [];
|
|
28709
|
+
let requestedPages = 0;
|
|
28710
|
+
let invalidRows = 0;
|
|
28711
|
+
let returnedRows = 0;
|
|
28712
|
+
let partial = false;
|
|
28713
|
+
let totalPages = 0;
|
|
28714
|
+
for (let page = 0; page < input.pages; page += 1) {
|
|
28715
|
+
requestedPages += 1;
|
|
28716
|
+
let response;
|
|
28717
|
+
try {
|
|
28718
|
+
response = await input.client.getUrlLinks(input.site, input.target.url, page);
|
|
28719
|
+
} catch (error) {
|
|
28720
|
+
if (error instanceof SeoError && (error.code === "AUTH_REQUIRED" || error.code === "ACCESS_DENIED")) throw error;
|
|
28721
|
+
return {
|
|
28722
|
+
rows,
|
|
28723
|
+
requestedPages,
|
|
28724
|
+
invalidRows,
|
|
28725
|
+
returnedRows,
|
|
28726
|
+
partial: true,
|
|
28727
|
+
failed: true
|
|
28728
|
+
};
|
|
28729
|
+
}
|
|
28730
|
+
invalidRows += response.invalidRows;
|
|
28731
|
+
returnedRows += response.returnedRows;
|
|
28732
|
+
rows.push(...response.rows);
|
|
28733
|
+
totalPages = response.totalPages;
|
|
28734
|
+
partial ||= response.capped;
|
|
28735
|
+
if (page + 1 >= response.totalPages || response.rows.length === 0) break;
|
|
28736
|
+
}
|
|
28737
|
+
partial ||= requestedPages < totalPages;
|
|
28738
|
+
return {
|
|
28739
|
+
rows,
|
|
28740
|
+
requestedPages,
|
|
28741
|
+
invalidRows,
|
|
28742
|
+
returnedRows,
|
|
28743
|
+
partial,
|
|
28744
|
+
failed: false
|
|
28745
|
+
};
|
|
28746
|
+
}
|
|
28747
|
+
async function collectBingLinkEvidence(input) {
|
|
28748
|
+
if (!input.site.trim()) throw new SeoError("INVALID_INPUT", "Pass a Bing Webmaster site URL.");
|
|
28749
|
+
const targetLimit = integer(input.targetLimit, DEFAULT_TARGET_LIMIT, MAX_TARGET_LIMIT, "Bing link target limit");
|
|
28750
|
+
const detailPages = integer(input.detailPagesPerTarget, DEFAULT_DETAIL_PAGES, MAX_DETAIL_PAGES, "Bing detail page limit");
|
|
28751
|
+
const rowLimit = integer(input.rowLimit, DEFAULT_ROW_LIMIT$2, MAX_ROW_LIMIT$2, "Bing link row limit");
|
|
28752
|
+
const client = input.client ?? (await createBingWebmasterClient()).client;
|
|
28753
|
+
const countRows = [];
|
|
28754
|
+
let countPagesRequested = 0;
|
|
28755
|
+
let invalidRows = 0;
|
|
28756
|
+
let countInvalidRows = 0;
|
|
28757
|
+
let countRowsReturned = 0;
|
|
28758
|
+
let partial = false;
|
|
28759
|
+
let totalCountPages = 0;
|
|
28760
|
+
for (let page = 0; page < MAX_COUNT_PAGES; page += 1) {
|
|
28761
|
+
const response = await client.getLinkCounts(input.site, page);
|
|
28762
|
+
countPagesRequested += 1;
|
|
28763
|
+
invalidRows += response.invalidRows;
|
|
28764
|
+
countInvalidRows += response.invalidRows;
|
|
28765
|
+
countRowsReturned += response.returnedRows;
|
|
28766
|
+
countRows.push(...response.rows);
|
|
28767
|
+
totalCountPages = response.totalPages;
|
|
28768
|
+
partial ||= response.capped;
|
|
28769
|
+
if (countRows.length >= targetLimit || page + 1 >= response.totalPages || response.rows.length === 0) break;
|
|
28770
|
+
}
|
|
28771
|
+
partial ||= countPagesRequested < totalCountPages;
|
|
28772
|
+
const countsByUrl = /* @__PURE__ */ new Map();
|
|
28773
|
+
for (const row of countRows) countsByUrl.set(row.url, Math.max(row.count, countsByUrl.get(row.url) ?? 0));
|
|
28774
|
+
const counts = [...countsByUrl].map(([url, count]) => ({
|
|
28775
|
+
url,
|
|
28776
|
+
count
|
|
28777
|
+
}));
|
|
28778
|
+
counts.sort((a, b) => b.count - a.count || a.url.localeCompare(b.url, "en"));
|
|
28779
|
+
const selectedTargets = counts.slice(0, targetLimit);
|
|
28780
|
+
partial ||= counts.length > targetLimit;
|
|
28781
|
+
const details = await mapConcurrent(selectedTargets, MAX_CONCURRENCY, async (target) => targetDetails({
|
|
28782
|
+
client,
|
|
28783
|
+
site: input.site,
|
|
28784
|
+
target,
|
|
28785
|
+
pages: detailPages
|
|
28786
|
+
}));
|
|
28787
|
+
const rows = [];
|
|
28788
|
+
const keys = /* @__PURE__ */ new Set();
|
|
28789
|
+
let duplicateRows = 0;
|
|
28790
|
+
let detailPagesRequested = 0;
|
|
28791
|
+
let detailInvalidRows = 0;
|
|
28792
|
+
let detailRowsReturned = 0;
|
|
28793
|
+
let failedTargets = 0;
|
|
28794
|
+
for (const [index, detail] of details.entries()) {
|
|
28795
|
+
const target = selectedTargets[index];
|
|
28796
|
+
if (!target) continue;
|
|
28797
|
+
detailPagesRequested += detail.requestedPages;
|
|
28798
|
+
if (detail.failed) failedTargets += 1;
|
|
28799
|
+
invalidRows += detail.invalidRows;
|
|
28800
|
+
detailInvalidRows += detail.invalidRows;
|
|
28801
|
+
detailRowsReturned += detail.returnedRows;
|
|
28802
|
+
partial ||= detail.partial;
|
|
28803
|
+
for (const source of detail.rows) {
|
|
28804
|
+
if (rows.length >= rowLimit) {
|
|
28805
|
+
partial = true;
|
|
28806
|
+
break;
|
|
28807
|
+
}
|
|
28808
|
+
const row = normalizeLinkEvidenceRow({
|
|
28809
|
+
sourceUrl: source.url,
|
|
28810
|
+
targetUrl: target.url,
|
|
28811
|
+
anchorText: source.anchorText
|
|
28812
|
+
});
|
|
28813
|
+
if (!row) {
|
|
28814
|
+
invalidRows += 1;
|
|
28815
|
+
continue;
|
|
28816
|
+
}
|
|
28817
|
+
const key = linkEvidenceKey(row);
|
|
28818
|
+
if (keys.has(key)) {
|
|
28819
|
+
duplicateRows += 1;
|
|
28820
|
+
continue;
|
|
28821
|
+
}
|
|
28822
|
+
keys.add(key);
|
|
28823
|
+
rows.push(row);
|
|
28824
|
+
}
|
|
28825
|
+
}
|
|
28826
|
+
rows.sort(compareLinkEvidence);
|
|
28827
|
+
return {
|
|
28828
|
+
rows,
|
|
28829
|
+
targetCounts: selectedTargets.map((target) => ({
|
|
28830
|
+
targetUrl: target.url,
|
|
28831
|
+
providerReportedLinks: target.count,
|
|
28832
|
+
observedLinks: rows.filter((row) => row.targetUrl === target.url).length
|
|
28833
|
+
})),
|
|
28834
|
+
provenance: {
|
|
28835
|
+
provider: "bing-webmaster",
|
|
28836
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
28837
|
+
cached: false,
|
|
28838
|
+
suppliedRows: detailRowsReturned + detailInvalidRows,
|
|
28839
|
+
validRows: rows.length,
|
|
28840
|
+
invalidRows,
|
|
28841
|
+
duplicateRows,
|
|
28842
|
+
capped: partial,
|
|
28843
|
+
rowLimit,
|
|
28844
|
+
completeness: partial ? "partial" : "unknown",
|
|
28845
|
+
providerRequests: {
|
|
28846
|
+
methods: ["GetLinkCounts", "GetUrlLinks"],
|
|
28847
|
+
targetPagesRequested: countPagesRequested,
|
|
28848
|
+
detailPagesRequested,
|
|
28849
|
+
maxConcurrentRequests: MAX_CONCURRENCY
|
|
28850
|
+
},
|
|
28851
|
+
providerCoverage: {
|
|
28852
|
+
targetCountRows: {
|
|
28853
|
+
returnedRows: countRowsReturned,
|
|
28854
|
+
retainedRows: counts.length,
|
|
28855
|
+
invalidRows: countInvalidRows
|
|
28856
|
+
},
|
|
28857
|
+
detailRows: {
|
|
28858
|
+
returnedRows: detailRowsReturned,
|
|
28859
|
+
retainedRows: rows.length,
|
|
28860
|
+
invalidRows: detailInvalidRows
|
|
28861
|
+
}
|
|
28862
|
+
}
|
|
28863
|
+
},
|
|
28864
|
+
warnings: [
|
|
28865
|
+
...countRowsReturned === 0 ? ["Bing returned no target-link rows. This does not prove that the site has no links."] : [],
|
|
28866
|
+
...partial ? ["Bing link evidence was bounded by provider pagination, failed requests, or report limits."] : [],
|
|
28867
|
+
...failedTargets ? [`${failedTargets} target detail request${failedTargets === 1 ? "" : "s"} failed while other evidence was retained.`] : []
|
|
28868
|
+
]
|
|
28869
|
+
};
|
|
28870
|
+
}
|
|
28871
|
+
//#endregion
|
|
28872
|
+
//#region packages/core/dist/links/import.js
|
|
28873
|
+
const DEFAULT_ROW_LIMIT$1 = 1e4;
|
|
28874
|
+
const MAX_ROW_LIMIT$1 = 1e5;
|
|
28875
|
+
const MAX_STREAM_BYTES = 5e7;
|
|
28876
|
+
const MAX_JSON_BYTES = 1e7;
|
|
28877
|
+
const MAX_CELL_BYTES = 1e6;
|
|
28878
|
+
function formatFor$1(path, explicit) {
|
|
28879
|
+
if (explicit) return explicit;
|
|
28880
|
+
const extension = extname(path).toLowerCase();
|
|
28881
|
+
if (extension === ".csv") return "csv";
|
|
28882
|
+
if (extension === ".jsonl" || extension === ".ndjson") return "jsonl";
|
|
28883
|
+
if (extension === ".json") return "json";
|
|
28884
|
+
throw new SeoError("INVALID_INPUT", "Link imports must be CSV, JSON, JSONL, or NDJSON.");
|
|
28885
|
+
}
|
|
28886
|
+
function boundedRowLimit(value) {
|
|
28887
|
+
const limit = value ?? DEFAULT_ROW_LIMIT$1;
|
|
28888
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_ROW_LIMIT$1) throw new SeoError("INVALID_INPUT", `Link import row limit must be between 1 and ${MAX_ROW_LIMIT$1}.`);
|
|
28889
|
+
return limit;
|
|
28890
|
+
}
|
|
28891
|
+
async function* csvRecords(path, byteLimit, progress) {
|
|
28892
|
+
const stream = createReadStream(path, { encoding: "utf8" });
|
|
28893
|
+
let record = [];
|
|
28894
|
+
let field = "";
|
|
28895
|
+
let quoted = false;
|
|
28896
|
+
let pendingQuote = false;
|
|
28897
|
+
for await (const chunk of stream) {
|
|
28898
|
+
progress.bytesRead += Buffer.byteLength(chunk);
|
|
28899
|
+
if (progress.bytesRead > byteLimit) {
|
|
28900
|
+
stream.destroy();
|
|
28901
|
+
throw new SeoError("INVALID_INPUT", `Link import exceeds the ${byteLimit}-byte streaming limit.`);
|
|
28902
|
+
}
|
|
28903
|
+
for (const character of chunk) {
|
|
28904
|
+
if (pendingQuote) {
|
|
28905
|
+
if (character === "\"") {
|
|
28906
|
+
field += "\"";
|
|
28907
|
+
pendingQuote = false;
|
|
28908
|
+
continue;
|
|
28909
|
+
}
|
|
28910
|
+
quoted = false;
|
|
28911
|
+
pendingQuote = false;
|
|
28912
|
+
}
|
|
28913
|
+
if (quoted) if (character === "\"") pendingQuote = true;
|
|
28914
|
+
else field += character;
|
|
28915
|
+
else if (character === "\"" && field.length === 0) quoted = true;
|
|
28916
|
+
else if (character === ",") {
|
|
28917
|
+
record.push(field);
|
|
28918
|
+
field = "";
|
|
28919
|
+
} else if (character === "\n") {
|
|
28920
|
+
record.push(field.replace(/\r$/, ""));
|
|
28921
|
+
field = "";
|
|
28922
|
+
if (record.some((value) => value.length > 0)) yield record;
|
|
28923
|
+
record = [];
|
|
28924
|
+
} else field += character;
|
|
28925
|
+
if (Buffer.byteLength(field) > MAX_CELL_BYTES) {
|
|
28926
|
+
stream.destroy();
|
|
28927
|
+
throw new SeoError("INVALID_INPUT", `Link import contains a cell larger than ${MAX_CELL_BYTES} bytes.`);
|
|
28928
|
+
}
|
|
28929
|
+
}
|
|
28930
|
+
}
|
|
28931
|
+
if (quoted && !pendingQuote) throw new SeoError("INVALID_INPUT", "Link CSV contains an unclosed quote.");
|
|
28932
|
+
if (field || record.length) {
|
|
28933
|
+
record.push(field.replace(/\r$/, ""));
|
|
28934
|
+
if (record.some((value) => value.length > 0)) yield record;
|
|
28935
|
+
}
|
|
28936
|
+
}
|
|
28937
|
+
async function* jsonlRecords(path, byteLimit, progress) {
|
|
28938
|
+
const stream = createReadStream(path, { encoding: "utf8" });
|
|
28939
|
+
let remainder = "";
|
|
28940
|
+
for await (const chunk of stream) {
|
|
28941
|
+
progress.bytesRead += Buffer.byteLength(chunk);
|
|
28942
|
+
if (progress.bytesRead > byteLimit) {
|
|
28943
|
+
stream.destroy();
|
|
28944
|
+
throw new SeoError("INVALID_INPUT", `Link import exceeds the ${byteLimit}-byte streaming limit.`);
|
|
28945
|
+
}
|
|
28946
|
+
remainder += chunk;
|
|
28947
|
+
let newline = remainder.indexOf("\n");
|
|
28948
|
+
while (newline >= 0) {
|
|
28949
|
+
const line = remainder.slice(0, newline).trim();
|
|
28950
|
+
remainder = remainder.slice(newline + 1);
|
|
28951
|
+
if (line) yield parseObject(line);
|
|
28952
|
+
newline = remainder.indexOf("\n");
|
|
28953
|
+
}
|
|
28954
|
+
if (Buffer.byteLength(remainder) > MAX_CELL_BYTES) {
|
|
28955
|
+
stream.destroy();
|
|
28956
|
+
throw new SeoError("INVALID_INPUT", `Link import contains a line larger than ${MAX_CELL_BYTES} bytes.`);
|
|
28957
|
+
}
|
|
28958
|
+
}
|
|
28959
|
+
if (remainder.trim()) yield parseObject(remainder);
|
|
28960
|
+
}
|
|
28961
|
+
function parseObject(value) {
|
|
28962
|
+
try {
|
|
28963
|
+
const parsed = JSON.parse(value);
|
|
28964
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("not an object");
|
|
28965
|
+
return parsed;
|
|
28966
|
+
} catch {
|
|
28967
|
+
throw new SeoError("INVALID_INPUT", "Link JSONL must contain one JSON object per line.");
|
|
28968
|
+
}
|
|
28969
|
+
}
|
|
28970
|
+
function rowFromCsv(headers, values) {
|
|
28971
|
+
return Object.fromEntries(headers.map((header, index) => [header, values[index]]));
|
|
28972
|
+
}
|
|
28973
|
+
async function importLinkEvidence(input) {
|
|
28974
|
+
const path = resolve(input.file);
|
|
28975
|
+
const file = await stat(path).catch(() => void 0);
|
|
28976
|
+
if (!file?.isFile()) throw new SeoError("INVALID_INPUT", `Link import file was not found: ${path}`);
|
|
28977
|
+
const format = formatFor$1(path, input.format);
|
|
28978
|
+
const rowLimit = boundedRowLimit(input.rowLimit);
|
|
28979
|
+
if (format === "json" && file.size > MAX_JSON_BYTES) throw new SeoError("INVALID_INPUT", `JSON link imports are limited to ${MAX_JSON_BYTES} bytes. Use CSV or JSONL for larger files.`);
|
|
28980
|
+
const progress = { bytesRead: 0 };
|
|
28981
|
+
let records;
|
|
28982
|
+
if (format === "csv") {
|
|
28983
|
+
const csv = csvRecords(path, MAX_STREAM_BYTES, progress);
|
|
28984
|
+
const first = await csv.next();
|
|
28985
|
+
const headers = (Array.isArray(first.value) ? first.value : []).map((value, index) => index === 0 ? value.replace(/^\uFEFF/, "") : value);
|
|
28986
|
+
if (!headers.length) throw new SeoError("INVALID_INPUT", "Link CSV has no header row.");
|
|
28987
|
+
records = (async function* () {
|
|
28988
|
+
for await (const values of csv) yield rowFromCsv(headers, values);
|
|
28989
|
+
})();
|
|
28990
|
+
} else if (format === "jsonl") records = jsonlRecords(path, MAX_STREAM_BYTES, progress);
|
|
28991
|
+
else {
|
|
28992
|
+
const body = await readFile(path, "utf8");
|
|
28993
|
+
progress.bytesRead = Buffer.byteLength(body);
|
|
28994
|
+
let parsed;
|
|
28995
|
+
try {
|
|
28996
|
+
parsed = JSON.parse(body);
|
|
28997
|
+
} catch {
|
|
28998
|
+
throw new SeoError("INVALID_INPUT", "Link JSON is not valid JSON.");
|
|
28999
|
+
}
|
|
29000
|
+
if (!Array.isArray(parsed)) throw new SeoError("INVALID_INPUT", "Link JSON must be an array of objects.");
|
|
29001
|
+
records = (async function* () {
|
|
29002
|
+
for (const row of parsed) yield row && typeof row === "object" && !Array.isArray(row) ? row : {};
|
|
29003
|
+
})();
|
|
29004
|
+
}
|
|
29005
|
+
const rows = [];
|
|
29006
|
+
const keys = /* @__PURE__ */ new Set();
|
|
29007
|
+
let suppliedRows = 0;
|
|
29008
|
+
let invalidRows = 0;
|
|
29009
|
+
let duplicateRows = 0;
|
|
29010
|
+
let capped = false;
|
|
29011
|
+
for await (const raw of records) {
|
|
29012
|
+
if (suppliedRows >= rowLimit) {
|
|
29013
|
+
capped = true;
|
|
29014
|
+
break;
|
|
29015
|
+
}
|
|
29016
|
+
suppliedRows += 1;
|
|
29017
|
+
const row = normalizeLinkEvidenceRow(raw);
|
|
29018
|
+
if (!row) {
|
|
29019
|
+
invalidRows += 1;
|
|
29020
|
+
continue;
|
|
29021
|
+
}
|
|
29022
|
+
const key = linkEvidenceKey(row);
|
|
29023
|
+
if (keys.has(key)) {
|
|
29024
|
+
duplicateRows += 1;
|
|
29025
|
+
continue;
|
|
29026
|
+
}
|
|
29027
|
+
keys.add(key);
|
|
29028
|
+
rows.push(row);
|
|
29029
|
+
}
|
|
29030
|
+
rows.sort(compareLinkEvidence);
|
|
29031
|
+
const observedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
29032
|
+
return {
|
|
29033
|
+
rows,
|
|
29034
|
+
targetCounts: [],
|
|
29035
|
+
provenance: {
|
|
29036
|
+
provider: `${format}-import`,
|
|
29037
|
+
observedAt,
|
|
29038
|
+
cached: false,
|
|
29039
|
+
suppliedRows,
|
|
29040
|
+
validRows: rows.length,
|
|
29041
|
+
invalidRows,
|
|
29042
|
+
duplicateRows,
|
|
29043
|
+
capped,
|
|
29044
|
+
rowLimit,
|
|
29045
|
+
completeness: capped ? "partial" : "unknown",
|
|
29046
|
+
file: {
|
|
29047
|
+
path,
|
|
29048
|
+
format,
|
|
29049
|
+
bytesRead: progress.bytesRead,
|
|
29050
|
+
fileBytes: file.size
|
|
29051
|
+
}
|
|
29052
|
+
},
|
|
29053
|
+
warnings: capped ? [`Import stopped after ${rowLimit} rows. The file contains more data.`] : []
|
|
29054
|
+
};
|
|
29055
|
+
}
|
|
29056
|
+
//#endregion
|
|
29057
|
+
//#region packages/core/dist/links/report.js
|
|
29058
|
+
const DEFAULT_OUTPUT_LIMIT = 100;
|
|
29059
|
+
const MAX_OUTPUT_LIMIT = 500;
|
|
29060
|
+
function boundedLimit(value) {
|
|
29061
|
+
const limit = value ?? DEFAULT_OUTPUT_LIMIT;
|
|
29062
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_OUTPUT_LIMIT) throw new SeoError("INVALID_INPUT", `Link report limit must be between 1 and ${MAX_OUTPUT_LIMIT}.`);
|
|
29063
|
+
return limit;
|
|
29064
|
+
}
|
|
29065
|
+
function targetCounts(rows) {
|
|
29066
|
+
const counts = /* @__PURE__ */ new Map();
|
|
29067
|
+
for (const row of rows) counts.set(row.targetUrl, (counts.get(row.targetUrl) ?? 0) + 1);
|
|
29068
|
+
return [...counts].map(([targetUrl, observedLinks]) => ({
|
|
29069
|
+
targetUrl,
|
|
29070
|
+
observedLinks
|
|
29071
|
+
})).sort((a, b) => b.observedLinks - a.observedLinks || a.targetUrl.localeCompare(b.targetUrl, "en"));
|
|
29072
|
+
}
|
|
29073
|
+
function linkEvidenceReport(input) {
|
|
29074
|
+
const limit = boundedLimit(input.limit);
|
|
29075
|
+
const rows = input.evidence.rows.slice(0, limit);
|
|
29076
|
+
const domains = new Set(input.evidence.rows.map((row) => row.sourceDomain));
|
|
29077
|
+
const observedTargets = targetCounts(input.evidence.rows);
|
|
29078
|
+
const counts = input.evidence.targetCounts.length ? input.evidence.targetCounts : observedTargets;
|
|
29079
|
+
return {
|
|
29080
|
+
schemaVersion: 1,
|
|
27347
29081
|
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
27348
|
-
|
|
29082
|
+
dataStatus: input.evidence.provenance.completeness === "complete" ? "complete" : "partial",
|
|
29083
|
+
summary: {
|
|
29084
|
+
observedLinks: input.evidence.rows.length,
|
|
29085
|
+
referringDomains: domains.size,
|
|
29086
|
+
targetPages: new Set(input.evidence.rows.map((row) => row.targetUrl)).size,
|
|
29087
|
+
providerTargetPages: counts.length
|
|
29088
|
+
},
|
|
29089
|
+
provenance: input.evidence.provenance,
|
|
29090
|
+
selection: {
|
|
29091
|
+
availableRows: input.evidence.rows.length,
|
|
29092
|
+
returnedRows: rows.length,
|
|
29093
|
+
omittedRows: Math.max(0, input.evidence.rows.length - rows.length),
|
|
29094
|
+
limit
|
|
29095
|
+
},
|
|
29096
|
+
targetCounts: counts.slice(0, 100),
|
|
29097
|
+
links: rows,
|
|
29098
|
+
warnings: input.evidence.warnings,
|
|
29099
|
+
caveats: [
|
|
29100
|
+
"This is bounded provider or imported evidence, not a complete backlink index.",
|
|
29101
|
+
"A missing link is not proof that no link exists, and an observed link is not a quality or ranking verdict.",
|
|
29102
|
+
"Referring-domain counts describe the retained rows only."
|
|
29103
|
+
]
|
|
27349
29104
|
};
|
|
27350
29105
|
}
|
|
27351
29106
|
//#endregion
|
|
@@ -27603,6 +29358,472 @@ var LocalCredentialsProvider = class {
|
|
|
27603
29358
|
}
|
|
27604
29359
|
};
|
|
27605
29360
|
new LocalCredentialsProvider();
|
|
29361
|
+
//#endregion
|
|
29362
|
+
//#region packages/core/dist/server-logs/parse.js
|
|
29363
|
+
const MONTHS = {
|
|
29364
|
+
Jan: 0,
|
|
29365
|
+
Feb: 1,
|
|
29366
|
+
Mar: 2,
|
|
29367
|
+
Apr: 3,
|
|
29368
|
+
May: 4,
|
|
29369
|
+
Jun: 5,
|
|
29370
|
+
Jul: 6,
|
|
29371
|
+
Aug: 7,
|
|
29372
|
+
Sep: 8,
|
|
29373
|
+
Oct: 9,
|
|
29374
|
+
Nov: 10,
|
|
29375
|
+
Dec: 11
|
|
29376
|
+
};
|
|
29377
|
+
const CRAWLERS = [
|
|
29378
|
+
{
|
|
29379
|
+
family: "Googlebot",
|
|
29380
|
+
category: "search",
|
|
29381
|
+
patterns: ["googlebot", "google-inspectiontool"]
|
|
29382
|
+
},
|
|
29383
|
+
{
|
|
29384
|
+
family: "Bingbot",
|
|
29385
|
+
category: "search",
|
|
29386
|
+
patterns: ["bingbot", "bingpreview"]
|
|
29387
|
+
},
|
|
29388
|
+
{
|
|
29389
|
+
family: "Applebot",
|
|
29390
|
+
category: "search",
|
|
29391
|
+
patterns: ["applebot"]
|
|
29392
|
+
},
|
|
29393
|
+
{
|
|
29394
|
+
family: "DuckDuckBot",
|
|
29395
|
+
category: "search",
|
|
29396
|
+
patterns: ["duckduckbot"]
|
|
29397
|
+
},
|
|
29398
|
+
{
|
|
29399
|
+
family: "YandexBot",
|
|
29400
|
+
category: "search",
|
|
29401
|
+
patterns: ["yandexbot"]
|
|
29402
|
+
},
|
|
29403
|
+
{
|
|
29404
|
+
family: "Baiduspider",
|
|
29405
|
+
category: "search",
|
|
29406
|
+
patterns: ["baiduspider"]
|
|
29407
|
+
},
|
|
29408
|
+
{
|
|
29409
|
+
family: "PetalBot",
|
|
29410
|
+
category: "search",
|
|
29411
|
+
patterns: ["petalbot"]
|
|
29412
|
+
},
|
|
29413
|
+
{
|
|
29414
|
+
family: "OpenAI",
|
|
29415
|
+
category: "ai",
|
|
29416
|
+
patterns: [
|
|
29417
|
+
"gptbot",
|
|
29418
|
+
"chatgpt-user",
|
|
29419
|
+
"oai-searchbot"
|
|
29420
|
+
]
|
|
29421
|
+
},
|
|
29422
|
+
{
|
|
29423
|
+
family: "Anthropic",
|
|
29424
|
+
category: "ai",
|
|
29425
|
+
patterns: [
|
|
29426
|
+
"claudebot",
|
|
29427
|
+
"claude-searchbot",
|
|
29428
|
+
"claude-user"
|
|
29429
|
+
]
|
|
29430
|
+
},
|
|
29431
|
+
{
|
|
29432
|
+
family: "Perplexity",
|
|
29433
|
+
category: "ai",
|
|
29434
|
+
patterns: ["perplexitybot", "perplexity-user"]
|
|
29435
|
+
},
|
|
29436
|
+
{
|
|
29437
|
+
family: "Amazonbot",
|
|
29438
|
+
category: "ai",
|
|
29439
|
+
patterns: ["amazonbot"]
|
|
29440
|
+
},
|
|
29441
|
+
{
|
|
29442
|
+
family: "Bytespider",
|
|
29443
|
+
category: "ai",
|
|
29444
|
+
patterns: ["bytespider"]
|
|
29445
|
+
},
|
|
29446
|
+
{
|
|
29447
|
+
family: "CCBot",
|
|
29448
|
+
category: "ai",
|
|
29449
|
+
patterns: ["ccbot"]
|
|
29450
|
+
},
|
|
29451
|
+
{
|
|
29452
|
+
family: "Cohere",
|
|
29453
|
+
category: "ai",
|
|
29454
|
+
patterns: ["cohere-ai"]
|
|
29455
|
+
}
|
|
29456
|
+
];
|
|
29457
|
+
const COMBINED_LOG = /^\S+\s+\S+\s+\S+\s+\[([^\]]+)]\s+"([^"\s]+)\s+([^"\s]+)(?:\s+[^"\s]+)?"\s+(\d{3})\s+(\d+|-)\s*(?:"[^"]*"\s+"([^"]*)")?\s*$/;
|
|
29458
|
+
function classifyCrawler(userAgent) {
|
|
29459
|
+
if (!userAgent) return void 0;
|
|
29460
|
+
const value = userAgent.toLowerCase();
|
|
29461
|
+
const match = CRAWLERS.find((crawler) => crawler.patterns.some((pattern) => value.includes(pattern)));
|
|
29462
|
+
return match ? {
|
|
29463
|
+
family: match.family,
|
|
29464
|
+
category: match.category
|
|
29465
|
+
} : void 0;
|
|
29466
|
+
}
|
|
29467
|
+
function logTimestamp(value) {
|
|
29468
|
+
const match = /^(\d{2})\/([A-Z][a-z]{2})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})$/.exec(value);
|
|
29469
|
+
if (!match) return void 0;
|
|
29470
|
+
const month = MONTHS[match[2] ?? ""];
|
|
29471
|
+
if (month === void 0) return void 0;
|
|
29472
|
+
const utc = Date.UTC(Number(match[3]), month, Number(match[1]), Number(match[4]), Number(match[5]), Number(match[6]));
|
|
29473
|
+
const offsetMinutes = Number(match[8]) * 60 + Number(match[9]);
|
|
29474
|
+
const adjusted = utc + (match[7] === "+" ? -1 : 1) * offsetMinutes * 6e4;
|
|
29475
|
+
const date = new Date(adjusted);
|
|
29476
|
+
return Number.isNaN(date.getTime()) ? void 0 : date.toISOString();
|
|
29477
|
+
}
|
|
29478
|
+
function normalizedPath(value) {
|
|
29479
|
+
if (typeof value !== "string" || value.length < 1 || value.length > 2e3) return;
|
|
29480
|
+
try {
|
|
29481
|
+
if (value.startsWith("/")) return new URL(value, "https://server-log.invalid").pathname;
|
|
29482
|
+
const url = new URL(value);
|
|
29483
|
+
return ["http:", "https:"].includes(url.protocol) ? url.pathname : void 0;
|
|
29484
|
+
} catch {
|
|
29485
|
+
return;
|
|
29486
|
+
}
|
|
29487
|
+
}
|
|
29488
|
+
function normalizedMethod(value) {
|
|
29489
|
+
if (typeof value !== "string" || !/^[A-Za-z]{1,16}$/.test(value)) return;
|
|
29490
|
+
return value.toUpperCase();
|
|
29491
|
+
}
|
|
29492
|
+
function normalizedStatus(value) {
|
|
29493
|
+
const status = typeof value === "number" ? value : Number(value);
|
|
29494
|
+
return Number.isInteger(status) && status >= 100 && status <= 599 ? status : void 0;
|
|
29495
|
+
}
|
|
29496
|
+
function normalizedBytes(value) {
|
|
29497
|
+
if (value === void 0 || value === null || value === "" || value === "-") return;
|
|
29498
|
+
const bytes = typeof value === "number" ? value : Number(value);
|
|
29499
|
+
return Number.isSafeInteger(bytes) && bytes >= 0 ? bytes : void 0;
|
|
29500
|
+
}
|
|
29501
|
+
function record(input) {
|
|
29502
|
+
let timestamp;
|
|
29503
|
+
if (typeof input.timestamp === "string") {
|
|
29504
|
+
const date = new Date(input.timestamp);
|
|
29505
|
+
if (!Number.isNaN(date.getTime())) timestamp = date.toISOString();
|
|
29506
|
+
}
|
|
29507
|
+
const method = normalizedMethod(input.method);
|
|
29508
|
+
const path = normalizedPath(input.path);
|
|
29509
|
+
const status = normalizedStatus(input.status);
|
|
29510
|
+
if (!timestamp || !method || !path || status === void 0) return void 0;
|
|
29511
|
+
const userAgent = typeof input.userAgent === "string" && input.userAgent.length <= 2e3 ? input.userAgent : void 0;
|
|
29512
|
+
return {
|
|
29513
|
+
timestamp,
|
|
29514
|
+
method,
|
|
29515
|
+
path,
|
|
29516
|
+
status,
|
|
29517
|
+
bytes: normalizedBytes(input.bytes),
|
|
29518
|
+
userAgent,
|
|
29519
|
+
crawler: classifyCrawler(userAgent)
|
|
29520
|
+
};
|
|
29521
|
+
}
|
|
29522
|
+
function parseCombinedLogLine(line) {
|
|
29523
|
+
const match = COMBINED_LOG.exec(line);
|
|
29524
|
+
if (!match) return void 0;
|
|
29525
|
+
const timestamp = logTimestamp(match[1] ?? "");
|
|
29526
|
+
if (!timestamp) return void 0;
|
|
29527
|
+
return record({
|
|
29528
|
+
timestamp,
|
|
29529
|
+
method: match[2],
|
|
29530
|
+
path: match[3],
|
|
29531
|
+
status: match[4],
|
|
29532
|
+
bytes: match[5],
|
|
29533
|
+
userAgent: match[6]
|
|
29534
|
+
});
|
|
29535
|
+
}
|
|
29536
|
+
function firstValue(value, names) {
|
|
29537
|
+
for (const name of names) if (value[name] !== void 0) return value[name];
|
|
29538
|
+
}
|
|
29539
|
+
function parseJsonLogLine(line) {
|
|
29540
|
+
let value;
|
|
29541
|
+
try {
|
|
29542
|
+
value = JSON.parse(line);
|
|
29543
|
+
} catch {
|
|
29544
|
+
return;
|
|
29545
|
+
}
|
|
29546
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return;
|
|
29547
|
+
const row = value;
|
|
29548
|
+
return record({
|
|
29549
|
+
timestamp: firstValue(row, [
|
|
29550
|
+
"timestamp",
|
|
29551
|
+
"time",
|
|
29552
|
+
"datetime",
|
|
29553
|
+
"@timestamp"
|
|
29554
|
+
]),
|
|
29555
|
+
method: firstValue(row, [
|
|
29556
|
+
"method",
|
|
29557
|
+
"requestMethod",
|
|
29558
|
+
"http_method"
|
|
29559
|
+
]),
|
|
29560
|
+
path: firstValue(row, [
|
|
29561
|
+
"path",
|
|
29562
|
+
"url",
|
|
29563
|
+
"requestUri",
|
|
29564
|
+
"request_uri"
|
|
29565
|
+
]),
|
|
29566
|
+
status: firstValue(row, [
|
|
29567
|
+
"status",
|
|
29568
|
+
"statusCode",
|
|
29569
|
+
"status_code"
|
|
29570
|
+
]),
|
|
29571
|
+
bytes: firstValue(row, [
|
|
29572
|
+
"bytes",
|
|
29573
|
+
"bodyBytesSent",
|
|
29574
|
+
"body_bytes_sent"
|
|
29575
|
+
]),
|
|
29576
|
+
userAgent: firstValue(row, [
|
|
29577
|
+
"userAgent",
|
|
29578
|
+
"user_agent",
|
|
29579
|
+
"httpUserAgent",
|
|
29580
|
+
"http_user_agent"
|
|
29581
|
+
])
|
|
29582
|
+
});
|
|
29583
|
+
}
|
|
29584
|
+
function emptyStatusBreakdown() {
|
|
29585
|
+
return {
|
|
29586
|
+
success: 0,
|
|
29587
|
+
redirect: 0,
|
|
29588
|
+
clientError: 0,
|
|
29589
|
+
serverError: 0,
|
|
29590
|
+
other: 0
|
|
29591
|
+
};
|
|
29592
|
+
}
|
|
29593
|
+
function addStatus(breakdown, status) {
|
|
29594
|
+
if (status >= 200 && status < 300) breakdown.success += 1;
|
|
29595
|
+
else if (status >= 300 && status < 400) breakdown.redirect += 1;
|
|
29596
|
+
else if (status >= 400 && status < 500) breakdown.clientError += 1;
|
|
29597
|
+
else if (status >= 500) breakdown.serverError += 1;
|
|
29598
|
+
else breakdown.other += 1;
|
|
29599
|
+
}
|
|
29600
|
+
//#endregion
|
|
29601
|
+
//#region packages/core/dist/server-logs/import.js
|
|
29602
|
+
const DEFAULT_ROW_LIMIT = 1e6;
|
|
29603
|
+
const MAX_ROW_LIMIT = 1e7;
|
|
29604
|
+
const DEFAULT_PATH_LIMIT = 25e3;
|
|
29605
|
+
const MAX_PATH_LIMIT = 1e5;
|
|
29606
|
+
const BYTE_LIMIT = 1e9;
|
|
29607
|
+
const MAX_LINE_BYTES = 65536;
|
|
29608
|
+
function boundedInteger(value, fallback, maximum, label) {
|
|
29609
|
+
const result = value ?? fallback;
|
|
29610
|
+
if (!Number.isInteger(result) || result < 1 || result > maximum) throw new SeoError("INVALID_INPUT", `${label} must be between 1 and ${maximum}.`);
|
|
29611
|
+
return result;
|
|
29612
|
+
}
|
|
29613
|
+
function formatFor(path, explicit) {
|
|
29614
|
+
if (explicit) return explicit;
|
|
29615
|
+
const extension = extname(path).toLowerCase();
|
|
29616
|
+
return [".jsonl", ".ndjson"].includes(extension) ? "jsonl" : "combined";
|
|
29617
|
+
}
|
|
29618
|
+
async function* lines(path, progress) {
|
|
29619
|
+
const stream = createReadStream(path, {
|
|
29620
|
+
encoding: "utf8",
|
|
29621
|
+
highWaterMark: 64 * 1024
|
|
29622
|
+
});
|
|
29623
|
+
let remainder = "";
|
|
29624
|
+
let discarding = false;
|
|
29625
|
+
for await (const chunk of stream) {
|
|
29626
|
+
const remainingBytes = BYTE_LIMIT - progress.bytesRead;
|
|
29627
|
+
if (remainingBytes <= 0) {
|
|
29628
|
+
progress.bytesCapped = true;
|
|
29629
|
+
break;
|
|
29630
|
+
}
|
|
29631
|
+
const chunkBytes = Buffer.byteLength(chunk);
|
|
29632
|
+
const accepted = chunkBytes > remainingBytes ? Buffer.from(chunk).subarray(0, remainingBytes).toString("utf8") : chunk;
|
|
29633
|
+
progress.bytesRead += Buffer.byteLength(accepted);
|
|
29634
|
+
if (chunkBytes > remainingBytes) progress.bytesCapped = true;
|
|
29635
|
+
remainder += accepted;
|
|
29636
|
+
let newline = remainder.indexOf("\n");
|
|
29637
|
+
while (newline >= 0) {
|
|
29638
|
+
const line = remainder.slice(0, newline).replace(/\r$/, "");
|
|
29639
|
+
remainder = remainder.slice(newline + 1);
|
|
29640
|
+
if (discarding || Buffer.byteLength(line) > MAX_LINE_BYTES) {
|
|
29641
|
+
yield { tooLong: true };
|
|
29642
|
+
discarding = false;
|
|
29643
|
+
} else if (line.trim()) yield { line };
|
|
29644
|
+
newline = remainder.indexOf("\n");
|
|
29645
|
+
}
|
|
29646
|
+
if (Buffer.byteLength(remainder) > MAX_LINE_BYTES) {
|
|
29647
|
+
remainder = "";
|
|
29648
|
+
discarding = true;
|
|
29649
|
+
}
|
|
29650
|
+
if (progress.bytesCapped) break;
|
|
29651
|
+
}
|
|
29652
|
+
if (discarding || Buffer.byteLength(remainder) > MAX_LINE_BYTES) yield { tooLong: true };
|
|
29653
|
+
else if (remainder.trim()) yield { line: remainder.replace(/\r$/, "") };
|
|
29654
|
+
}
|
|
29655
|
+
function compareCrawler(a, b) {
|
|
29656
|
+
return b.requests - a.requests || a.family.localeCompare(b.family, "en");
|
|
29657
|
+
}
|
|
29658
|
+
function compareCrawlerPath(a, b) {
|
|
29659
|
+
return b.requests - a.requests || a.family.localeCompare(b.family, "en") || a.path.localeCompare(b.path, "en");
|
|
29660
|
+
}
|
|
29661
|
+
function updatePeriod(timestamp, period) {
|
|
29662
|
+
if (!period.firstSeenAt || timestamp < period.firstSeenAt) period.firstSeenAt = timestamp;
|
|
29663
|
+
if (!period.lastSeenAt || timestamp > period.lastSeenAt) period.lastSeenAt = timestamp;
|
|
29664
|
+
}
|
|
29665
|
+
function newCrawler(record) {
|
|
29666
|
+
const crawler = record.crawler;
|
|
29667
|
+
if (!crawler) throw new Error("Crawler evidence is required.");
|
|
29668
|
+
return {
|
|
29669
|
+
family: crawler.family,
|
|
29670
|
+
category: crawler.category,
|
|
29671
|
+
requests: 0,
|
|
29672
|
+
lastSeenAt: record.timestamp,
|
|
29673
|
+
...emptyStatusBreakdown()
|
|
29674
|
+
};
|
|
29675
|
+
}
|
|
29676
|
+
function newCrawlerPath(record) {
|
|
29677
|
+
return {
|
|
29678
|
+
...newCrawler(record),
|
|
29679
|
+
path: record.path
|
|
29680
|
+
};
|
|
29681
|
+
}
|
|
29682
|
+
async function importServerLog(input) {
|
|
29683
|
+
const path = resolve(input.file);
|
|
29684
|
+
const file = await stat(path).catch(() => void 0);
|
|
29685
|
+
if (!file?.isFile()) throw new SeoError("INVALID_INPUT", `Server log was not found: ${path}`);
|
|
29686
|
+
const format = formatFor(path, input.format);
|
|
29687
|
+
const rowLimit = boundedInteger(input.rowLimit, DEFAULT_ROW_LIMIT, MAX_ROW_LIMIT, "Server log row limit");
|
|
29688
|
+
const pathLimit = boundedInteger(input.pathLimit, DEFAULT_PATH_LIMIT, MAX_PATH_LIMIT, "Server log path limit");
|
|
29689
|
+
const parse = format === "jsonl" ? parseJsonLogLine : parseCombinedLogLine;
|
|
29690
|
+
const progress = {
|
|
29691
|
+
bytesRead: 0,
|
|
29692
|
+
bytesCapped: false
|
|
29693
|
+
};
|
|
29694
|
+
const period = {};
|
|
29695
|
+
const statusCodes = /* @__PURE__ */ new Map();
|
|
29696
|
+
const crawlers = /* @__PURE__ */ new Map();
|
|
29697
|
+
const crawlerPaths = /* @__PURE__ */ new Map();
|
|
29698
|
+
let suppliedRows = 0;
|
|
29699
|
+
let parsedRows = 0;
|
|
29700
|
+
let invalidRows = 0;
|
|
29701
|
+
let crawlerRows = 0;
|
|
29702
|
+
let responseBytes = 0;
|
|
29703
|
+
let rowsCapped = false;
|
|
29704
|
+
let pathsCapped = false;
|
|
29705
|
+
let untrackedCrawlerPathRows = 0;
|
|
29706
|
+
for await (const item of lines(path, progress)) {
|
|
29707
|
+
if (suppliedRows >= rowLimit) {
|
|
29708
|
+
rowsCapped = true;
|
|
29709
|
+
break;
|
|
29710
|
+
}
|
|
29711
|
+
suppliedRows += 1;
|
|
29712
|
+
const parsed = item.line ? parse(item.line) : void 0;
|
|
29713
|
+
if (!parsed) {
|
|
29714
|
+
invalidRows += 1;
|
|
29715
|
+
continue;
|
|
29716
|
+
}
|
|
29717
|
+
parsedRows += 1;
|
|
29718
|
+
responseBytes += parsed.bytes ?? 0;
|
|
29719
|
+
updatePeriod(parsed.timestamp, period);
|
|
29720
|
+
statusCodes.set(parsed.status, (statusCodes.get(parsed.status) ?? 0) + 1);
|
|
29721
|
+
if (!parsed.crawler) continue;
|
|
29722
|
+
crawlerRows += 1;
|
|
29723
|
+
const crawler = crawlers.get(parsed.crawler.family) ?? newCrawler(parsed);
|
|
29724
|
+
crawler.requests += 1;
|
|
29725
|
+
if (parsed.timestamp > crawler.lastSeenAt) crawler.lastSeenAt = parsed.timestamp;
|
|
29726
|
+
addStatus(crawler, parsed.status);
|
|
29727
|
+
crawlers.set(parsed.crawler.family, crawler);
|
|
29728
|
+
const key = `${parsed.crawler.family}\u0000${parsed.path}`;
|
|
29729
|
+
let crawlerPath = crawlerPaths.get(key);
|
|
29730
|
+
if (!crawlerPath && crawlerPaths.size >= pathLimit) {
|
|
29731
|
+
pathsCapped = true;
|
|
29732
|
+
untrackedCrawlerPathRows += 1;
|
|
29733
|
+
continue;
|
|
29734
|
+
}
|
|
29735
|
+
crawlerPath ??= newCrawlerPath(parsed);
|
|
29736
|
+
crawlerPath.requests += 1;
|
|
29737
|
+
if (parsed.timestamp > crawlerPath.lastSeenAt) crawlerPath.lastSeenAt = parsed.timestamp;
|
|
29738
|
+
addStatus(crawlerPath, parsed.status);
|
|
29739
|
+
crawlerPaths.set(key, crawlerPath);
|
|
29740
|
+
}
|
|
29741
|
+
const fileReadCompletely = !rowsCapped && !progress.bytesCapped && progress.bytesRead >= file.size;
|
|
29742
|
+
const completeness = fileReadCompletely && invalidRows === 0 && !pathsCapped ? "complete" : "partial";
|
|
29743
|
+
const warnings = [];
|
|
29744
|
+
if (rowsCapped) warnings.push(`Analysis stopped after ${rowLimit} rows.`);
|
|
29745
|
+
if (progress.bytesCapped) warnings.push(`Analysis stopped after ${BYTE_LIMIT} input bytes.`);
|
|
29746
|
+
if (invalidRows) warnings.push(`${invalidRows} malformed or unsupported rows were skipped.`);
|
|
29747
|
+
if (pathsCapped) warnings.push(`Crawler path tracking stopped at ${pathLimit} unique crawler and path pairs. Overall crawler and status totals still include later valid rows.`);
|
|
29748
|
+
return {
|
|
29749
|
+
summary: {
|
|
29750
|
+
suppliedRows,
|
|
29751
|
+
parsedRows,
|
|
29752
|
+
invalidRows,
|
|
29753
|
+
crawlerRows,
|
|
29754
|
+
nonCrawlerRows: parsedRows - crawlerRows,
|
|
29755
|
+
responseBytes,
|
|
29756
|
+
...period
|
|
29757
|
+
},
|
|
29758
|
+
statusCodes: [...statusCodes].map(([status, requests]) => ({
|
|
29759
|
+
status,
|
|
29760
|
+
requests
|
|
29761
|
+
})).sort((a, b) => a.status - b.status),
|
|
29762
|
+
crawlers: [...crawlers.values()].sort(compareCrawler),
|
|
29763
|
+
crawlerPaths: [...crawlerPaths.values()].sort(compareCrawlerPath),
|
|
29764
|
+
provenance: {
|
|
29765
|
+
source: "local-server-log",
|
|
29766
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
29767
|
+
cached: false,
|
|
29768
|
+
file: {
|
|
29769
|
+
path,
|
|
29770
|
+
format,
|
|
29771
|
+
bytesRead: progress.bytesRead,
|
|
29772
|
+
fileBytes: file.size
|
|
29773
|
+
},
|
|
29774
|
+
limits: {
|
|
29775
|
+
rowLimit,
|
|
29776
|
+
pathLimit,
|
|
29777
|
+
byteLimit: BYTE_LIMIT,
|
|
29778
|
+
maxLineBytes: MAX_LINE_BYTES
|
|
29779
|
+
},
|
|
29780
|
+
coverage: {
|
|
29781
|
+
fileReadCompletely,
|
|
29782
|
+
rowsCapped,
|
|
29783
|
+
bytesCapped: progress.bytesCapped,
|
|
29784
|
+
pathsCapped,
|
|
29785
|
+
untrackedCrawlerPathRows
|
|
29786
|
+
},
|
|
29787
|
+
completeness
|
|
29788
|
+
},
|
|
29789
|
+
warnings
|
|
29790
|
+
};
|
|
29791
|
+
}
|
|
29792
|
+
//#endregion
|
|
29793
|
+
//#region packages/core/dist/server-logs/report.js
|
|
29794
|
+
const DEFAULT_LIMIT = 100;
|
|
29795
|
+
const MAX_LIMIT = 500;
|
|
29796
|
+
function outputLimit(value) {
|
|
29797
|
+
const limit = value ?? DEFAULT_LIMIT;
|
|
29798
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) throw new SeoError("INVALID_INPUT", `Server log output limit must be between 1 and ${MAX_LIMIT}.`);
|
|
29799
|
+
return limit;
|
|
29800
|
+
}
|
|
29801
|
+
function serverLogReport(input) {
|
|
29802
|
+
const limit = outputLimit(input.limit);
|
|
29803
|
+
const crawlerPaths = input.evidence.crawlerPaths.slice(0, limit);
|
|
29804
|
+
return {
|
|
29805
|
+
schemaVersion: 1,
|
|
29806
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
29807
|
+
dataStatus: input.evidence.provenance.completeness,
|
|
29808
|
+
summary: input.evidence.summary,
|
|
29809
|
+
statusCodes: input.evidence.statusCodes,
|
|
29810
|
+
crawlers: input.evidence.crawlers,
|
|
29811
|
+
crawlerPaths,
|
|
29812
|
+
selection: {
|
|
29813
|
+
availableCrawlerPaths: input.evidence.crawlerPaths.length,
|
|
29814
|
+
returnedCrawlerPaths: crawlerPaths.length,
|
|
29815
|
+
omittedCrawlerPaths: Math.max(0, input.evidence.crawlerPaths.length - crawlerPaths.length),
|
|
29816
|
+
limit
|
|
29817
|
+
},
|
|
29818
|
+
provenance: input.evidence.provenance,
|
|
29819
|
+
warnings: input.evidence.warnings,
|
|
29820
|
+
caveats: [
|
|
29821
|
+
"Crawler families are identified from user-agent strings, which can be missing or spoofed.",
|
|
29822
|
+
"This report describes only the supplied log file and cannot prove that every server or time period is represented.",
|
|
29823
|
+
"Crawler path rankings describe retained path aggregates when the path limit is reached."
|
|
29824
|
+
]
|
|
29825
|
+
};
|
|
29826
|
+
}
|
|
27606
29827
|
const TELEMETRY_TIMEOUT_MS = 2e3;
|
|
27607
29828
|
const TELEMETRY_REPORTS = [
|
|
27608
29829
|
"affected-urls",
|
|
@@ -27612,6 +29833,7 @@ const TELEMETRY_REPORTS = [
|
|
|
27612
29833
|
"ai-search-scorecard",
|
|
27613
29834
|
"audit-page",
|
|
27614
29835
|
"audit-urls",
|
|
29836
|
+
"bing-webmaster-overview",
|
|
27615
29837
|
"cannibalisation",
|
|
27616
29838
|
"community-intent",
|
|
27617
29839
|
"compare-crawls",
|
|
@@ -27630,6 +29852,8 @@ const TELEMETRY_REPORTS = [
|
|
|
27630
29852
|
"index-monitor",
|
|
27631
29853
|
"index-watch",
|
|
27632
29854
|
"internal-links",
|
|
29855
|
+
"link-evidence",
|
|
29856
|
+
"server-log-analysis",
|
|
27633
29857
|
"link-recovery",
|
|
27634
29858
|
"crawl-history",
|
|
27635
29859
|
"crawler-rules",
|
|
@@ -29602,6 +31826,87 @@ function registerOpportunityTools(server, dependencies = {}) {
|
|
|
29602
31826
|
});
|
|
29603
31827
|
}
|
|
29604
31828
|
//#endregion
|
|
31829
|
+
//#region packages/mcp/src/provider-tools.ts
|
|
31830
|
+
const MAX_AGENT_ROWS_PER_SECTION = 14;
|
|
31831
|
+
function compactRows(data) {
|
|
31832
|
+
if (data.rows.length <= MAX_AGENT_ROWS_PER_SECTION && data.outputSelection) return data;
|
|
31833
|
+
const rows = data.rows.slice(-14);
|
|
31834
|
+
return {
|
|
31835
|
+
...data,
|
|
31836
|
+
rows,
|
|
31837
|
+
outputSelection: {
|
|
31838
|
+
strategy: "most-recent",
|
|
31839
|
+
availableRows: data.rows.length,
|
|
31840
|
+
returnedRows: rows.length,
|
|
31841
|
+
omittedRows: Math.max(0, data.rows.length - rows.length)
|
|
31842
|
+
}
|
|
31843
|
+
};
|
|
31844
|
+
}
|
|
31845
|
+
function compactBingWebmasterOverview(result) {
|
|
31846
|
+
return {
|
|
31847
|
+
...result,
|
|
31848
|
+
traffic: result.traffic.status === "unavailable" ? result.traffic : {
|
|
31849
|
+
...result.traffic,
|
|
31850
|
+
data: compactRows(result.traffic.data)
|
|
31851
|
+
},
|
|
31852
|
+
crawl: result.crawl.status === "unavailable" ? result.crawl : {
|
|
31853
|
+
...result.crawl,
|
|
31854
|
+
data: compactRows(result.crawl.data)
|
|
31855
|
+
},
|
|
31856
|
+
outputBudget: result.outputBudget
|
|
31857
|
+
};
|
|
31858
|
+
}
|
|
31859
|
+
function registerProviderTools(server) {
|
|
31860
|
+
server.registerTool("seo_bing_webmaster_overview", {
|
|
31861
|
+
description: "Find bounded Bing traffic, crawl, query, and page insights for one verified site",
|
|
31862
|
+
inputSchema: { site: z$1.string().url().max(2e3) }
|
|
31863
|
+
}, async ({ site }) => {
|
|
31864
|
+
try {
|
|
31865
|
+
const result = await bingWebmasterOverview({ site });
|
|
31866
|
+
const firstFinding = result.findings[0]?.title;
|
|
31867
|
+
return toolSuccess(`Bing evidence is ${result.dataStatus} with ${result.summary.findings} findings.${firstFinding ? ` Review first: ${firstFinding}.` : ""}`, compactBingWebmasterOverview(result));
|
|
31868
|
+
} catch (error) {
|
|
31869
|
+
return toolError(error);
|
|
31870
|
+
}
|
|
31871
|
+
});
|
|
31872
|
+
server.registerTool("seo_link_evidence", {
|
|
31873
|
+
description: "Review bounded referring-link evidence from Bing Webmaster or a local export",
|
|
31874
|
+
inputSchema: {
|
|
31875
|
+
site: z$1.string().url().max(2e3).optional(),
|
|
31876
|
+
file: z$1.string().min(1).max(4096).optional(),
|
|
31877
|
+
format: z$1.enum([
|
|
31878
|
+
"csv",
|
|
31879
|
+
"json",
|
|
31880
|
+
"jsonl"
|
|
31881
|
+
]).optional(),
|
|
31882
|
+
rowLimit: z$1.number().int().min(1).max(1e5).optional(),
|
|
31883
|
+
targetLimit: z$1.number().int().min(1).max(50).optional(),
|
|
31884
|
+
detailPagesPerTarget: z$1.number().int().min(1).max(3).optional(),
|
|
31885
|
+
limit: z$1.number().int().min(1).max(500).optional()
|
|
31886
|
+
}
|
|
31887
|
+
}, async ({ site, file, format, rowLimit, targetLimit, detailPagesPerTarget, limit }) => {
|
|
31888
|
+
try {
|
|
31889
|
+
if (Boolean(site) === Boolean(file)) throw new SeoError("INVALID_INPUT", "Pass exactly one of site for Bing or file for a local import.");
|
|
31890
|
+
const report = linkEvidenceReport({
|
|
31891
|
+
evidence: file ? await importLinkEvidence({
|
|
31892
|
+
file,
|
|
31893
|
+
format,
|
|
31894
|
+
rowLimit
|
|
31895
|
+
}) : await collectBingLinkEvidence({
|
|
31896
|
+
site: site ?? "",
|
|
31897
|
+
rowLimit,
|
|
31898
|
+
targetLimit,
|
|
31899
|
+
detailPagesPerTarget
|
|
31900
|
+
}),
|
|
31901
|
+
limit
|
|
31902
|
+
});
|
|
31903
|
+
return toolSuccess(`${report.summary.observedLinks} referring links were retained from ${report.provenance.provider}.`, report);
|
|
31904
|
+
} catch (error) {
|
|
31905
|
+
return toolError(error);
|
|
31906
|
+
}
|
|
31907
|
+
});
|
|
31908
|
+
}
|
|
31909
|
+
//#endregion
|
|
29605
31910
|
//#region packages/mcp/src/pseo-tools.ts
|
|
29606
31911
|
function compactPseoReport(result) {
|
|
29607
31912
|
return {
|
|
@@ -29894,6 +32199,91 @@ const REPORT_DEPTH = {
|
|
|
29894
32199
|
}
|
|
29895
32200
|
]
|
|
29896
32201
|
},
|
|
32202
|
+
"bing-webmaster-overview": {
|
|
32203
|
+
readOrder: [
|
|
32204
|
+
"dataStatus and provenance",
|
|
32205
|
+
"summary and findings, including verification steps",
|
|
32206
|
+
"traffic analysis current and previous periods, then changes",
|
|
32207
|
+
"crawl analysis current and previous snapshots, then changes",
|
|
32208
|
+
"query and page coverage before movements or opportunities",
|
|
32209
|
+
"outputBudget, per-section outputSelection, and caveats"
|
|
32210
|
+
],
|
|
32211
|
+
doNotClaim: [
|
|
32212
|
+
"Bing evidence does not describe Google or the whole search market, and inIndex is not URL-level proof of indexing.",
|
|
32213
|
+
"A missing weekly top-list entry is not zero traffic, and movements require observation in every week of both periods.",
|
|
32214
|
+
"Opportunity thresholds prioritize review and are not Bing requirements.",
|
|
32215
|
+
"Partial, capped, invalid, or unavailable rows cannot support an all-clear."
|
|
32216
|
+
],
|
|
32217
|
+
verify: "Check the same verified site in Bing Webmaster Tools and investigate the exact dates behind any crawl or traffic change.",
|
|
32218
|
+
related: [
|
|
32219
|
+
{
|
|
32220
|
+
id: "search-performance-overview",
|
|
32221
|
+
reason: "Adds Search Console evidence."
|
|
32222
|
+
},
|
|
32223
|
+
{
|
|
32224
|
+
id: "site-crawl",
|
|
32225
|
+
reason: "Checks the current site directly."
|
|
32226
|
+
},
|
|
32227
|
+
{
|
|
32228
|
+
id: "link-recovery",
|
|
32229
|
+
reason: "Finds broken search-value URLs."
|
|
32230
|
+
}
|
|
32231
|
+
]
|
|
32232
|
+
},
|
|
32233
|
+
"link-evidence": {
|
|
32234
|
+
readOrder: [
|
|
32235
|
+
"dataStatus, provenance, warnings, and caveats",
|
|
32236
|
+
"selection and summary coverage counts",
|
|
32237
|
+
"targetCounts, then the bounded links list"
|
|
32238
|
+
],
|
|
32239
|
+
doNotClaim: [
|
|
32240
|
+
"The retained rows are not a complete backlink index.",
|
|
32241
|
+
"Missing rows do not prove a link does not exist.",
|
|
32242
|
+
"Observed links do not establish quality, authority, value, or ranking impact."
|
|
32243
|
+
],
|
|
32244
|
+
verify: "Open a selected referring URL and confirm the link is still present before acting on it.",
|
|
32245
|
+
related: [
|
|
32246
|
+
{
|
|
32247
|
+
id: "link-recovery",
|
|
32248
|
+
reason: "Finds broken search-value URLs."
|
|
32249
|
+
},
|
|
32250
|
+
{
|
|
32251
|
+
id: "redirect-trace",
|
|
32252
|
+
reason: "Checks a target redirect chain."
|
|
32253
|
+
},
|
|
32254
|
+
{
|
|
32255
|
+
id: "site-crawl",
|
|
32256
|
+
reason: "Checks internal links and live pages."
|
|
32257
|
+
}
|
|
32258
|
+
]
|
|
32259
|
+
},
|
|
32260
|
+
"server-log-analysis": {
|
|
32261
|
+
readOrder: [
|
|
32262
|
+
"dataStatus, provenance coverage, warnings, and caveats",
|
|
32263
|
+
"summary and statusCodes",
|
|
32264
|
+
"crawlers, then the bounded crawlerPaths list and selection"
|
|
32265
|
+
],
|
|
32266
|
+
doNotClaim: [
|
|
32267
|
+
"A user-agent match does not verify the crawler identity or source IP.",
|
|
32268
|
+
"The supplied file does not prove that every server or time period is represented.",
|
|
32269
|
+
"Path rankings are partial when path tracking reached its limit."
|
|
32270
|
+
],
|
|
32271
|
+
verify: "Check important crawler errors against the original log line and deployment or server configuration before changing the site.",
|
|
32272
|
+
related: [
|
|
32273
|
+
{
|
|
32274
|
+
id: "site-crawl",
|
|
32275
|
+
reason: "Checks the current site directly."
|
|
32276
|
+
},
|
|
32277
|
+
{
|
|
32278
|
+
id: "crawler-rules",
|
|
32279
|
+
reason: "Explains current crawl checks."
|
|
32280
|
+
},
|
|
32281
|
+
{
|
|
32282
|
+
id: "redirect-trace",
|
|
32283
|
+
reason: "Traces one failing or redirecting path."
|
|
32284
|
+
}
|
|
32285
|
+
]
|
|
32286
|
+
},
|
|
29897
32287
|
cannibalisation: {
|
|
29898
32288
|
readOrder: [
|
|
29899
32289
|
"dataStatus and both source.pageExposure and source.propertyDemand with validation counts and completeness",
|
|
@@ -31177,6 +33567,27 @@ const REPORT_GUIDANCE = {
|
|
|
31177
33567
|
avoidWhen: ["You need the crawler to discover pages for you.", "You only need a deep review of one page."],
|
|
31178
33568
|
outcome: "Consistent technical checks for each supplied URL."
|
|
31179
33569
|
},
|
|
33570
|
+
"bing-webmaster-overview": {
|
|
33571
|
+
name: "Bing Webmaster overview",
|
|
33572
|
+
description: "Review bounded Bing traffic trends, crawl changes, and query and page opportunities for one verified site.",
|
|
33573
|
+
useWhen: ["You want processed search and crawl insights observed by Bing.", "You need a second search-engine view beside Search Console."],
|
|
33574
|
+
avoidWhen: ["Bing Webmaster is not connected for this installation.", "You need URL-level backlink rows or proof that a page is indexed."],
|
|
33575
|
+
outcome: "Prioritized Bing findings with traffic comparisons, crawl changes, query and page opportunities, provider coverage, and caveats."
|
|
33576
|
+
},
|
|
33577
|
+
"link-evidence": {
|
|
33578
|
+
name: "Referring link evidence",
|
|
33579
|
+
description: "Review a bounded set of pages that link to the site from Bing Webmaster or a local export.",
|
|
33580
|
+
useWhen: ["You want concrete referring URLs and anchor text where available.", "You need one normalized view of Bing or an exported CSV, JSON, or JSONL file."],
|
|
33581
|
+
avoidWhen: ["You need a complete backlink index or a domain authority score.", "You want to find broken internal links during a crawl."],
|
|
33582
|
+
outcome: "A bounded, deduplicated link list with target-page counts and explicit source coverage."
|
|
33583
|
+
},
|
|
33584
|
+
"server-log-analysis": {
|
|
33585
|
+
name: "Server log crawler analysis",
|
|
33586
|
+
description: "Stream a local access log to see observed search and AI crawler requests, statuses, and paths.",
|
|
33587
|
+
useWhen: ["You need evidence of crawler requests that reached the web server.", "You want to find crawler-facing 4xx or 5xx responses in a local access log."],
|
|
33588
|
+
avoidWhen: ["You do not have a combined or JSONL access log.", "You need to prove crawler identity from a user-agent string alone."],
|
|
33589
|
+
outcome: "Bounded crawler, status, and path aggregates with file coverage and parser limits kept visible."
|
|
33590
|
+
},
|
|
31180
33591
|
cannibalisation: {
|
|
31181
33592
|
name: "Query and URL overlap",
|
|
31182
33593
|
description: "Find search queries that appear across several URLs and decide whether the overlap needs attention.",
|
|
@@ -31683,6 +34094,35 @@ function registerReportTools(server) {
|
|
|
31683
34094
|
registerSecondPageTool(server);
|
|
31684
34095
|
}
|
|
31685
34096
|
//#endregion
|
|
34097
|
+
//#region packages/mcp/src/server-log-tools.ts
|
|
34098
|
+
function registerServerLogTools(server) {
|
|
34099
|
+
server.registerTool("seo_server_log_analysis", {
|
|
34100
|
+
description: "Stream a local access log into bounded search and AI crawler evidence",
|
|
34101
|
+
inputSchema: {
|
|
34102
|
+
file: z$1.string().min(1).max(4096),
|
|
34103
|
+
format: z$1.enum(["combined", "jsonl"]).optional(),
|
|
34104
|
+
rowLimit: z$1.number().int().min(1).max(1e7).optional(),
|
|
34105
|
+
pathLimit: z$1.number().int().min(1).max(1e5).optional(),
|
|
34106
|
+
limit: z$1.number().int().min(1).max(500).optional()
|
|
34107
|
+
}
|
|
34108
|
+
}, async ({ file, format, rowLimit, pathLimit, limit }) => {
|
|
34109
|
+
try {
|
|
34110
|
+
const report = serverLogReport({
|
|
34111
|
+
evidence: await importServerLog({
|
|
34112
|
+
file,
|
|
34113
|
+
format,
|
|
34114
|
+
rowLimit,
|
|
34115
|
+
pathLimit
|
|
34116
|
+
}),
|
|
34117
|
+
limit
|
|
34118
|
+
});
|
|
34119
|
+
return toolSuccess(`${report.summary.crawlerRows} crawler requests were observed in ${report.summary.parsedRows} parsed log rows.`, report);
|
|
34120
|
+
} catch (error) {
|
|
34121
|
+
return toolError(error);
|
|
34122
|
+
}
|
|
34123
|
+
});
|
|
34124
|
+
}
|
|
34125
|
+
//#endregion
|
|
31686
34126
|
//#region packages/mcp/src/agent-output-budget.ts
|
|
31687
34127
|
const AGENT_STRUCTURED_OUTPUT_MAX_BYTES = 96 * 1024;
|
|
31688
34128
|
function jsonBytes(value) {
|
|
@@ -32079,6 +34519,16 @@ const reportGroups = [
|
|
|
32079
34519
|
"seo_redirect_trace"
|
|
32080
34520
|
]
|
|
32081
34521
|
},
|
|
34522
|
+
{
|
|
34523
|
+
category: "monitoring",
|
|
34524
|
+
register: registerProviderTools,
|
|
34525
|
+
names: ["seo_bing_webmaster_overview", "seo_link_evidence"]
|
|
34526
|
+
},
|
|
34527
|
+
{
|
|
34528
|
+
category: "monitoring",
|
|
34529
|
+
register: registerServerLogTools,
|
|
34530
|
+
names: ["seo_server_log_analysis"]
|
|
34531
|
+
},
|
|
32082
34532
|
{
|
|
32083
34533
|
category: "reporting",
|
|
32084
34534
|
register: registerReportTools,
|