seo 0.2.9 → 0.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -1
- package/dist/cli.js +219 -109
- package/dist/cli.js.map +1 -1
- package/dist/{dist-BrBR9umh.js → dist-D0EI_L8C.js} +1 -1
- package/dist/{dist-C6zXDARm.js → dist-DaHrxVkW.js} +85 -80
- package/dist/dist-DaHrxVkW.js.map +1 -0
- package/dist/index.d.ts +1223 -64
- package/dist/index.js +2801 -410
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +2574 -386
- package/dist/mcp.js.map +1 -1
- package/dist/{package-C51oI-6o.js → package-B-M-4PZ_.js} +2 -2
- package/dist/{package-C51oI-6o.js.map → package-B-M-4PZ_.js.map} +1 -1
- package/evals/bing-webmaster-overview.json +31 -0
- package/evals/link-evidence.json +29 -0
- package/evals/seo.json +16 -0
- package/evals/server-log-analysis.json +29 -0
- package/package.json +3 -3
- package/skills/seo/SKILL.md +56 -67
- package/dist/dist-C6zXDARm.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { n as load, t as version } from "./package-
|
|
2
|
+
import { n as load, t as version } from "./package-B-M-4PZ_.js";
|
|
3
3
|
import { Headers, fetch as fetch$1 } from "undici";
|
|
4
|
-
import { chmodSync, createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
5
|
-
import { arch, homedir, platform } from "node:os";
|
|
6
|
-
import { basename, dirname, join } from "node:path";
|
|
4
|
+
import { chmodSync, createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
5
|
+
import { arch, homedir, platform, totalmem } from "node:os";
|
|
6
|
+
import { basename, dirname, extname, join, resolve } from "node:path";
|
|
7
7
|
import { z } from "zod";
|
|
8
8
|
import lockfile from "proper-lockfile";
|
|
9
|
-
import crypto, { createHash, randomUUID, sign } from "node:crypto";
|
|
9
|
+
import crypto, { createHash, randomBytes, randomUUID, sign } from "node:crypto";
|
|
10
10
|
import http from "node:http";
|
|
11
11
|
import open from "open";
|
|
12
12
|
import Libsql from "libsql";
|
|
@@ -21,9 +21,9 @@ import { gunzipSync } from "node:zlib";
|
|
|
21
21
|
import { execFile } from "node:child_process";
|
|
22
22
|
import { fileURLToPath } from "node:url";
|
|
23
23
|
import { promisify } from "node:util";
|
|
24
|
+
import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
24
25
|
import pino from "pino";
|
|
25
26
|
import pretty from "pino-pretty";
|
|
26
|
-
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
27
27
|
//#region packages/core/src/errors.ts
|
|
28
28
|
const errorMetadata = {
|
|
29
29
|
ACCESS_DENIED: {
|
|
@@ -83,12 +83,12 @@ var SeoError = class extends Error {
|
|
|
83
83
|
this.retryable = errorMetadata[code].retryable;
|
|
84
84
|
}
|
|
85
85
|
};
|
|
86
|
-
function errorMessage$
|
|
86
|
+
function errorMessage$3(error) {
|
|
87
87
|
return error instanceof Error ? error.message : String(error);
|
|
88
88
|
}
|
|
89
89
|
function toSeoError(error) {
|
|
90
90
|
if (error instanceof SeoError) return error;
|
|
91
|
-
const message = errorMessage$
|
|
91
|
+
const message = errorMessage$3(error);
|
|
92
92
|
if (/not logged in/i.test(message)) return new SeoError("AUTH_REQUIRED", message);
|
|
93
93
|
if (/oauth client config.*missing|shared seo google app/i.test(message)) return new SeoError("AUTH_CONFIG_REQUIRED", message);
|
|
94
94
|
if (/invalid_grant|refresh token.*no longer valid/i.test(message)) return new SeoError("AUTH_EXPIRED", message);
|
|
@@ -145,6 +145,7 @@ function resolveSeoCliPaths(input = {}) {
|
|
|
145
145
|
configFile: join(configDir, "config.json"),
|
|
146
146
|
tokensFile: join(configDir, "tokens.json"),
|
|
147
147
|
oauthClientFile: join(configDir, "oauth-client.json"),
|
|
148
|
+
providerSecretsFile: join(configDir, "provider-secrets.json"),
|
|
148
149
|
telemetryStateFile: join(configDir, "telemetry.json"),
|
|
149
150
|
cacheDbFile: join(cacheDir, "cache.db")
|
|
150
151
|
};
|
|
@@ -231,6 +232,7 @@ const clientProfileSchema = z.object({
|
|
|
231
232
|
watchUrls: z.array(z.string()).default([]),
|
|
232
233
|
brandTerms: z.array(z.string()).default([]),
|
|
233
234
|
analytics: analyticsConnectionsSchema,
|
|
235
|
+
searchEngines: z.object({ bing: z.object({ siteUrl: z.string() }).optional() }).optional(),
|
|
234
236
|
reportDay: z.number().int().min(1).max(31).optional(),
|
|
235
237
|
technicalWeekday: z.number().int().min(0).max(7).optional(),
|
|
236
238
|
isDefault: z.boolean().optional(),
|
|
@@ -316,11 +318,11 @@ async function deleteKeyringPassword(service, account) {
|
|
|
316
318
|
}
|
|
317
319
|
//#endregion
|
|
318
320
|
//#region packages/core/src/storage/config.ts
|
|
319
|
-
const KEYRING_SERVICE = "seo";
|
|
320
|
-
const PRIVATE_FILE_MODE = 384;
|
|
321
|
+
const KEYRING_SERVICE$1 = "seo";
|
|
322
|
+
const PRIVATE_FILE_MODE$1 = 384;
|
|
321
323
|
function tightenConfigPermissions(path) {
|
|
322
324
|
try {
|
|
323
|
-
chmodSync(path, PRIVATE_FILE_MODE);
|
|
325
|
+
chmodSync(path, PRIVATE_FILE_MODE$1);
|
|
324
326
|
} catch (error) {
|
|
325
327
|
if (error.code !== "ENOENT") throw error;
|
|
326
328
|
}
|
|
@@ -334,7 +336,7 @@ function readConfig() {
|
|
|
334
336
|
}
|
|
335
337
|
function writeConfig(config) {
|
|
336
338
|
ensureSeoCliDirs();
|
|
337
|
-
writeJsonAtomic(getSeoCliPaths().configFile, configSchema.parse(config), PRIVATE_FILE_MODE);
|
|
339
|
+
writeJsonAtomic(getSeoCliPaths().configFile, configSchema.parse(config), PRIVATE_FILE_MODE$1);
|
|
338
340
|
}
|
|
339
341
|
function getKeyringAccount(tokens) {
|
|
340
342
|
return `google:${tokens.account_email}`;
|
|
@@ -354,7 +356,7 @@ function tokenMetadata(tokens) {
|
|
|
354
356
|
}
|
|
355
357
|
async function readKeyringTokens(tokens) {
|
|
356
358
|
const account = getKeyringAccount(tokens);
|
|
357
|
-
const [accessToken, refreshToken] = await Promise.all([getKeyringPassword(KEYRING_SERVICE, `${account}:access`), getKeyringPassword(KEYRING_SERVICE, `${account}:refresh`)]);
|
|
359
|
+
const [accessToken, refreshToken] = await Promise.all([getKeyringPassword(KEYRING_SERVICE$1, `${account}:access`), getKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`)]);
|
|
358
360
|
return {
|
|
359
361
|
accessToken: accessToken ?? void 0,
|
|
360
362
|
refreshToken: refreshToken ?? void 0
|
|
@@ -362,14 +364,14 @@ async function readKeyringTokens(tokens) {
|
|
|
362
364
|
}
|
|
363
365
|
async function writeKeyringTokens(tokens) {
|
|
364
366
|
const account = getKeyringAccount(tokens);
|
|
365
|
-
if (tokens.access_token) await setKeyringPassword(KEYRING_SERVICE, `${account}:access`, tokens.access_token);
|
|
366
|
-
else await deleteKeyringPassword(KEYRING_SERVICE, `${account}:access`);
|
|
367
|
-
if (tokens.refresh_token) await setKeyringPassword(KEYRING_SERVICE, `${account}:refresh`, tokens.refresh_token);
|
|
368
|
-
else await deleteKeyringPassword(KEYRING_SERVICE, `${account}:refresh`);
|
|
367
|
+
if (tokens.access_token) await setKeyringPassword(KEYRING_SERVICE$1, `${account}:access`, tokens.access_token);
|
|
368
|
+
else await deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:access`);
|
|
369
|
+
if (tokens.refresh_token) await setKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`, tokens.refresh_token);
|
|
370
|
+
else await deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`);
|
|
369
371
|
}
|
|
370
372
|
async function deleteKeyringTokens(tokens) {
|
|
371
373
|
const account = getKeyringAccount(tokens);
|
|
372
|
-
await Promise.all([deleteKeyringPassword(KEYRING_SERVICE, `${account}:access`), deleteKeyringPassword(KEYRING_SERVICE, `${account}:refresh`)]);
|
|
374
|
+
await Promise.all([deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:access`), deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`)]);
|
|
373
375
|
}
|
|
374
376
|
async function readTokens() {
|
|
375
377
|
ensureSeoCliDirs();
|
|
@@ -381,7 +383,7 @@ async function readTokens() {
|
|
|
381
383
|
const keyringTokens = await readKeyringTokens(parsed);
|
|
382
384
|
if (hasTokenSecrets(parsed)) {
|
|
383
385
|
await writeKeyringTokens(parsed);
|
|
384
|
-
writeJsonAtomic(getSeoCliPaths().tokensFile, tokenMetadata(parsed), PRIVATE_FILE_MODE);
|
|
386
|
+
writeJsonAtomic(getSeoCliPaths().tokensFile, tokenMetadata(parsed), PRIVATE_FILE_MODE$1);
|
|
385
387
|
return parsed;
|
|
386
388
|
}
|
|
387
389
|
return {
|
|
@@ -398,10 +400,10 @@ async function writeTokens(tokens) {
|
|
|
398
400
|
const parsed = tokenSchema.parse(tokens);
|
|
399
401
|
if (readConfig().security.useKeychain) try {
|
|
400
402
|
await writeKeyringTokens(parsed);
|
|
401
|
-
writeJsonAtomic(getSeoCliPaths().tokensFile, tokenMetadata(parsed), PRIVATE_FILE_MODE);
|
|
403
|
+
writeJsonAtomic(getSeoCliPaths().tokensFile, tokenMetadata(parsed), PRIVATE_FILE_MODE$1);
|
|
402
404
|
return;
|
|
403
405
|
} catch {}
|
|
404
|
-
writeJsonAtomic(getSeoCliPaths().tokensFile, parsed, PRIVATE_FILE_MODE);
|
|
406
|
+
writeJsonAtomic(getSeoCliPaths().tokensFile, parsed, PRIVATE_FILE_MODE$1);
|
|
405
407
|
}
|
|
406
408
|
async function deleteTokens() {
|
|
407
409
|
const tokens = await readTokens();
|
|
@@ -493,6 +495,12 @@ function getPrivacySnapshot() {
|
|
|
493
495
|
mode: fileMode(paths.oauthClientFile),
|
|
494
496
|
sizeBytes: sizeOf(paths.oauthClientFile)
|
|
495
497
|
},
|
|
498
|
+
{
|
|
499
|
+
label: "Provider secrets",
|
|
500
|
+
path: paths.providerSecretsFile,
|
|
501
|
+
mode: fileMode(paths.providerSecretsFile),
|
|
502
|
+
sizeBytes: sizeOf(paths.providerSecretsFile)
|
|
503
|
+
},
|
|
496
504
|
{
|
|
497
505
|
label: "Telemetry state",
|
|
498
506
|
path: paths.telemetryStateFile,
|
|
@@ -1082,15 +1090,15 @@ async function exchangeCode(input) {
|
|
|
1082
1090
|
}
|
|
1083
1091
|
//#endregion
|
|
1084
1092
|
//#region packages/core/src/storage/cache-maintenance.ts
|
|
1085
|
-
const MEBIBYTE$
|
|
1086
|
-
const CACHE_MAX_BYTES = 256 * MEBIBYTE$
|
|
1093
|
+
const MEBIBYTE$2 = 1024 * 1024;
|
|
1094
|
+
const CACHE_MAX_BYTES = 256 * MEBIBYTE$2;
|
|
1087
1095
|
const CACHE_MAX_AGE_MS = 720 * 60 * 60 * 1e3;
|
|
1088
1096
|
const CACHE_LIMITS = {
|
|
1089
|
-
gsc_cache: 48 * MEBIBYTE$
|
|
1090
|
-
ga4_cache: 24 * MEBIBYTE$
|
|
1091
|
-
semrush_cache: 16 * MEBIBYTE$
|
|
1092
|
-
http_cache: 160 * MEBIBYTE$
|
|
1093
|
-
performance_reports: 8 * MEBIBYTE$
|
|
1097
|
+
gsc_cache: 48 * MEBIBYTE$2,
|
|
1098
|
+
ga4_cache: 24 * MEBIBYTE$2,
|
|
1099
|
+
semrush_cache: 16 * MEBIBYTE$2,
|
|
1100
|
+
http_cache: 160 * MEBIBYTE$2,
|
|
1101
|
+
performance_reports: 8 * MEBIBYTE$2
|
|
1094
1102
|
};
|
|
1095
1103
|
const CACHE_POLICIES = [
|
|
1096
1104
|
{
|
|
@@ -2963,7 +2971,7 @@ function isValidStructuredDate(value) {
|
|
|
2963
2971
|
const parsed = new Date(trimmed);
|
|
2964
2972
|
return Number.isFinite(parsed.getTime()) && parsed.toISOString().slice(0, 10) === datePart;
|
|
2965
2973
|
}
|
|
2966
|
-
function parseJsonLd(blocks, baseUrl) {
|
|
2974
|
+
function parseJsonLd(blocks, baseUrl, retainJsonLd) {
|
|
2967
2975
|
const jsonLd = [];
|
|
2968
2976
|
const invalidJsonLdSamples = [];
|
|
2969
2977
|
const unrecognizedJsonLdTypes = [];
|
|
@@ -3035,7 +3043,7 @@ function parseJsonLd(blocks, baseUrl) {
|
|
|
3035
3043
|
blocks.forEach((block, index) => {
|
|
3036
3044
|
try {
|
|
3037
3045
|
const parsed = JSON.parse(block);
|
|
3038
|
-
jsonLd.push(parsed);
|
|
3046
|
+
if (retainJsonLd) jsonLd.push(parsed);
|
|
3039
3047
|
visit(parsed, { prefixes: /* @__PURE__ */ new Map() }, index, "$");
|
|
3040
3048
|
} catch (error) {
|
|
3041
3049
|
invalidJsonLdSamples.push({
|
|
@@ -3096,8 +3104,8 @@ function usableElementValue($, selector) {
|
|
|
3096
3104
|
return ($(element).attr("content") ?? $(element).attr("datetime") ?? $(element).text()).trim().length > 0;
|
|
3097
3105
|
});
|
|
3098
3106
|
}
|
|
3099
|
-
function extractStructuredData($, baseUrl) {
|
|
3100
|
-
const parsed = parseJsonLd($("script[type=\"application/ld+json\"]").toArray().map((element) => $(element).html() ?? ""), baseUrl);
|
|
3107
|
+
function extractStructuredData($, baseUrl, options = {}) {
|
|
3108
|
+
const parsed = parseJsonLd($("script[type=\"application/ld+json\"]").toArray().map((element) => $(element).html() ?? ""), baseUrl, options.retainJsonLd ?? true);
|
|
3101
3109
|
const microdata = microdataTypes($);
|
|
3102
3110
|
const rdfa = rdfaTypes($);
|
|
3103
3111
|
const schemaTypes = [.../* @__PURE__ */ new Set([
|
|
@@ -3131,6 +3139,16 @@ function extractStructuredData($, baseUrl) {
|
|
|
3131
3139
|
}
|
|
3132
3140
|
//#endregion
|
|
3133
3141
|
//#region packages/core/src/extract/page-extractor.ts
|
|
3142
|
+
const SOCIAL_PROFILE_HOSTS = [
|
|
3143
|
+
"facebook.com",
|
|
3144
|
+
"instagram.com",
|
|
3145
|
+
"linkedin.com",
|
|
3146
|
+
"pinterest.com",
|
|
3147
|
+
"tiktok.com",
|
|
3148
|
+
"twitter.com",
|
|
3149
|
+
"x.com",
|
|
3150
|
+
"youtube.com"
|
|
3151
|
+
];
|
|
3134
3152
|
function safeText$1(value) {
|
|
3135
3153
|
const trimmed = value?.replace(/\s+/g, " ").trim();
|
|
3136
3154
|
return trimmed ? trimmed : void 0;
|
|
@@ -3147,7 +3165,7 @@ function absoluteUrl$2(value, base) {
|
|
|
3147
3165
|
return;
|
|
3148
3166
|
}
|
|
3149
3167
|
}
|
|
3150
|
-
function httpUrl(value, base) {
|
|
3168
|
+
function httpUrl$1(value, base) {
|
|
3151
3169
|
const resolved = absoluteUrl$2(value, base);
|
|
3152
3170
|
if (!resolved) return void 0;
|
|
3153
3171
|
const protocol = new URL(resolved).protocol;
|
|
@@ -3203,6 +3221,28 @@ function imageDimensionsFromUrl(src) {
|
|
|
3203
3221
|
height
|
|
3204
3222
|
};
|
|
3205
3223
|
}
|
|
3224
|
+
function addAnchorSample(samples, seen, href, text) {
|
|
3225
|
+
if (samples.length >= 25 || !text) return;
|
|
3226
|
+
const sampleText = text.length > 120 ? text.slice(0, 120).trimEnd() : text;
|
|
3227
|
+
const key = `${href}\n${sampleText}`;
|
|
3228
|
+
if (seen.has(key)) return;
|
|
3229
|
+
seen.add(key);
|
|
3230
|
+
samples.push({
|
|
3231
|
+
href,
|
|
3232
|
+
text: sampleText
|
|
3233
|
+
});
|
|
3234
|
+
}
|
|
3235
|
+
function addSocialProfile(profiles, href) {
|
|
3236
|
+
if (profiles.size >= 50) return;
|
|
3237
|
+
try {
|
|
3238
|
+
const url = new URL(href);
|
|
3239
|
+
const host = url.hostname.replace(/^www\./, "").toLowerCase();
|
|
3240
|
+
if (SOCIAL_PROFILE_HOSTS.some((value) => host === value || host.endsWith(`.${value}`))) {
|
|
3241
|
+
url.hash = "";
|
|
3242
|
+
profiles.add(url.toString());
|
|
3243
|
+
}
|
|
3244
|
+
} catch {}
|
|
3245
|
+
}
|
|
3206
3246
|
async function extractPage(fetchResult, extractor = "defuddle", dependencies = {}) {
|
|
3207
3247
|
const $ = extractor === "crawler" && /<html(?:\s|>)/i.test(fetchResult.html) ? load(fetchResult.html, { xml: {
|
|
3208
3248
|
xmlMode: false,
|
|
@@ -3210,65 +3250,106 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
3210
3250
|
} }) : load(fetchResult.html);
|
|
3211
3251
|
const url = new URL(fetchResult.finalUrl);
|
|
3212
3252
|
const canonicalEvidence = extractCanonicalEvidence($, fetchResult.headers, fetchResult.finalUrl);
|
|
3213
|
-
const headings =
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3253
|
+
const headings = [];
|
|
3254
|
+
let crawlerH1;
|
|
3255
|
+
let crawlerH1Count = 0;
|
|
3256
|
+
let crawlerH2Count = 0;
|
|
3257
|
+
let crawlerH3Count = 0;
|
|
3258
|
+
let questionHeadings = 0;
|
|
3259
|
+
$("h1, h2, h3, h4, h5, h6").each((_index, element) => {
|
|
3260
|
+
const level = Number.parseInt(element.tagName.slice(1), 10);
|
|
3261
|
+
const text = $(element).text().replace(/\s+/g, " ").trim();
|
|
3262
|
+
if (!text) return;
|
|
3263
|
+
if (text.trimEnd().endsWith("?")) questionHeadings += 1;
|
|
3264
|
+
if (level === 1) {
|
|
3265
|
+
crawlerH1 ??= text;
|
|
3266
|
+
crawlerH1Count += 1;
|
|
3267
|
+
} else if (level === 2) crawlerH2Count += 1;
|
|
3268
|
+
else if (level === 3) crawlerH3Count += 1;
|
|
3269
|
+
if (extractor !== "crawler") headings.push({
|
|
3270
|
+
level,
|
|
3271
|
+
text
|
|
3272
|
+
});
|
|
3273
|
+
});
|
|
3217
3274
|
const links = [];
|
|
3275
|
+
const crawlerInternalLinks = /* @__PURE__ */ new Set();
|
|
3276
|
+
const crawlerExternalLinks = /* @__PURE__ */ new Set();
|
|
3277
|
+
const internalAnchorSamples = [];
|
|
3278
|
+
const externalAnchorSamples = [];
|
|
3279
|
+
const internalAnchorKeys = /* @__PURE__ */ new Set();
|
|
3280
|
+
const externalAnchorKeys = /* @__PURE__ */ new Set();
|
|
3281
|
+
const socialProfileLinks = /* @__PURE__ */ new Set();
|
|
3218
3282
|
let invalidLinkCount = 0;
|
|
3219
3283
|
let unsupportedLinkCount = 0;
|
|
3220
|
-
|
|
3284
|
+
$("a[href]").each((_index, element) => {
|
|
3221
3285
|
const href = $(element).attr("href") ?? "";
|
|
3222
3286
|
try {
|
|
3223
3287
|
const absolute = new URL(href, fetchResult.finalUrl);
|
|
3224
3288
|
if (!["http:", "https:"].includes(absolute.protocol)) {
|
|
3225
3289
|
unsupportedLinkCount += 1;
|
|
3226
|
-
|
|
3290
|
+
return;
|
|
3227
3291
|
}
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3292
|
+
const absoluteHref = absolute.toString();
|
|
3293
|
+
const text = $(element).text().replace(/\s+/g, " ").trim();
|
|
3294
|
+
const internal = absolute.host === url.host;
|
|
3295
|
+
if (extractor === "crawler") {
|
|
3296
|
+
if (absolute.origin === url.origin) {
|
|
3297
|
+
absolute.hash = "";
|
|
3298
|
+
crawlerInternalLinks.add(absolute.toString());
|
|
3299
|
+
}
|
|
3300
|
+
if (!internal) {
|
|
3301
|
+
crawlerExternalLinks.add(absoluteHref);
|
|
3302
|
+
addSocialProfile(socialProfileLinks, absoluteHref);
|
|
3303
|
+
}
|
|
3304
|
+
addAnchorSample(internal ? internalAnchorSamples : externalAnchorSamples, internal ? internalAnchorKeys : externalAnchorKeys, absoluteHref, text);
|
|
3305
|
+
} else links.push({
|
|
3306
|
+
href: absoluteHref,
|
|
3307
|
+
text,
|
|
3231
3308
|
rel: ($(element).attr("rel") ?? "").split(/\s+/).filter(Boolean),
|
|
3232
|
-
internal
|
|
3309
|
+
internal,
|
|
3233
3310
|
location: $(element).closest("nav, header").length ? "navigation" : $(element).closest("footer").length ? "footer" : $(element).closest("main, article").length ? "main-content" : "other"
|
|
3234
3311
|
});
|
|
3235
3312
|
} catch {
|
|
3236
3313
|
invalidLinkCount += 1;
|
|
3237
3314
|
}
|
|
3238
|
-
}
|
|
3315
|
+
});
|
|
3239
3316
|
const hreflang = [];
|
|
3240
3317
|
let invalidHreflangCount = 0;
|
|
3241
3318
|
for (const element of $("link[rel~=\"alternate\"][hreflang][href]").toArray()) {
|
|
3242
3319
|
const language = $(element).attr("hreflang") ?? "";
|
|
3243
|
-
const href = httpUrl($(element).attr("href"), fetchResult.finalUrl);
|
|
3320
|
+
const href = httpUrl$1($(element).attr("href"), fetchResult.finalUrl);
|
|
3244
3321
|
if (language && href) hreflang.push({
|
|
3245
3322
|
hreflang: language,
|
|
3246
3323
|
href
|
|
3247
3324
|
});
|
|
3248
3325
|
else invalidHreflangCount += 1;
|
|
3249
3326
|
}
|
|
3250
|
-
const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
|
|
3327
|
+
const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl$1($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
|
|
3251
3328
|
const openGraph = Object.fromEntries($("meta[property^=\"og:\"]").toArray().map((element) => [$(element).attr("property") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
3252
3329
|
const twitter = Object.fromEntries($("meta[name^=\"twitter:\"]").toArray().map((element) => [$(element).attr("name") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
3253
|
-
const structuredData = extractStructuredData($, fetchResult.finalUrl);
|
|
3330
|
+
const structuredData = extractStructuredData($, fetchResult.finalUrl, { retainJsonLd: extractor !== "crawler" });
|
|
3254
3331
|
const hasDate = structuredData.hasDate || isValidStructuredDate($("meta[property=\"article:published_time\"]").attr("content")) || isValidStructuredDate($("meta[property=\"article:modified_time\"]").attr("content")) || $("article time[datetime], main time[datetime]").toArray().some((element) => isValidStructuredDate($(element).attr("datetime")));
|
|
3255
3332
|
const author = safeText$1($("meta[name=\"author\"]").attr("content"));
|
|
3256
3333
|
const hasAuthor = Boolean(author) || structuredData.hasAuthor || $("[rel~=author], .author, .byline").toArray().some((element) => Boolean(safeText$1($(element).text())));
|
|
3257
3334
|
const semanticHtml = $("main, article").length > 0;
|
|
3258
|
-
const questionHeadings = headings.filter((heading) => heading.text.trimEnd().endsWith("?")).length;
|
|
3259
3335
|
const listCount = $("ul, ol").length;
|
|
3260
3336
|
const tableCount = $("table").length;
|
|
3261
3337
|
const structuredBlocks = listCount + tableCount;
|
|
3262
3338
|
const answerable = $("main p, article p, p").toArray().slice(0, 3).some((element) => $(element).text().replace(/\s+/g, " ").trim().split(/\s+/).filter(Boolean).length >= 25);
|
|
3263
|
-
|
|
3264
|
-
|
|
3339
|
+
let imagesTotal = 0;
|
|
3340
|
+
let imagesMissingAlt = 0;
|
|
3341
|
+
const oversizedImageCandidates = [];
|
|
3342
|
+
$("img").each((_index, element) => {
|
|
3343
|
+
imagesTotal += 1;
|
|
3344
|
+
if ($(element).attr("alt") === void 0) imagesMissingAlt += 1;
|
|
3345
|
+
if (oversizedImageCandidates.length >= 25) return;
|
|
3265
3346
|
const src = absoluteUrl$2($(element).attr("src"), fetchResult.finalUrl);
|
|
3266
|
-
if (!src) return
|
|
3347
|
+
if (!src) return;
|
|
3267
3348
|
const width = numericAttribute($(element).attr("width"));
|
|
3268
3349
|
const height = numericAttribute($(element).attr("height"));
|
|
3269
3350
|
const srcsetWidth = largestSrcsetWidth($(element).attr("srcset"));
|
|
3270
3351
|
const filenameDimensions = imageDimensionsFromUrl(src);
|
|
3271
|
-
if (Math.max(width ?? 0, height ?? 0, srcsetWidth ?? 0, filenameDimensions?.width ?? 0, filenameDimensions?.height ?? 0) < 2e3) return
|
|
3352
|
+
if (Math.max(width ?? 0, height ?? 0, srcsetWidth ?? 0, filenameDimensions?.width ?? 0, filenameDimensions?.height ?? 0) < 2e3) return;
|
|
3272
3353
|
const detectedFrom = [
|
|
3273
3354
|
width && width >= 2e3 ? "width" : void 0,
|
|
3274
3355
|
height && height >= 2e3 ? "height" : void 0,
|
|
@@ -3277,13 +3358,13 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
3277
3358
|
].filter(Boolean).join(",");
|
|
3278
3359
|
const candidateWidth = Math.max(width ?? 0, srcsetWidth ?? 0, filenameDimensions?.width ?? 0);
|
|
3279
3360
|
const candidateHeight = Math.max(height ?? 0, filenameDimensions?.height ?? 0);
|
|
3280
|
-
|
|
3361
|
+
oversizedImageCandidates.push({
|
|
3281
3362
|
src,
|
|
3282
3363
|
detectedFrom,
|
|
3283
3364
|
...candidateWidth > 0 ? { width: candidateWidth } : {},
|
|
3284
3365
|
...candidateHeight > 0 ? { height: candidateHeight } : {}
|
|
3285
|
-
};
|
|
3286
|
-
})
|
|
3366
|
+
});
|
|
3367
|
+
});
|
|
3287
3368
|
const mixedContentUrls = url.protocol === "https:" ? [...new Set($("img[src], script[src], iframe[src], source[src], video[src], audio[src], link[href]").toArray().map((element) => absoluteUrl$2($(element).attr("src") ?? $(element).attr("href"), fetchResult.finalUrl)).filter((value) => typeof value === "string" && new URL(value).protocol === "http:"))] : [];
|
|
3288
3369
|
const metaRobots = combineRobotsValues($("meta[name]").toArray().filter((element) => {
|
|
3289
3370
|
const name = $(element).attr("name")?.trim().toLowerCase();
|
|
@@ -3307,6 +3388,18 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
3307
3388
|
hasViewport: Boolean($("meta[name=\"viewport\"]").attr("content")),
|
|
3308
3389
|
headings,
|
|
3309
3390
|
links,
|
|
3391
|
+
...extractor === "crawler" ? { crawlerEvidence: {
|
|
3392
|
+
h1: crawlerH1,
|
|
3393
|
+
h1Count: crawlerH1Count,
|
|
3394
|
+
h2Count: crawlerH2Count,
|
|
3395
|
+
h3Count: crawlerH3Count,
|
|
3396
|
+
questionHeadings,
|
|
3397
|
+
internalLinks: [...crawlerInternalLinks],
|
|
3398
|
+
externalLinks: [...crawlerExternalLinks],
|
|
3399
|
+
internalAnchorSamples,
|
|
3400
|
+
externalAnchorSamples,
|
|
3401
|
+
socialProfileLinks: [...socialProfileLinks]
|
|
3402
|
+
} } : {},
|
|
3310
3403
|
markdownAlternates,
|
|
3311
3404
|
hreflang,
|
|
3312
3405
|
jsonLd: structuredData.jsonLd,
|
|
@@ -3315,19 +3408,17 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
3315
3408
|
unrecognizedJsonLdTypes: structuredData.unrecognizedJsonLdTypes.slice(0, 25),
|
|
3316
3409
|
structuredDataFormats: structuredData.formats,
|
|
3317
3410
|
googleRichResults: structuredData.googleRichResults,
|
|
3318
|
-
schemaSameAsEvidence: structuredData.sameAs,
|
|
3319
|
-
invalidSchemaSameAs: structuredData.invalidSameAs,
|
|
3411
|
+
schemaSameAsEvidence: extractor === "crawler" ? structuredData.sameAs.slice(0, 50) : structuredData.sameAs,
|
|
3412
|
+
invalidSchemaSameAs: extractor === "crawler" ? structuredData.invalidSameAs.slice(0, 25) : structuredData.invalidSameAs,
|
|
3320
3413
|
schemaTypes: structuredData.schemaTypes,
|
|
3321
3414
|
openGraph,
|
|
3322
3415
|
twitter,
|
|
3323
3416
|
author,
|
|
3324
3417
|
hasAuthor,
|
|
3325
3418
|
hasDate,
|
|
3326
|
-
imagesTotal
|
|
3327
|
-
imagesMissingAlt
|
|
3328
|
-
|
|
3329
|
-
}).length,
|
|
3330
|
-
oversizedImageCandidates: oversizedImageCandidates.slice(0, 25),
|
|
3419
|
+
imagesTotal,
|
|
3420
|
+
imagesMissingAlt,
|
|
3421
|
+
oversizedImageCandidates,
|
|
3331
3422
|
mixedContentUrls,
|
|
3332
3423
|
semanticHtml,
|
|
3333
3424
|
questionHeadings,
|
|
@@ -3335,7 +3426,7 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
3335
3426
|
tableCount,
|
|
3336
3427
|
structuredBlocks,
|
|
3337
3428
|
answerable,
|
|
3338
|
-
contentText: text.replace(/\s+/g, " ").trim(),
|
|
3429
|
+
contentText: extractor === "crawler" ? text : text.replace(/\s+/g, " ").trim(),
|
|
3339
3430
|
excerpt,
|
|
3340
3431
|
wordCount: content.wordCount,
|
|
3341
3432
|
contentExtraction: content.diagnostics,
|
|
@@ -3867,7 +3958,7 @@ function decodePageFetchCacheEvidence(value) {
|
|
|
3867
3958
|
return;
|
|
3868
3959
|
}
|
|
3869
3960
|
}
|
|
3870
|
-
async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, respectRobots = false) {
|
|
3961
|
+
async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, respectRobots = false, writeCache = true) {
|
|
3871
3962
|
const startedAt = Date.now();
|
|
3872
3963
|
const db = getDb();
|
|
3873
3964
|
const key = hashKey(["page", url]);
|
|
@@ -3979,7 +4070,7 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
|
|
|
3979
4070
|
warnings: [...robots.allowed === null ? [`robots.txt availability is unknown${robots.status ? ` after HTTP ${robots.status}` : ""}; this fetch does not prove crawler access is allowed.`] : [], ...accessBlock ? [`${accessBlock.guidance.summary} Identify the crawler as ${accessBlock.crawler.userAgent}.`] : []],
|
|
3980
4071
|
robotsTxt: pageRobotsEvidence(robots)
|
|
3981
4072
|
};
|
|
3982
|
-
if (!accessBlock) {
|
|
4073
|
+
if (!accessBlock && writeCache) {
|
|
3983
4074
|
db.prepare(`INSERT OR REPLACE INTO http_cache
|
|
3984
4075
|
(url_hash, url, status, headers_json, body_blob, metadata_json, etag, fetched_at, expires_at)
|
|
3985
4076
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(key, url, response.response.status, JSON.stringify(headerMap), Buffer.from(html), encodePageFetchCacheEvidence(result), response.response.headers.get("etag"), Date.now(), Date.now() + 36e5);
|
|
@@ -4547,7 +4638,7 @@ async function fetchPage(url, opts = {}, dependencies = {}) {
|
|
|
4547
4638
|
const makeRenderer = dependencies.createPageRenderer ?? createPageRenderer;
|
|
4548
4639
|
const rate = normalizeRateControls(opts.rate);
|
|
4549
4640
|
return await queueForHost(new URL(url).host, rate).add(async () => {
|
|
4550
|
-
const first = await plainFetch(url, opts.refresh, opts.timeoutMs, rate, opts.signal, opts.respectRobots);
|
|
4641
|
+
const first = await plainFetch(url, opts.refresh, opts.timeoutMs, rate, opts.signal, opts.respectRobots, opts.writeCache);
|
|
4551
4642
|
if (!(!first.diagnostics.accessBlock && (renderingMode === "on" || renderingMode === "auto" && looksLikeSpa(first.html))) || opts.signal?.aborted) return withRenderingState(first, renderingMode, opts.signal?.aborted ? "skipped" : renderingMode === "off" ? "not-requested" : "not-needed");
|
|
4552
4643
|
const ownsRenderer = !opts.renderer;
|
|
4553
4644
|
const renderer = opts.renderer ?? makeRenderer();
|
|
@@ -4652,7 +4743,7 @@ function urlInspectionQuotaDate(now) {
|
|
|
4652
4743
|
function urlInspectionQuotaResetAt(now) {
|
|
4653
4744
|
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1));
|
|
4654
4745
|
}
|
|
4655
|
-
function boundedLimit(value) {
|
|
4746
|
+
function boundedLimit$1(value) {
|
|
4656
4747
|
const limit = value ?? 2e3;
|
|
4657
4748
|
if (!Number.isInteger(limit) || limit < 1 || limit > 2e3) throw new SeoError("INVALID_INPUT", "URL Inspection daily limit must be an integer from 1 to 2000.");
|
|
4658
4749
|
return limit;
|
|
@@ -4685,7 +4776,7 @@ function reserveUrlInspectionQuota(input) {
|
|
|
4685
4776
|
const database = input.database ?? getDb();
|
|
4686
4777
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
4687
4778
|
const nowMs = now.getTime();
|
|
4688
|
-
const requestedLimit = boundedLimit(input.limit);
|
|
4779
|
+
const requestedLimit = boundedLimit$1(input.limit);
|
|
4689
4780
|
const quotaDate = urlInspectionQuotaDate(now);
|
|
4690
4781
|
const resetAt = urlInspectionQuotaResetAt(now);
|
|
4691
4782
|
const result = database.transaction(() => {
|
|
@@ -6591,13 +6682,13 @@ function isLowActionabilityQuery(query) {
|
|
|
6591
6682
|
//#endregion
|
|
6592
6683
|
//#region packages/core/src/analyze/page-opportunities-selection.ts
|
|
6593
6684
|
const DEFAULT_MIN_IMPRESSIONS$3 = 10;
|
|
6594
|
-
const DEFAULT_LIMIT$
|
|
6595
|
-
const MAX_LIMIT$
|
|
6685
|
+
const DEFAULT_LIMIT$4 = 25;
|
|
6686
|
+
const MAX_LIMIT$4 = 100;
|
|
6596
6687
|
function integerInRange$2(value, fallback, minimum, maximum) {
|
|
6597
6688
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
6598
6689
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
6599
6690
|
}
|
|
6600
|
-
function isHttpUrl$
|
|
6691
|
+
function isHttpUrl$2(value) {
|
|
6601
6692
|
try {
|
|
6602
6693
|
const parsed = new URL(value);
|
|
6603
6694
|
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
@@ -6619,7 +6710,7 @@ function isBrandRow(input) {
|
|
|
6619
6710
|
function normalizePageOpportunityOptions(input) {
|
|
6620
6711
|
return {
|
|
6621
6712
|
minImpressions: integerInRange$2(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$3, 1, Number.MAX_SAFE_INTEGER),
|
|
6622
|
-
limit: integerInRange$2(input.limit, DEFAULT_LIMIT$
|
|
6713
|
+
limit: integerInRange$2(input.limit, DEFAULT_LIMIT$4, 1, MAX_LIMIT$4)
|
|
6623
6714
|
};
|
|
6624
6715
|
}
|
|
6625
6716
|
function selectTargetRows(input) {
|
|
@@ -6638,7 +6729,7 @@ function selectTargetRows(input) {
|
|
|
6638
6729
|
for (const row of input.rows) {
|
|
6639
6730
|
const query = row.keys[0]?.trim() ?? "";
|
|
6640
6731
|
const page = row.keys[1] ?? "";
|
|
6641
|
-
if (row.keys.length < 2 || !query || !isHttpUrl$
|
|
6732
|
+
if (row.keys.length < 2 || !query || !isHttpUrl$2(page) || !hasValidMetrics(row)) {
|
|
6642
6733
|
selection.invalidRows++;
|
|
6643
6734
|
continue;
|
|
6644
6735
|
}
|
|
@@ -6674,7 +6765,7 @@ function selectTargetRows(input) {
|
|
|
6674
6765
|
function selectBenchmarkRows$1(input) {
|
|
6675
6766
|
return input.rows.filter((row) => {
|
|
6676
6767
|
const query = row.keys[0]?.trim() ?? "";
|
|
6677
|
-
return row.keys.length >= 2 && isHttpUrl$
|
|
6768
|
+
return row.keys.length >= 2 && isHttpUrl$2(row.keys[1] ?? "") && hasValidMetrics(row) && row.position >= 1 && row.position <= 10 && Boolean(query) && !isLowActionabilityQuery(query) && !isBrandRow({
|
|
6678
6769
|
query,
|
|
6679
6770
|
site: input.site,
|
|
6680
6771
|
brandTerms: input.brandTerms,
|
|
@@ -6856,7 +6947,7 @@ const defaultDependencies$15 = {
|
|
|
6856
6947
|
extract: extractPage,
|
|
6857
6948
|
now: () => /* @__PURE__ */ new Date()
|
|
6858
6949
|
};
|
|
6859
|
-
function errorMessage$
|
|
6950
|
+
function errorMessage$2(error) {
|
|
6860
6951
|
return error instanceof Error ? error.message : String(error);
|
|
6861
6952
|
}
|
|
6862
6953
|
function unique$4(values) {
|
|
@@ -6877,7 +6968,7 @@ async function verifyPage(input) {
|
|
|
6877
6968
|
});
|
|
6878
6969
|
warnings.push(...fetched.warnings);
|
|
6879
6970
|
} catch (error) {
|
|
6880
|
-
const reason = `Page fetch failed: ${errorMessage$
|
|
6971
|
+
const reason = `Page fetch failed: ${errorMessage$2(error)}`;
|
|
6881
6972
|
return {
|
|
6882
6973
|
verification: {
|
|
6883
6974
|
status: "failed",
|
|
@@ -6899,7 +6990,7 @@ async function verifyPage(input) {
|
|
|
6899
6990
|
warnings: unique$4(warnings)
|
|
6900
6991
|
};
|
|
6901
6992
|
} catch (error) {
|
|
6902
|
-
const reason = `Page extraction failed: ${errorMessage$
|
|
6993
|
+
const reason = `Page extraction failed: ${errorMessage$2(error)}`;
|
|
6903
6994
|
return {
|
|
6904
6995
|
fetched,
|
|
6905
6996
|
verification: {
|
|
@@ -8094,7 +8185,7 @@ function check$1(section, input) {
|
|
|
8094
8185
|
...input
|
|
8095
8186
|
};
|
|
8096
8187
|
}
|
|
8097
|
-
function section$
|
|
8188
|
+
function section$2(id, title, checks) {
|
|
8098
8189
|
return {
|
|
8099
8190
|
id,
|
|
8100
8191
|
title,
|
|
@@ -8532,19 +8623,19 @@ function agentReadiness(report) {
|
|
|
8532
8623
|
headline: "No agent-discovery evidence is stored in this crawl. Run a fresh focused check.",
|
|
8533
8624
|
profileApplicability: profiles,
|
|
8534
8625
|
summary: summariseChecks(checks),
|
|
8535
|
-
sections: [section$
|
|
8626
|
+
sections: [section$2("collection", "Evidence collection", [missing]), section$2("profile", "Profile scope", scopeChecks)],
|
|
8536
8627
|
checks,
|
|
8537
8628
|
topActions: [missing],
|
|
8538
8629
|
caveats: ["Not evaluated does not mean absent or broken.", "This report does not measure rankings, AI mentions, citations, or selection."]
|
|
8539
8630
|
};
|
|
8540
8631
|
}
|
|
8541
8632
|
const sections = [
|
|
8542
|
-
section$
|
|
8543
|
-
section$
|
|
8544
|
-
section$
|
|
8545
|
-
section$
|
|
8546
|
-
section$
|
|
8547
|
-
section$
|
|
8633
|
+
section$2("profile", "Profile scope", profileChecks(discovery)),
|
|
8634
|
+
section$2("access", "Crawler and protocol access", accessChecks(report, discovery)),
|
|
8635
|
+
section$2("representations", "HTML and Markdown representations", markdownChecks(discovery)),
|
|
8636
|
+
section$2("discovery", "Agent discovery files", discoveryChecks(discovery, report.ai?.robotsTxt?.contentSignals)),
|
|
8637
|
+
section$2("endpoints", "Agent endpoints and auth discovery", endpointChecks(discovery)),
|
|
8638
|
+
section$2("identity", "Identity evidence", identityChecks(report))
|
|
8548
8639
|
];
|
|
8549
8640
|
const checks = sections.flatMap((item) => item.checks);
|
|
8550
8641
|
const actionable = checks.filter((item) => [
|
|
@@ -8731,7 +8822,7 @@ function status(score, maxScore) {
|
|
|
8731
8822
|
if (ratio >= .5) return "warning";
|
|
8732
8823
|
return "fail";
|
|
8733
8824
|
}
|
|
8734
|
-
function section(id, title, checks) {
|
|
8825
|
+
function section$1(id, title, checks) {
|
|
8735
8826
|
return {
|
|
8736
8827
|
id,
|
|
8737
8828
|
title,
|
|
@@ -8789,7 +8880,7 @@ function aiReadiness(report) {
|
|
|
8789
8880
|
const blockedSnippets = restrictedSnippets.filter(({ control }) => control.status === "blocked");
|
|
8790
8881
|
const limitedSnippets = restrictedSnippets.filter(({ control }) => control.status === "limited");
|
|
8791
8882
|
const sections = [
|
|
8792
|
-
section("agent-access", "Agent access", [
|
|
8883
|
+
section$1("agent-access", "Agent access", [
|
|
8793
8884
|
check({
|
|
8794
8885
|
id: "robots-ai-bots",
|
|
8795
8886
|
section: "agent-access",
|
|
@@ -8832,7 +8923,7 @@ function aiReadiness(report) {
|
|
|
8832
8923
|
evidence: { resources: report.ai?.agentResources ?? [] }
|
|
8833
8924
|
})
|
|
8834
8925
|
]),
|
|
8835
|
-
section("machine-readable", "Machine-readable signals", [check({
|
|
8926
|
+
section$1("machine-readable", "Machine-readable signals", [check({
|
|
8836
8927
|
id: "llms-txt",
|
|
8837
8928
|
section: "machine-readable",
|
|
8838
8929
|
maxScore: 0,
|
|
@@ -8857,7 +8948,7 @@ function aiReadiness(report) {
|
|
|
8857
8948
|
action: invalidJsonLdPages.length ? "Fix invalid JSON-LD before relying on its structured-data claims." : "Use structured data only where it accurately represents the visible page and a supported use case.",
|
|
8858
8949
|
urls: invalidJsonLdPages.slice(0, 10).map((page) => page.finalUrl)
|
|
8859
8950
|
})]),
|
|
8860
|
-
section("content-clarity", "Content clarity", [
|
|
8951
|
+
section$1("content-clarity", "Content clarity", [
|
|
8861
8952
|
check({
|
|
8862
8953
|
id: "answerable-content",
|
|
8863
8954
|
section: "content-clarity",
|
|
@@ -8907,7 +8998,7 @@ function aiReadiness(report) {
|
|
|
8907
8998
|
action: "Set the html lang attribute so crawlers and assistive tech understand the page language."
|
|
8908
8999
|
})
|
|
8909
9000
|
]),
|
|
8910
|
-
section("entity-signals", "Entity signals", [check({
|
|
9001
|
+
section$1("entity-signals", "Entity signals", [check({
|
|
8911
9002
|
id: "entity-schema",
|
|
8912
9003
|
section: "entity-signals",
|
|
8913
9004
|
maxScore: 0,
|
|
@@ -8926,7 +9017,7 @@ function aiReadiness(report) {
|
|
|
8926
9017
|
plainEnglish: "Author and date signals help agents judge freshness and accountability.",
|
|
8927
9018
|
action: "Add visible authors, reviewed dates, and updated dates to content where trust and freshness matter."
|
|
8928
9019
|
})]),
|
|
8929
|
-
section("technical-ux", "Technical UX", [
|
|
9020
|
+
section$1("technical-ux", "Technical UX", [
|
|
8930
9021
|
check({
|
|
8931
9022
|
id: "snippet-controls",
|
|
8932
9023
|
section: "technical-ux",
|
|
@@ -8964,7 +9055,7 @@ function aiReadiness(report) {
|
|
|
8964
9055
|
action: "Fix 4xx/5xx pages, accidental noindex, and blocked internal URLs before expanding AI discovery work."
|
|
8965
9056
|
})
|
|
8966
9057
|
]),
|
|
8967
|
-
section("agent-content", "Agent content observations", [
|
|
9058
|
+
section$1("agent-content", "Agent content observations", [
|
|
8968
9059
|
check({
|
|
8969
9060
|
id: "markdown-alternates",
|
|
8970
9061
|
section: "agent-content",
|
|
@@ -9004,7 +9095,7 @@ function aiReadiness(report) {
|
|
|
9004
9095
|
evidence: agentDiscovery?.profileApplicability
|
|
9005
9096
|
})
|
|
9006
9097
|
]),
|
|
9007
|
-
section("crawl-completeness", "Crawl completeness", [check({
|
|
9098
|
+
section$1("crawl-completeness", "Crawl completeness", [check({
|
|
9008
9099
|
id: "crawl-depth",
|
|
9009
9100
|
section: "crawl-completeness",
|
|
9010
9101
|
maxScore: 8,
|
|
@@ -10958,6 +11049,7 @@ const CRAWL_SKIP_REASONS = [
|
|
|
10958
11049
|
"configured-exclusion",
|
|
10959
11050
|
"duplicate-normalization",
|
|
10960
11051
|
"invalid-url",
|
|
11052
|
+
"memory-pressure",
|
|
10961
11053
|
"off-origin",
|
|
10962
11054
|
"origin-backpressure",
|
|
10963
11055
|
"queue-safety-limit",
|
|
@@ -10970,6 +11062,7 @@ const SKIP_IMPACT = {
|
|
|
10970
11062
|
"configured-exclusion": "non-impacting",
|
|
10971
11063
|
"duplicate-normalization": "non-impacting",
|
|
10972
11064
|
"invalid-url": "non-impacting",
|
|
11065
|
+
"memory-pressure": "coverage-affecting",
|
|
10973
11066
|
"off-origin": "non-impacting",
|
|
10974
11067
|
"origin-backpressure": "coverage-affecting",
|
|
10975
11068
|
"queue-safety-limit": "coverage-affecting",
|
|
@@ -11148,17 +11241,44 @@ function sanitizeTenantString(value) {
|
|
|
11148
11241
|
return value.replace(/https?:\/\/[^\s"'<>`]+/g, (match) => sanitizeUrlString(match)).replace(/\b(access_?token|api_?key|auth|client_?secret|jwt|password|refresh_?token|secret|session|signature|token)(\s*[=:]\s*)[^\s,;&]+/gi, (_match, key, separator) => `${key}${separator}[redacted]`).replace(/\b[A-Za-z]:\\[^\s"'<>`]+/g, "[local-path]").replace(/\/(?:Users|home|tmp|var\/folders)\/[^\s"'<>`]+/g, "[local-path]");
|
|
11149
11242
|
}
|
|
11150
11243
|
function sanitizeTenantValue(value, key) {
|
|
11151
|
-
if (key && isSensitiveFieldName(key)) return "[redacted]";
|
|
11244
|
+
if (key && isSensitiveFieldName(key)) return value === "[redacted]" ? value : "[redacted]";
|
|
11152
11245
|
if (typeof value === "string") return sanitizeTenantString(value);
|
|
11153
|
-
if (Array.isArray(value))
|
|
11246
|
+
if (Array.isArray(value)) {
|
|
11247
|
+
let sanitized;
|
|
11248
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
11249
|
+
const item = sanitizeTenantValue(value[index]);
|
|
11250
|
+
if (sanitized) sanitized.push(item);
|
|
11251
|
+
else if (item !== value[index]) sanitized = [...value.slice(0, index), item];
|
|
11252
|
+
}
|
|
11253
|
+
return sanitized ?? value;
|
|
11254
|
+
}
|
|
11154
11255
|
if (!value || typeof value !== "object") return value;
|
|
11155
|
-
|
|
11256
|
+
const record = value;
|
|
11257
|
+
let sanitized;
|
|
11258
|
+
for (const [entryKey, entry] of Object.entries(record)) {
|
|
11259
|
+
const safeEntry = sanitizeTenantValue(entry, entryKey);
|
|
11260
|
+
if (safeEntry === entry) continue;
|
|
11261
|
+
sanitized ??= { ...record };
|
|
11262
|
+
sanitized[entryKey] = safeEntry;
|
|
11263
|
+
}
|
|
11264
|
+
return sanitized ?? value;
|
|
11156
11265
|
}
|
|
11157
11266
|
function sanitizeCrawlConfig(config) {
|
|
11158
11267
|
return sanitizeTenantValue(config);
|
|
11159
11268
|
}
|
|
11160
11269
|
function sanitizePages(pages) {
|
|
11161
|
-
|
|
11270
|
+
const safePages = sanitizeTenantValue(pages);
|
|
11271
|
+
let normalized;
|
|
11272
|
+
for (let index = 0; index < safePages.length; index += 1) {
|
|
11273
|
+
const page = safePages[index];
|
|
11274
|
+
if (!page) continue;
|
|
11275
|
+
if ("seoScore" in page || "geoScore" in page) {
|
|
11276
|
+
normalized ??= safePages.slice(0, index);
|
|
11277
|
+
const { seoScore: _seoScore, geoScore: _geoScore, ...current } = page;
|
|
11278
|
+
normalized.push(current);
|
|
11279
|
+
} else if (normalized) normalized.push(page);
|
|
11280
|
+
}
|
|
11281
|
+
return normalized ?? safePages;
|
|
11162
11282
|
}
|
|
11163
11283
|
function sanitizeIssues(issues) {
|
|
11164
11284
|
return sanitizeTenantValue(issues);
|
|
@@ -11187,18 +11307,28 @@ function summarizeCrawlReport(input) {
|
|
|
11187
11307
|
const byCategory = {};
|
|
11188
11308
|
let responseMs = 0;
|
|
11189
11309
|
let responseCount = 0;
|
|
11310
|
+
let statusOnlyPages = 0;
|
|
11311
|
+
let indexablePages = 0;
|
|
11312
|
+
let nonIndexablePages = 0;
|
|
11313
|
+
let statusErrors = 0;
|
|
11190
11314
|
for (const page of input.pages) {
|
|
11191
11315
|
byStatus[String(page.status)] = (byStatus[String(page.status)] ?? 0) + 1;
|
|
11192
11316
|
if (typeof page.responseTimeMs === "number") {
|
|
11193
11317
|
responseMs += page.responseTimeMs;
|
|
11194
11318
|
responseCount += 1;
|
|
11195
11319
|
}
|
|
11320
|
+
if (page.auditScope === "status") statusOnlyPages += 1;
|
|
11321
|
+
else if (page.indexable) indexablePages += 1;
|
|
11322
|
+
else nonIndexablePages += 1;
|
|
11323
|
+
if (page.status === 0 || page.status >= 400) statusErrors += 1;
|
|
11196
11324
|
}
|
|
11197
11325
|
const requests = input.requests ?? [];
|
|
11198
|
-
const failedRequests = requests.filter((request) => request.outcome === "failure" && request.failureKind !== "aborted");
|
|
11199
|
-
const abortedRequests = requests.filter((request) => request.outcome === "failure" && request.failureKind === "aborted");
|
|
11200
11326
|
let requestMs = 0;
|
|
11201
11327
|
let requestMsCount = 0;
|
|
11328
|
+
let responseRequests = 0;
|
|
11329
|
+
let failedRequests = 0;
|
|
11330
|
+
let abortedRequests = 0;
|
|
11331
|
+
let extractionFailures = 0;
|
|
11202
11332
|
for (const request of requests) {
|
|
11203
11333
|
const status = request.outcome === "response" ? String(request.status) : request.outcome === "skipped" ? request.reason : request.failureKind === "aborted" ? "aborted" : "no-response";
|
|
11204
11334
|
requestByStatus[status] = (requestByStatus[status] ?? 0) + 1;
|
|
@@ -11206,27 +11336,39 @@ function summarizeCrawlReport(input) {
|
|
|
11206
11336
|
requestMs += request.durationMs;
|
|
11207
11337
|
requestMsCount += 1;
|
|
11208
11338
|
}
|
|
11339
|
+
if (request.outcome === "response") {
|
|
11340
|
+
responseRequests += 1;
|
|
11341
|
+
if (request.extraction === "failed") extractionFailures += 1;
|
|
11342
|
+
} else if (request.outcome === "failure") if (request.failureKind === "aborted") abortedRequests += 1;
|
|
11343
|
+
else failedRequests += 1;
|
|
11344
|
+
}
|
|
11345
|
+
let highIssues = 0;
|
|
11346
|
+
let mediumIssues = 0;
|
|
11347
|
+
let lowIssues = 0;
|
|
11348
|
+
for (const issue of input.issues) {
|
|
11349
|
+
byCategory[issue.category] = (byCategory[issue.category] ?? 0) + 1;
|
|
11350
|
+
if (issue.severity === "high") highIssues += 1;
|
|
11351
|
+
else if (issue.severity === "medium") mediumIssues += 1;
|
|
11352
|
+
else lowIssues += 1;
|
|
11209
11353
|
}
|
|
11210
|
-
for (const issue of input.issues) byCategory[issue.category] = (byCategory[issue.category] ?? 0) + 1;
|
|
11211
11354
|
const runStats = crawlRunStats(input.pages, input.stats);
|
|
11212
|
-
const indexabilityPages = input.pages.filter((page) => page.auditScope !== "status");
|
|
11213
11355
|
return {
|
|
11214
11356
|
totalPages: input.pages.length,
|
|
11215
|
-
statusOnlyPages
|
|
11216
|
-
indexablePages
|
|
11217
|
-
nonIndexablePages
|
|
11218
|
-
statusErrors
|
|
11357
|
+
statusOnlyPages,
|
|
11358
|
+
indexablePages,
|
|
11359
|
+
nonIndexablePages,
|
|
11360
|
+
statusErrors,
|
|
11219
11361
|
...runStats,
|
|
11220
11362
|
attemptedRequests: input.requestEvidenceStatus === "unavailable" ? 0 : requests.length,
|
|
11221
|
-
responseRequests
|
|
11222
|
-
failedRequests
|
|
11223
|
-
abortedRequests
|
|
11224
|
-
extractionFailures
|
|
11363
|
+
responseRequests,
|
|
11364
|
+
failedRequests,
|
|
11365
|
+
abortedRequests,
|
|
11366
|
+
extractionFailures,
|
|
11225
11367
|
requestByStatus,
|
|
11226
11368
|
avgRequestMs: requestMsCount ? Math.round(requestMs / requestMsCount) : void 0,
|
|
11227
|
-
highIssues
|
|
11228
|
-
mediumIssues
|
|
11229
|
-
lowIssues
|
|
11369
|
+
highIssues,
|
|
11370
|
+
mediumIssues,
|
|
11371
|
+
lowIssues,
|
|
11230
11372
|
avgResponseMs: responseCount ? Math.round(responseMs / responseCount) : void 0,
|
|
11231
11373
|
byStatus,
|
|
11232
11374
|
byCategory
|
|
@@ -11234,42 +11376,55 @@ function summarizeCrawlReport(input) {
|
|
|
11234
11376
|
}
|
|
11235
11377
|
const ACCESS_BLOCK_SAMPLE_LIMIT = 10;
|
|
11236
11378
|
function crawlAccessSummary(requests) {
|
|
11237
|
-
const blocked = requests.flatMap((request) => request.outcome === "response" && request.accessBlock ? [{
|
|
11238
|
-
url: request.requestedUrl,
|
|
11239
|
-
evidence: request.accessBlock
|
|
11240
|
-
}] : []);
|
|
11241
11379
|
const providers = {};
|
|
11242
|
-
|
|
11380
|
+
const samples = [];
|
|
11381
|
+
let blockedRequests = 0;
|
|
11382
|
+
for (const request of requests) {
|
|
11383
|
+
if (request.outcome !== "response" || !request.accessBlock) continue;
|
|
11384
|
+
blockedRequests += 1;
|
|
11385
|
+
providers[request.accessBlock.provider] = (providers[request.accessBlock.provider] ?? 0) + 1;
|
|
11386
|
+
if (samples.length < ACCESS_BLOCK_SAMPLE_LIMIT) samples.push({
|
|
11387
|
+
url: request.requestedUrl,
|
|
11388
|
+
evidence: request.accessBlock
|
|
11389
|
+
});
|
|
11390
|
+
}
|
|
11243
11391
|
return {
|
|
11244
11392
|
crawler: SEO_CRAWLER_IDENTITY,
|
|
11245
|
-
blockedRequests
|
|
11393
|
+
blockedRequests,
|
|
11246
11394
|
providers,
|
|
11247
|
-
samples
|
|
11395
|
+
samples,
|
|
11248
11396
|
sampleLimit: ACCESS_BLOCK_SAMPLE_LIMIT,
|
|
11249
|
-
truncated:
|
|
11397
|
+
truncated: blockedRequests > ACCESS_BLOCK_SAMPLE_LIMIT
|
|
11250
11398
|
};
|
|
11251
11399
|
}
|
|
11252
11400
|
function crawlRunStats(pages, stats = {}) {
|
|
11253
|
-
const
|
|
11401
|
+
const needsDiscoveredUrls = stats.discoveredUrls === void 0;
|
|
11402
|
+
const needsObservedInternalLinks = stats.observedInternalLinks === void 0;
|
|
11403
|
+
const needsFailedUrls = stats.failedUrls === void 0;
|
|
11404
|
+
const discovered = needsDiscoveredUrls ? /* @__PURE__ */ new Set() : void 0;
|
|
11254
11405
|
let observedInternalLinks = 0;
|
|
11255
|
-
|
|
11256
|
-
|
|
11257
|
-
|
|
11258
|
-
|
|
11259
|
-
|
|
11406
|
+
let failedUrls = 0;
|
|
11407
|
+
if (needsDiscoveredUrls || needsObservedInternalLinks || needsFailedUrls) for (const page of pages) {
|
|
11408
|
+
if (discovered) {
|
|
11409
|
+
discovered.add(page.url);
|
|
11410
|
+
for (const url of page.sampleInternalLinks ?? []) discovered.add(url);
|
|
11411
|
+
for (const url of page.sampleExternalLinks ?? []) discovered.add(url);
|
|
11412
|
+
}
|
|
11413
|
+
if (needsObservedInternalLinks) observedInternalLinks += page.outgoingInternalCount;
|
|
11414
|
+
if (needsFailedUrls && (page.status === 0 || page.status >= 400)) failedUrls += 1;
|
|
11260
11415
|
}
|
|
11261
11416
|
const skips = normalizeCrawlSkipReasonCounts({
|
|
11262
11417
|
skippedUrls: stats.skippedUrls,
|
|
11263
11418
|
skipReasons: stats.skipReasons ?? crawlSkipReasonCountsFromRecord(stats.skipReasonCounts)
|
|
11264
11419
|
});
|
|
11265
11420
|
return {
|
|
11266
|
-
discoveredUrls: stats.discoveredUrls ?? discovered
|
|
11421
|
+
discoveredUrls: stats.discoveredUrls ?? discovered?.size ?? 0,
|
|
11267
11422
|
queuedUrls: stats.queuedUrls ?? pages.length,
|
|
11268
11423
|
crawledUrls: stats.crawledUrls ?? pages.length,
|
|
11269
11424
|
skippedUrls: skips.skippedUrls,
|
|
11270
11425
|
skipReasons: skips.skipReasons,
|
|
11271
11426
|
skippedUrlsByImpact: skips.skippedUrlsByImpact,
|
|
11272
|
-
failedUrls: stats.failedUrls ??
|
|
11427
|
+
failedUrls: stats.failedUrls ?? failedUrls,
|
|
11273
11428
|
observedInternalLinks: stats.observedInternalLinks ?? observedInternalLinks,
|
|
11274
11429
|
pageLimitReached: stats.pageLimitReached ?? false
|
|
11275
11430
|
};
|
|
@@ -11286,11 +11441,14 @@ function normalizeLinkUrl(value) {
|
|
|
11286
11441
|
function deriveInternalLinkAuthority(pages, linkGraph) {
|
|
11287
11442
|
const hasLinkGraph = linkGraph && Object.keys(linkGraph).length > 0;
|
|
11288
11443
|
const hasStoredScores = pages.some((page) => page.internalInlinkCount !== void 0 || page.internalLinkAuthorityScore !== void 0);
|
|
11289
|
-
if (!hasLinkGraph && hasStoredScores)
|
|
11290
|
-
|
|
11291
|
-
|
|
11292
|
-
|
|
11293
|
-
|
|
11444
|
+
if (!hasLinkGraph && hasStoredScores) {
|
|
11445
|
+
if (pages.every((page) => page.internalInlinkCount !== void 0 && page.internalLinkAuthorityScore !== void 0)) return pages;
|
|
11446
|
+
return pages.map((page) => page.internalInlinkCount !== void 0 && page.internalLinkAuthorityScore !== void 0 ? page : {
|
|
11447
|
+
...page,
|
|
11448
|
+
internalInlinkCount: page.internalInlinkCount ?? 0,
|
|
11449
|
+
internalLinkAuthorityScore: page.internalLinkAuthorityScore ?? 0
|
|
11450
|
+
});
|
|
11451
|
+
}
|
|
11294
11452
|
const pageByUrl = /* @__PURE__ */ new Map();
|
|
11295
11453
|
for (const page of pages) for (const value of [page.url, page.finalUrl]) {
|
|
11296
11454
|
const normalized = normalizeLinkUrl(value);
|
|
@@ -11309,7 +11467,8 @@ function deriveInternalLinkAuthority(pages, linkGraph) {
|
|
|
11309
11467
|
}
|
|
11310
11468
|
for (const linkedPage of linkedPages) inlinks.set(linkedPage, (inlinks.get(linkedPage) ?? 0) + 1);
|
|
11311
11469
|
}
|
|
11312
|
-
|
|
11470
|
+
let maxInlinks = 0;
|
|
11471
|
+
for (const count of inlinks.values()) maxInlinks = Math.max(maxInlinks, count);
|
|
11313
11472
|
return pages.map((page) => {
|
|
11314
11473
|
const internalInlinkCount = inlinks.get(page.url) ?? 0;
|
|
11315
11474
|
return {
|
|
@@ -11567,6 +11726,7 @@ function crawlCaveats(input) {
|
|
|
11567
11726
|
const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
|
|
11568
11727
|
if (input.queueSafetySkippedUrls > 0) caveats.push(`Left ${input.queueSafetySkippedUrls} eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
|
|
11569
11728
|
if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
|
|
11729
|
+
if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
|
|
11570
11730
|
return caveats;
|
|
11571
11731
|
}
|
|
11572
11732
|
//#endregion
|
|
@@ -11593,6 +11753,26 @@ function cancellationRace(signal) {
|
|
|
11593
11753
|
});
|
|
11594
11754
|
}
|
|
11595
11755
|
//#endregion
|
|
11756
|
+
//#region packages/core/src/analyze/crawler/crawl-memory.ts
|
|
11757
|
+
const MEBIBYTE$1 = 1024 * 1024;
|
|
11758
|
+
const MIN_CRAWL_MEMORY_LIMIT = 384 * MEBIBYTE$1;
|
|
11759
|
+
const MAX_CRAWL_MEMORY_LIMIT = 640 * MEBIBYTE$1;
|
|
11760
|
+
const MIN_CRAWL_RSS_LIMIT = 512 * MEBIBYTE$1;
|
|
11761
|
+
const MAX_CRAWL_RSS_LIMIT = 896 * MEBIBYTE$1;
|
|
11762
|
+
const SYSTEM_MEMORY_FRACTION = .1;
|
|
11763
|
+
const SYSTEM_RSS_FRACTION = .2;
|
|
11764
|
+
function crawlMemoryLimitBytes(totalMemoryBytes) {
|
|
11765
|
+
if (!Number.isFinite(totalMemoryBytes) || totalMemoryBytes <= 0) return MAX_CRAWL_MEMORY_LIMIT;
|
|
11766
|
+
return Math.min(MAX_CRAWL_MEMORY_LIMIT, Math.max(MIN_CRAWL_MEMORY_LIMIT, Math.floor(totalMemoryBytes * SYSTEM_MEMORY_FRACTION)));
|
|
11767
|
+
}
|
|
11768
|
+
function crawlRssLimitBytes(totalMemoryBytes) {
|
|
11769
|
+
if (!Number.isFinite(totalMemoryBytes) || totalMemoryBytes <= 0) return MAX_CRAWL_RSS_LIMIT;
|
|
11770
|
+
return Math.min(MAX_CRAWL_RSS_LIMIT, Math.max(MIN_CRAWL_RSS_LIMIT, Math.floor(totalMemoryBytes * SYSTEM_RSS_FRACTION)));
|
|
11771
|
+
}
|
|
11772
|
+
function crawlMemoryPressure(input) {
|
|
11773
|
+
return input.memoryUsage.heapUsed + input.memoryUsage.external >= crawlMemoryLimitBytes(input.totalMemoryBytes) || input.memoryUsage.rss >= crawlRssLimitBytes(input.totalMemoryBytes);
|
|
11774
|
+
}
|
|
11775
|
+
//#endregion
|
|
11596
11776
|
//#region packages/core/src/analyze/crawler/crawl-url-queue.ts
|
|
11597
11777
|
function before(left, right) {
|
|
11598
11778
|
return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
|
|
@@ -12310,6 +12490,7 @@ function crawlProviderLimits(input) {
|
|
|
12310
12490
|
}
|
|
12311
12491
|
async function crawlDataSources(input) {
|
|
12312
12492
|
const window = crawlMetricsWindow(input.now());
|
|
12493
|
+
const skippedForMemory = input.skipReason === "memory-pressure";
|
|
12313
12494
|
const dataSources = {
|
|
12314
12495
|
searchConsole: {
|
|
12315
12496
|
status: "skipped",
|
|
@@ -12320,7 +12501,7 @@ async function crawlDataSources(input) {
|
|
|
12320
12501
|
pageLimit: input.searchMetricsLimit,
|
|
12321
12502
|
pageLimitReached: false,
|
|
12322
12503
|
retainedRowLimitReached: false,
|
|
12323
|
-
warning: input.cancelled ? "Search Console join skipped because the crawl was cancelled." : "Search Console join skipped because no property was selected."
|
|
12504
|
+
warning: skippedForMemory ? "Search Console join skipped after the local memory safety limit was reached." : input.skipReason === "cancelled" ? "Search Console join skipped because the crawl was cancelled." : "Search Console join skipped because no property was selected."
|
|
12324
12505
|
},
|
|
12325
12506
|
analytics: {
|
|
12326
12507
|
status: "skipped",
|
|
@@ -12329,10 +12510,10 @@ async function crawlDataSources(input) {
|
|
|
12329
12510
|
joinedPages: 0,
|
|
12330
12511
|
retainedRowLimit: input.analyticsLimit,
|
|
12331
12512
|
retainedRowLimitReached: false,
|
|
12332
|
-
warning: input.cancelled ? "Google Analytics join skipped because the crawl was cancelled." : "Google Analytics join skipped because no property was selected."
|
|
12513
|
+
warning: skippedForMemory ? "Google Analytics join skipped after the local memory safety limit was reached." : input.skipReason === "cancelled" ? "Google Analytics join skipped because the crawl was cancelled." : "Google Analytics join skipped because no property was selected."
|
|
12333
12514
|
}
|
|
12334
12515
|
};
|
|
12335
|
-
if (!input.
|
|
12516
|
+
if (!input.skipReason && input.site) dataSources.searchConsole = await joinSearchMetrics({
|
|
12336
12517
|
site: input.site,
|
|
12337
12518
|
pages: input.pages,
|
|
12338
12519
|
warnings: input.warnings,
|
|
@@ -12345,7 +12526,7 @@ async function crawlDataSources(input) {
|
|
|
12345
12526
|
queryPagesMetricsBatch: input.queryPagesMetricsBatch,
|
|
12346
12527
|
queryPagesTopQueriesBatch: input.queryPagesTopQueriesBatch
|
|
12347
12528
|
});
|
|
12348
|
-
if (!input.
|
|
12529
|
+
if (!input.skipReason && input.googleAnalyticsPropertyId) dataSources.analytics = await joinAnalytics({
|
|
12349
12530
|
propertyId: input.googleAnalyticsPropertyId,
|
|
12350
12531
|
pages: input.pages,
|
|
12351
12532
|
warnings: input.warnings,
|
|
@@ -12358,17 +12539,6 @@ async function crawlDataSources(input) {
|
|
|
12358
12539
|
}
|
|
12359
12540
|
//#endregion
|
|
12360
12541
|
//#region packages/core/src/analyze/monitoring/crawl-page.ts
|
|
12361
|
-
function sameOriginUrl(href, base) {
|
|
12362
|
-
try {
|
|
12363
|
-
const url = new URL(href, base);
|
|
12364
|
-
url.hash = "";
|
|
12365
|
-
if (url.origin !== base.origin) return void 0;
|
|
12366
|
-
if (!["http:", "https:"].includes(url.protocol)) return void 0;
|
|
12367
|
-
return url.toString();
|
|
12368
|
-
} catch {
|
|
12369
|
-
return;
|
|
12370
|
-
}
|
|
12371
|
-
}
|
|
12372
12542
|
function resolvedUrl(href, base) {
|
|
12373
12543
|
if (!href) return void 0;
|
|
12374
12544
|
try {
|
|
@@ -12398,49 +12568,6 @@ function safeResponseHeaders(headers) {
|
|
|
12398
12568
|
function truncate$1(value, max) {
|
|
12399
12569
|
return value.length > max ? value.slice(0, max).trimEnd() : value;
|
|
12400
12570
|
}
|
|
12401
|
-
function anchorSamples(links, internal) {
|
|
12402
|
-
const seen = /* @__PURE__ */ new Set();
|
|
12403
|
-
const samples = [];
|
|
12404
|
-
for (const link of links) {
|
|
12405
|
-
if (link.internal !== internal) continue;
|
|
12406
|
-
const text = truncate$1(link.text.replace(/\s+/g, " ").trim(), 120);
|
|
12407
|
-
if (!text) continue;
|
|
12408
|
-
const key = `${link.href}\n${text}`;
|
|
12409
|
-
if (seen.has(key)) continue;
|
|
12410
|
-
seen.add(key);
|
|
12411
|
-
samples.push({
|
|
12412
|
-
href: link.href,
|
|
12413
|
-
text
|
|
12414
|
-
});
|
|
12415
|
-
if (samples.length >= 25) break;
|
|
12416
|
-
}
|
|
12417
|
-
return samples;
|
|
12418
|
-
}
|
|
12419
|
-
function socialProfileLinks(links) {
|
|
12420
|
-
const hosts = [
|
|
12421
|
-
"facebook.com",
|
|
12422
|
-
"instagram.com",
|
|
12423
|
-
"linkedin.com",
|
|
12424
|
-
"pinterest.com",
|
|
12425
|
-
"tiktok.com",
|
|
12426
|
-
"twitter.com",
|
|
12427
|
-
"x.com",
|
|
12428
|
-
"youtube.com"
|
|
12429
|
-
];
|
|
12430
|
-
const profiles = /* @__PURE__ */ new Set();
|
|
12431
|
-
for (const link of links) {
|
|
12432
|
-
if (link.internal) continue;
|
|
12433
|
-
try {
|
|
12434
|
-
const url = new URL(link.href);
|
|
12435
|
-
const host = url.hostname.replace(/^www\./, "").toLowerCase();
|
|
12436
|
-
if (hosts.some((value) => host === value || host.endsWith(`.${value}`))) {
|
|
12437
|
-
url.hash = "";
|
|
12438
|
-
profiles.add(url.toString());
|
|
12439
|
-
}
|
|
12440
|
-
} catch {}
|
|
12441
|
-
}
|
|
12442
|
-
return [...profiles].slice(0, 50);
|
|
12443
|
-
}
|
|
12444
12571
|
function sameDocumentUrl(left, right) {
|
|
12445
12572
|
if (!left || !right) return false;
|
|
12446
12573
|
try {
|
|
@@ -12540,17 +12667,9 @@ function statusOnlyPage(fetched) {
|
|
|
12540
12667
|
finalUrl: fetched.finalUrl,
|
|
12541
12668
|
status: fetched.status,
|
|
12542
12669
|
contentType,
|
|
12543
|
-
responseHeaders: safeResponseHeaders(fetched.headers),
|
|
12544
12670
|
responseTimeMs: fetched.diagnostics.durationMs,
|
|
12545
|
-
usedJs: false,
|
|
12546
|
-
fetchSource: "network",
|
|
12547
|
-
cacheState: "bypass",
|
|
12548
|
-
fetchDiagnostics: fetched.diagnostics,
|
|
12549
12671
|
auditScope: "status",
|
|
12550
|
-
accessBlock: fetched.diagnostics.accessBlock,
|
|
12551
|
-
blocked: fetched.diagnostics.blocked,
|
|
12552
12672
|
contentAuditAllowed: false,
|
|
12553
|
-
robotsTxt: fetched.robotsTxt,
|
|
12554
12673
|
indexable: false,
|
|
12555
12674
|
indexability: "Not evaluated by status-only health probe",
|
|
12556
12675
|
declaredIndexability: "unknown",
|
|
@@ -12558,8 +12677,7 @@ function statusOnlyPage(fetched) {
|
|
|
12558
12677
|
wordCount: 0,
|
|
12559
12678
|
contentHash: "",
|
|
12560
12679
|
outgoingInternalCount: 0,
|
|
12561
|
-
outgoingExternalCount: 0
|
|
12562
|
-
isHttps: new URL(fetched.finalUrl).protocol === "https:"
|
|
12680
|
+
outgoingExternalCount: 0
|
|
12563
12681
|
};
|
|
12564
12682
|
}
|
|
12565
12683
|
function indexabilityReason(page) {
|
|
@@ -12635,12 +12753,11 @@ async function crawlOne(url, opts = {}) {
|
|
|
12635
12753
|
};
|
|
12636
12754
|
try {
|
|
12637
12755
|
const extracted = await extractPage(fetched, "crawler");
|
|
12638
|
-
const
|
|
12639
|
-
|
|
12640
|
-
const
|
|
12641
|
-
const
|
|
12642
|
-
const
|
|
12643
|
-
const uniqueExternalLinks = [...new Set(externalLinks)];
|
|
12756
|
+
const crawlerEvidence = extracted.crawlerEvidence;
|
|
12757
|
+
if (!crawlerEvidence) throw new Error("Crawler extraction did not return crawl evidence.");
|
|
12758
|
+
const h1 = crawlerEvidence.h1;
|
|
12759
|
+
const uniqueInternalLinks = crawlerEvidence.internalLinks;
|
|
12760
|
+
const uniqueExternalLinks = crawlerEvidence.externalLinks;
|
|
12644
12761
|
const canonical = extracted.canonicalEvidence?.selectedUrl ?? resolvedUrl(extracted.canonical, extracted.finalUrl);
|
|
12645
12762
|
const richResults = selectGoogleRichResultAssessments(extracted.googleRichResults ?? []);
|
|
12646
12763
|
const page = {
|
|
@@ -12666,9 +12783,9 @@ async function crawlOne(url, opts = {}) {
|
|
|
12666
12783
|
metaRobots: extracted.metaRobots,
|
|
12667
12784
|
xRobotsTag: extracted.xRobotsTag,
|
|
12668
12785
|
h1,
|
|
12669
|
-
h1Count:
|
|
12670
|
-
h2Count:
|
|
12671
|
-
h3Count:
|
|
12786
|
+
h1Count: crawlerEvidence.h1Count,
|
|
12787
|
+
h2Count: crawlerEvidence.h2Count,
|
|
12788
|
+
h3Count: crawlerEvidence.h3Count,
|
|
12672
12789
|
indexable: false,
|
|
12673
12790
|
extractionStatus: "complete",
|
|
12674
12791
|
wordCount: extracted.wordCount,
|
|
@@ -12704,8 +12821,8 @@ async function crawlOne(url, opts = {}) {
|
|
|
12704
12821
|
outgoingExternalCount: uniqueExternalLinks.length,
|
|
12705
12822
|
sampleInternalLinks: uniqueInternalLinks.slice(0, 25),
|
|
12706
12823
|
sampleExternalLinks: uniqueExternalLinks.slice(0, 25),
|
|
12707
|
-
internalAnchorSamples:
|
|
12708
|
-
externalAnchorSamples:
|
|
12824
|
+
internalAnchorSamples: crawlerEvidence.internalAnchorSamples,
|
|
12825
|
+
externalAnchorSamples: crawlerEvidence.externalAnchorSamples,
|
|
12709
12826
|
markdownAlternates: extracted.markdownAlternates,
|
|
12710
12827
|
schemaTypes: extracted.schemaTypes,
|
|
12711
12828
|
structuredDataFormats: extracted.structuredDataFormats ?? [],
|
|
@@ -12714,7 +12831,7 @@ async function crawlOne(url, opts = {}) {
|
|
|
12714
12831
|
schemaSameAs: [...new Set((extracted.schemaSameAsEvidence ?? []).map((evidence) => evidence.url))].slice(0, 50),
|
|
12715
12832
|
schemaSameAsEvidence: (extracted.schemaSameAsEvidence ?? []).slice(0, 50),
|
|
12716
12833
|
invalidSchemaSameAs: (extracted.invalidSchemaSameAs ?? []).slice(0, 25),
|
|
12717
|
-
socialProfileLinks: socialProfileLinks
|
|
12834
|
+
socialProfileLinks: crawlerEvidence.socialProfileLinks,
|
|
12718
12835
|
invalidJsonLdCount: extracted.invalidJsonLdCount,
|
|
12719
12836
|
invalidJsonLdSamples: extracted.invalidJsonLdSamples,
|
|
12720
12837
|
unrecognizedJsonLdTypes: extracted.unrecognizedJsonLdTypes ?? [],
|
|
@@ -13367,7 +13484,9 @@ function resolveCrawlSiteDependencies(dependencies = {}) {
|
|
|
13367
13484
|
fetchLandingPageValues: dependencies.fetchLandingPageValues ?? fetchLandingPageValues,
|
|
13368
13485
|
landingValueForUrl: dependencies.landingValueForUrl ?? landingValueForUrl,
|
|
13369
13486
|
collectAgentDiscovery: dependencies.collectAgentDiscovery ?? collectAgentDiscovery,
|
|
13370
|
-
now: dependencies.now ?? (() => /* @__PURE__ */ new Date())
|
|
13487
|
+
now: dependencies.now ?? (() => /* @__PURE__ */ new Date()),
|
|
13488
|
+
memoryUsage: dependencies.memoryUsage ?? (() => process.memoryUsage()),
|
|
13489
|
+
totalMemory: dependencies.totalMemory ?? totalmem
|
|
13371
13490
|
};
|
|
13372
13491
|
}
|
|
13373
13492
|
//#endregion
|
|
@@ -13523,15 +13642,26 @@ async function crawlSite(input, dependencies) {
|
|
|
13523
13642
|
let queuedUrls = 0;
|
|
13524
13643
|
let skippedUrls = 0;
|
|
13525
13644
|
const skipReasonCounts = {};
|
|
13526
|
-
const
|
|
13527
|
-
|
|
13528
|
-
|
|
13645
|
+
const recordSkips = (reason, count = 1) => {
|
|
13646
|
+
if (count <= 0) return;
|
|
13647
|
+
skippedUrls += count;
|
|
13648
|
+
skipReasonCounts[reason] = (skipReasonCounts[reason] ?? 0) + count;
|
|
13529
13649
|
};
|
|
13650
|
+
const recordSkip = (reason) => recordSkips(reason);
|
|
13530
13651
|
let robotsDeferredUrls = 0;
|
|
13531
13652
|
let originBackpressureSkippedUrls = 0;
|
|
13532
13653
|
let queueSafetySkippedUrls = 0;
|
|
13533
13654
|
let failedUrls = 0;
|
|
13534
13655
|
let observedInternalLinks = 0;
|
|
13656
|
+
let memoryLimited = false;
|
|
13657
|
+
const memoryDeferredUrls = /* @__PURE__ */ new Set();
|
|
13658
|
+
const checkMemoryPressure = () => {
|
|
13659
|
+
if (memoryLimited) return;
|
|
13660
|
+
memoryLimited = crawlMemoryPressure({
|
|
13661
|
+
memoryUsage: deps.memoryUsage(),
|
|
13662
|
+
totalMemoryBytes: deps.totalMemory()
|
|
13663
|
+
});
|
|
13664
|
+
};
|
|
13535
13665
|
let sitemapDiscovery;
|
|
13536
13666
|
let externalLinkVerification;
|
|
13537
13667
|
let statusEventChain = Promise.resolve();
|
|
@@ -13661,6 +13791,7 @@ async function crawlSite(input, dependencies) {
|
|
|
13661
13791
|
for (const url of sitemap.urls) enqueue(url, 0);
|
|
13662
13792
|
sitemapDiscovery = sitemap.discovery;
|
|
13663
13793
|
}
|
|
13794
|
+
checkMemoryPressure();
|
|
13664
13795
|
const nextQueueItem = () => {
|
|
13665
13796
|
while (queue.size) {
|
|
13666
13797
|
const item = queue.take();
|
|
@@ -13686,6 +13817,7 @@ async function crawlSite(input, dependencies) {
|
|
|
13686
13817
|
timeoutMs: config.timeoutMs,
|
|
13687
13818
|
rate: config.fetchRate,
|
|
13688
13819
|
signal,
|
|
13820
|
+
writeCache: false,
|
|
13689
13821
|
respectRobots: config.respectRobots,
|
|
13690
13822
|
robotsResolver,
|
|
13691
13823
|
renderer
|
|
@@ -13713,8 +13845,8 @@ async function crawlSite(input, dependencies) {
|
|
|
13713
13845
|
depth: item.depth
|
|
13714
13846
|
});
|
|
13715
13847
|
};
|
|
13716
|
-
while (!isCancelled() && (queue.size || inFlight.size) && pages.length < config.maxPages) {
|
|
13717
|
-
while (!isCancelled() && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
|
|
13848
|
+
while (!isCancelled() && (queue.size || inFlight.size) && (!memoryLimited || inFlight.size > 0) && pages.length < config.maxPages) {
|
|
13849
|
+
while (!isCancelled() && !memoryLimited && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
|
|
13718
13850
|
const item = nextQueueItem();
|
|
13719
13851
|
if (!item) break;
|
|
13720
13852
|
startTask(item);
|
|
@@ -13742,6 +13874,7 @@ async function crawlSite(input, dependencies) {
|
|
|
13742
13874
|
healthyProbeStreak = 0;
|
|
13743
13875
|
activeConcurrency = 1;
|
|
13744
13876
|
}
|
|
13877
|
+
checkMemoryPressure();
|
|
13745
13878
|
if (request.outcome === "skipped") {
|
|
13746
13879
|
recordSkip(request.reason === "robots-disallowed" ? "robots-disallowed" : request.reason === "robots-deferred" ? "robots-uncertain" : "origin-backpressure");
|
|
13747
13880
|
if (request.reason === "robots-deferred") {
|
|
@@ -13872,10 +14005,24 @@ async function crawlSite(input, dependencies) {
|
|
|
13872
14005
|
depth: task.depth,
|
|
13873
14006
|
statusCode: page.status
|
|
13874
14007
|
});
|
|
13875
|
-
if (followLinks && task.depth < config.maxDepth) for (const
|
|
14008
|
+
if (followLinks && task.depth < config.maxDepth) if (memoryLimited) for (const value of result.urls) {
|
|
14009
|
+
const normalized = normalizeUrl$1(value, finalUrl);
|
|
14010
|
+
if (normalized && sameOrigin(normalized, origin) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
|
|
14011
|
+
discovered.add(normalized);
|
|
14012
|
+
memoryDeferredUrls.add(normalized);
|
|
14013
|
+
}
|
|
14014
|
+
}
|
|
14015
|
+
else for (const url of result.urls) enqueue(url, task.depth + 1);
|
|
14016
|
+
}
|
|
14017
|
+
const memoryPressureSkippedUrls = memoryLimited ? queue.size + memoryDeferredUrls.size : 0;
|
|
14018
|
+
const stoppedForMemory = memoryPressureSkippedUrls > 0;
|
|
14019
|
+
if (stoppedForMemory) {
|
|
14020
|
+
recordSkips("memory-pressure", memoryPressureSkippedUrls);
|
|
14021
|
+
warnings.push("Crawl stopped before local memory use could become unsafe. The retained report is partial.");
|
|
13876
14022
|
}
|
|
13877
14023
|
const resolvedInlinkCounts = resolveLinkCountAliases(incomingLinkCounts, fetchedAliases);
|
|
13878
|
-
|
|
14024
|
+
let maxInlinks = 0;
|
|
14025
|
+
for (const page of pages) maxInlinks = Math.max(maxInlinks, resolvedInlinkCounts.get(page.finalUrl) ?? 0);
|
|
13879
14026
|
for (const page of pages) {
|
|
13880
14027
|
const internalInlinkCount = resolvedInlinkCounts.get(page.finalUrl) ?? 0;
|
|
13881
14028
|
page.internalInlinkCount = internalInlinkCount;
|
|
@@ -13895,11 +14042,11 @@ async function crawlSite(input, dependencies) {
|
|
|
13895
14042
|
const sitemapEvidencePartial = sitemapDiscovery?.dataStatus === "partial";
|
|
13896
14043
|
const accessEvidencePartial = requests.some((request) => request.outcome === "response" && request.accessBlock);
|
|
13897
14044
|
const requiredSitemapUnavailable = config.mode === "sitemap" && sitemapDiscovery?.dataStatus === "unavailable";
|
|
13898
|
-
const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
|
|
14045
|
+
const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
|
|
13899
14046
|
const failed = !cancelled && pages.length === 0 && (requiredSitemapUnavailable || requests.length > 0 && requests.every((request) => request.outcome === "failure"));
|
|
13900
14047
|
const requestEvidenceStatus = cancelled && inFlight.size > 0 ? "partial" : "available";
|
|
13901
14048
|
const dataSources = await crawlDataSources({
|
|
13902
|
-
cancelled,
|
|
14049
|
+
skipReason: cancelled ? "cancelled" : stoppedForMemory ? "memory-pressure" : void 0,
|
|
13903
14050
|
site: healthStrategy ? void 0 : site,
|
|
13904
14051
|
googleAnalyticsPropertyId: healthStrategy ? void 0 : input.googleAnalyticsPropertyId,
|
|
13905
14052
|
pages,
|
|
@@ -13918,7 +14065,7 @@ async function crawlSite(input, dependencies) {
|
|
|
13918
14065
|
});
|
|
13919
14066
|
const sourceEvidencePartial = [dataSources.searchConsole.status, dataSources.analytics.status].some((status) => status === "partial" || status === "unavailable");
|
|
13920
14067
|
const reportStatus = failed ? "failed" : partial || sourceEvidencePartial ? "partial" : "completed";
|
|
13921
|
-
if (!cancelled && config.checkExternal) {
|
|
14068
|
+
if (!cancelled && !stoppedForMemory && config.checkExternal) {
|
|
13922
14069
|
emitStatus("external_links_started", {
|
|
13923
14070
|
url: config.url,
|
|
13924
14071
|
message: "Started external link checks."
|
|
@@ -13935,7 +14082,7 @@ async function crawlSite(input, dependencies) {
|
|
|
13935
14082
|
message: "Finished external link checks."
|
|
13936
14083
|
});
|
|
13937
14084
|
}
|
|
13938
|
-
const agentDiscovery = !cancelled && config.checkAgentDiscovery ? await deps.collectAgentDiscovery({
|
|
14085
|
+
const agentDiscovery = !cancelled && !stoppedForMemory && config.checkAgentDiscovery ? await deps.collectAgentDiscovery({
|
|
13939
14086
|
startUrl: config.url,
|
|
13940
14087
|
pages,
|
|
13941
14088
|
timeoutMs: config.timeoutMs,
|
|
@@ -13976,7 +14123,8 @@ async function crawlSite(input, dependencies) {
|
|
|
13976
14123
|
pageLimitReached,
|
|
13977
14124
|
maxPages: config.maxPages,
|
|
13978
14125
|
queueSafetySkippedUrls,
|
|
13979
|
-
originBackpressureSkippedUrls
|
|
14126
|
+
originBackpressureSkippedUrls,
|
|
14127
|
+
memoryPressureSkippedUrls
|
|
13980
14128
|
}),
|
|
13981
14129
|
stats: {
|
|
13982
14130
|
discoveredUrls: discovered.size,
|
|
@@ -15463,19 +15611,19 @@ function renderCrawlHtml(report, fixes = topFixes(report)) {
|
|
|
15463
15611
|
const reviews = reviewObservations(report);
|
|
15464
15612
|
const issueRows = report.issueGroups.map((group) => {
|
|
15465
15613
|
const rule = explainRule(group.ruleId);
|
|
15466
|
-
return `<tr><td>${escapeHtml(group.severity)}</td><td>${escapeHtml(group.title)}</td><td>${group.count}</td><td>${escapeHtml(rule?.howToFix ?? "")}</td><td>${escapeHtml(group.sampleUrls[0] ?? "")}</td></tr>`;
|
|
15614
|
+
return `<tr><td>${escapeHtml$1(group.severity)}</td><td>${escapeHtml$1(group.title)}</td><td>${group.count}</td><td>${escapeHtml$1(rule?.howToFix ?? "")}</td><td>${escapeHtml$1(group.sampleUrls[0] ?? "")}</td></tr>`;
|
|
15467
15615
|
}).join("\n");
|
|
15468
|
-
const fixItems = fixes.slice(0, 10).map((fix) => `<li><strong>${escapeHtml(fix.title)}</strong><br>${escapeHtml(fix.howToFix)}<br><span>Verify: ${escapeHtml(fix.howToVerify)}</span><br><code>${escapeHtml(fix.verification.command)}</code></li>`).join("\n");
|
|
15469
|
-
const reviewItems = reviews.slice(0, 10).map((observation) => `<li><strong>${escapeHtml(observation.title)}</strong><br>${escapeHtml(observation.howToFix)}<br><span>Check before scheduling work. ${escapeHtml(observation.howToVerify)}</span></li>`).join("\n");
|
|
15470
|
-
const caveatItems = report.caveats.map((caveat) => `<li>${escapeHtml(caveat)}</li>`).join("\n");
|
|
15471
|
-
const warningItems = report.warnings.map((warning) => `<li>${escapeHtml(warning)}</li>`).join("\n");
|
|
15472
|
-
const accessItems = report.access.samples.map((sample) => `<li><strong>${escapeHtml(sample.url)}</strong><br>${escapeHtml(sample.evidence.guidance.summary)}<br>${escapeHtml(sample.evidence.guidance.recommendedAction)}<br><span>${escapeHtml(sample.evidence.guidance.securityNote)}</span></li>`).join("\n");
|
|
15616
|
+
const fixItems = fixes.slice(0, 10).map((fix) => `<li><strong>${escapeHtml$1(fix.title)}</strong><br>${escapeHtml$1(fix.howToFix)}<br><span>Verify: ${escapeHtml$1(fix.howToVerify)}</span><br><code>${escapeHtml$1(fix.verification.command)}</code></li>`).join("\n");
|
|
15617
|
+
const reviewItems = reviews.slice(0, 10).map((observation) => `<li><strong>${escapeHtml$1(observation.title)}</strong><br>${escapeHtml$1(observation.howToFix)}<br><span>Check before scheduling work. ${escapeHtml$1(observation.howToVerify)}</span></li>`).join("\n");
|
|
15618
|
+
const caveatItems = report.caveats.map((caveat) => `<li>${escapeHtml$1(caveat)}</li>`).join("\n");
|
|
15619
|
+
const warningItems = report.warnings.map((warning) => `<li>${escapeHtml$1(warning)}</li>`).join("\n");
|
|
15620
|
+
const accessItems = report.access.samples.map((sample) => `<li><strong>${escapeHtml$1(sample.url)}</strong><br>${escapeHtml$1(sample.evidence.guidance.summary)}<br>${escapeHtml$1(sample.evidence.guidance.recommendedAction)}<br><span>${escapeHtml$1(sample.evidence.guidance.securityNote)}</span></li>`).join("\n");
|
|
15473
15621
|
return `<!doctype html>
|
|
15474
15622
|
<html lang="en">
|
|
15475
15623
|
<head>
|
|
15476
15624
|
<meta charset="utf-8">
|
|
15477
15625
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
15478
|
-
<title>Crawl report for ${escapeHtml(report.config.url)}</title>
|
|
15626
|
+
<title>Crawl report for ${escapeHtml$1(report.config.url)}</title>
|
|
15479
15627
|
<style>
|
|
15480
15628
|
body { color: #17202a; font: 15px/1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; margin: 32px; max-width: 1120px; }
|
|
15481
15629
|
h1, h2 { line-height: 1.2; }
|
|
@@ -15491,10 +15639,10 @@ function renderCrawlHtml(report, fixes = topFixes(report)) {
|
|
|
15491
15639
|
</head>
|
|
15492
15640
|
<body>
|
|
15493
15641
|
<h1>Crawl report</h1>
|
|
15494
|
-
<p>${escapeHtml(report.config.url)}</p>
|
|
15495
|
-
<p>Strategy: ${escapeHtml(report.config.strategy === "health" ? "sitemap health pass" : "full audit")}<br>Crawler: <code>${escapeHtml(report.access.crawler.userAgent)}</code></p>
|
|
15642
|
+
<p>${escapeHtml$1(report.config.url)}</p>
|
|
15643
|
+
<p>Strategy: ${escapeHtml$1(report.config.strategy === "health" ? "sitemap health pass" : "full audit")}<br>Crawler: <code>${escapeHtml$1(report.access.crawler.userAgent)}</code></p>
|
|
15496
15644
|
<section class="summary">
|
|
15497
|
-
<div class="metric"><span>Status</span><strong>${escapeHtml(report.status)}</strong></div>
|
|
15645
|
+
<div class="metric"><span>Status</span><strong>${escapeHtml$1(report.status)}</strong></div>
|
|
15498
15646
|
<div class="metric"><span>Documents</span><strong>${report.summary.totalPages}</strong></div>
|
|
15499
15647
|
<div class="metric"><span>Requests</span><strong>${report.requestEvidenceStatus === "unavailable" ? "n/a" : report.requests.length}${report.requestEvidenceStatus === "partial" ? " (partial)" : ""}</strong></div>
|
|
15500
15648
|
<div class="metric"><span>Discovered</span><strong>${report.summary.discoveredUrls}</strong></div>
|
|
@@ -15548,11 +15696,11 @@ function renderCrawlMarkdownTickets(report, fixes = topFixes(report)) {
|
|
|
15548
15696
|
if (reviews.length) lines.push("## Review observations", "", "These are not implementation tickets. Check the evidence before deciding whether to work on them.", "", ...reviews.slice(0, 10).flatMap((observation) => [`- ${observation.title} (${observation.severity}, ${observation.count} URL${observation.count === 1 ? "" : "s"}): ${observation.howToFix}`, ` - First URL: ${observation.sampleUrls[0] ?? "No sample URL recorded"}`]), "");
|
|
15549
15697
|
return `${lines.join("\n")}\n`;
|
|
15550
15698
|
}
|
|
15551
|
-
function escapeHtml(value) {
|
|
15699
|
+
function escapeHtml$1(value) {
|
|
15552
15700
|
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll("\"", """).replaceAll("'", "'");
|
|
15553
15701
|
}
|
|
15554
15702
|
function escapeXml(value) {
|
|
15555
|
-
return escapeHtml(value);
|
|
15703
|
+
return escapeHtml$1(value);
|
|
15556
15704
|
}
|
|
15557
15705
|
//#endregion
|
|
15558
15706
|
//#region packages/core/src/analyze/crawler/history-provenance.ts
|
|
@@ -17409,7 +17557,7 @@ function compareText$10(left, right) {
|
|
|
17409
17557
|
}
|
|
17410
17558
|
return leftPoints.length - rightPoints.length;
|
|
17411
17559
|
}
|
|
17412
|
-
function boundedInteger$
|
|
17560
|
+
function boundedInteger$6(value, fallback, minimum, maximum) {
|
|
17413
17561
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
17414
17562
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
17415
17563
|
}
|
|
@@ -17486,8 +17634,8 @@ function initialSelection(sourceRows) {
|
|
|
17486
17634
|
};
|
|
17487
17635
|
}
|
|
17488
17636
|
function analyzeCtrUnderperformersFromRows(input) {
|
|
17489
|
-
const minImpressions = boundedInteger$
|
|
17490
|
-
const limit = boundedInteger$
|
|
17637
|
+
const minImpressions = boundedInteger$6(input.minImpressions, 200, 1, CTR_MAX_MIN_IMPRESSIONS);
|
|
17638
|
+
const limit = boundedInteger$6(input.limit, 25, 1, 100);
|
|
17491
17639
|
const selection = initialSelection(input.rows.length);
|
|
17492
17640
|
const benchmarkRows = aggregateCtrRows(input.rows, selection).filter((row) => {
|
|
17493
17641
|
const query = row.keys[0] ?? "";
|
|
@@ -17986,7 +18134,7 @@ function analyzeSegmentRows(input) {
|
|
|
17986
18134
|
}
|
|
17987
18135
|
//#endregion
|
|
17988
18136
|
//#region packages/core/src/analyze/segment-impact-input.ts
|
|
17989
|
-
const DAY_MS$
|
|
18137
|
+
const DAY_MS$2 = 864e5;
|
|
17990
18138
|
const MAX_WINDOW_DAYS = 240;
|
|
17991
18139
|
function subtractDays$1(value, days) {
|
|
17992
18140
|
const date = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -17994,7 +18142,7 @@ function subtractDays$1(value, days) {
|
|
|
17994
18142
|
return date.toISOString().slice(0, 10);
|
|
17995
18143
|
}
|
|
17996
18144
|
function segmentRangeDays(range) {
|
|
17997
|
-
return Math.floor((Date.parse(`${range.endDate}T00:00:00.000Z`) - Date.parse(`${range.startDate}T00:00:00.000Z`)) / DAY_MS$
|
|
18145
|
+
return Math.floor((Date.parse(`${range.endDate}T00:00:00.000Z`) - Date.parse(`${range.startDate}T00:00:00.000Z`)) / DAY_MS$2) + 1;
|
|
17998
18146
|
}
|
|
17999
18147
|
function segmentComparisonRange(after, compareDays) {
|
|
18000
18148
|
const endDate = subtractDays$1(after.startDate, 1);
|
|
@@ -18621,13 +18769,13 @@ const MINIMUM_PAGE_SHARE = .1;
|
|
|
18621
18769
|
const MINIMUM_PAGE_IMPRESSIONS = 10;
|
|
18622
18770
|
const MAXIMUM_DOMINANT_SHARE = .8;
|
|
18623
18771
|
const MAX_SUPPRESSIONS = 100;
|
|
18624
|
-
function boundedInteger$
|
|
18772
|
+
function boundedInteger$5(value, fallback, minimum, maximum) {
|
|
18625
18773
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
18626
18774
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
18627
18775
|
}
|
|
18628
18776
|
function analyzeCannibalRows(input) {
|
|
18629
|
-
const minImpressions = boundedInteger$
|
|
18630
|
-
const limit = boundedInteger$
|
|
18777
|
+
const minImpressions = boundedInteger$5(input.minImpressions, 50, 0, 1e9);
|
|
18778
|
+
const limit = boundedInteger$5(input.limit, 25, 1, 100);
|
|
18631
18779
|
const selection = {
|
|
18632
18780
|
sourceRows: input.rows.length,
|
|
18633
18781
|
invalidRows: 0,
|
|
@@ -19409,7 +19557,7 @@ function groupQuickWins(items) {
|
|
|
19409
19557
|
}
|
|
19410
19558
|
//#endregion
|
|
19411
19559
|
//#region packages/core/src/analyze/site-diagnostics/quick-wins-analysis-primitives.ts
|
|
19412
|
-
function boundedInteger$
|
|
19560
|
+
function boundedInteger$4(value, fallback, minimum, maximum) {
|
|
19413
19561
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
19414
19562
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
19415
19563
|
}
|
|
@@ -19530,12 +19678,12 @@ function roundedSum(values) {
|
|
|
19530
19678
|
//#region packages/core/src/analyze/site-diagnostics/quick-wins-analysis.ts
|
|
19531
19679
|
const DEFAULT_MIN_IMPRESSIONS$2 = 200;
|
|
19532
19680
|
const MAX_MIN_IMPRESSIONS$2 = 1e9;
|
|
19533
|
-
const DEFAULT_LIMIT$
|
|
19534
|
-
const MAX_LIMIT$
|
|
19681
|
+
const DEFAULT_LIMIT$3 = 25;
|
|
19682
|
+
const MAX_LIMIT$3 = 100;
|
|
19535
19683
|
const FALLBACK_SOURCE = "builtin_position_ctr_curve_v1";
|
|
19536
19684
|
function analyzeQuickWinsFromRows(input) {
|
|
19537
|
-
const minImpressions = boundedInteger$
|
|
19538
|
-
const limit = boundedInteger$
|
|
19685
|
+
const minImpressions = boundedInteger$4(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$2, 0, MAX_MIN_IMPRESSIONS$2);
|
|
19686
|
+
const limit = boundedInteger$4(input.limit, DEFAULT_LIMIT$3, 1, MAX_LIMIT$3);
|
|
19539
19687
|
const selected = selectBenchmarkRows(input);
|
|
19540
19688
|
const benchmarkContext = createCtrBenchmarkContext(selected.rows, {
|
|
19541
19689
|
samplePopulation: "all_qualified_url_samples",
|
|
@@ -19876,7 +20024,7 @@ function compareText$5(left, right) {
|
|
|
19876
20024
|
}
|
|
19877
20025
|
return leftPoints.length - rightPoints.length;
|
|
19878
20026
|
}
|
|
19879
|
-
function isHttpUrl(value) {
|
|
20027
|
+
function isHttpUrl$1(value) {
|
|
19880
20028
|
try {
|
|
19881
20029
|
const parsed = new URL(value);
|
|
19882
20030
|
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
@@ -19885,7 +20033,7 @@ function isHttpUrl(value) {
|
|
|
19885
20033
|
}
|
|
19886
20034
|
}
|
|
19887
20035
|
function isValidStrikingDistanceRow(row, query, url) {
|
|
19888
|
-
return Boolean(query) && isHttpUrl(url) && Number.isFinite(row.clicks) && Number.isFinite(row.impressions) && Number.isFinite(row.ctr) && Number.isFinite(row.position) && row.clicks >= 0 && row.impressions > 0 && row.clicks <= row.impressions && row.ctr >= 0 && row.ctr <= 1 && row.position > 0;
|
|
20036
|
+
return Boolean(query) && isHttpUrl$1(url) && Number.isFinite(row.clicks) && Number.isFinite(row.impressions) && Number.isFinite(row.ctr) && Number.isFinite(row.position) && row.clicks >= 0 && row.impressions > 0 && row.clicks <= row.impressions && row.ctr >= 0 && row.ctr <= 1 && row.position > 0;
|
|
19889
20037
|
}
|
|
19890
20038
|
function isStrikingDistanceBrandRow(input) {
|
|
19891
20039
|
return shouldExcludeBrandQuery({
|
|
@@ -19975,8 +20123,8 @@ function groupStrikingDistanceItems(items) {
|
|
|
19975
20123
|
//#region packages/core/src/analyze/striking-distance-analysis.ts
|
|
19976
20124
|
const DEFAULT_MIN_IMPRESSIONS$1 = 100;
|
|
19977
20125
|
const MAX_MIN_IMPRESSIONS$1 = 1e9;
|
|
19978
|
-
const DEFAULT_LIMIT$
|
|
19979
|
-
const MAX_LIMIT$
|
|
20126
|
+
const DEFAULT_LIMIT$2 = 25;
|
|
20127
|
+
const MAX_LIMIT$2 = 100;
|
|
19980
20128
|
function integerInRange$1(value, fallback, minimum, maximum) {
|
|
19981
20129
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
19982
20130
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
@@ -20018,7 +20166,7 @@ function selectRows(input) {
|
|
|
20018
20166
|
}
|
|
20019
20167
|
function analyzeStrikingDistanceRows(input) {
|
|
20020
20168
|
const minImpressions = integerInRange$1(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$1, 0, MAX_MIN_IMPRESSIONS$1);
|
|
20021
|
-
const limit = integerInRange$1(input.limit, DEFAULT_LIMIT$
|
|
20169
|
+
const limit = integerInRange$1(input.limit, DEFAULT_LIMIT$2, 1, MAX_LIMIT$2);
|
|
20022
20170
|
const selected = selectRows({
|
|
20023
20171
|
...input,
|
|
20024
20172
|
minImpressions
|
|
@@ -20092,7 +20240,7 @@ const defaultDependencies$7 = {
|
|
|
20092
20240
|
verifyContent: verifyQueryContent,
|
|
20093
20241
|
now: () => /* @__PURE__ */ new Date()
|
|
20094
20242
|
};
|
|
20095
|
-
function boundedInteger$
|
|
20243
|
+
function boundedInteger$3(value, fallback, minimum, maximum) {
|
|
20096
20244
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
20097
20245
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
20098
20246
|
}
|
|
@@ -20156,7 +20304,7 @@ async function strikingDistance(input, dependencies = defaultDependencies$7) {
|
|
|
20156
20304
|
brandTerms: input.brandTerms,
|
|
20157
20305
|
includeBrand: input.includeBrand
|
|
20158
20306
|
});
|
|
20159
|
-
const verifyLimit = boundedInteger$
|
|
20307
|
+
const verifyLimit = boundedInteger$3(input.verifyLimit, DEFAULT_VERIFY_LIMIT, 0, analysis.limit);
|
|
20160
20308
|
const verificationRequested = input.verifyContent === true || input.verifyContent !== false && input.verifyLimit !== void 0;
|
|
20161
20309
|
const attempted = verificationRequested ? Math.min(verifyLimit, analysis.items.length) : 0;
|
|
20162
20310
|
const items = await Promise.all(analysis.items.map(async (item, index) => {
|
|
@@ -20345,7 +20493,7 @@ function standardDeviation(values) {
|
|
|
20345
20493
|
function toFixedNumber(value, digits = 3) {
|
|
20346
20494
|
return Number(value.toFixed(digits));
|
|
20347
20495
|
}
|
|
20348
|
-
function percentChange(baseline, comparison) {
|
|
20496
|
+
function percentChange$1(baseline, comparison) {
|
|
20349
20497
|
if (!baseline) return comparison ? null : 0;
|
|
20350
20498
|
return (comparison - baseline) / baseline * 100;
|
|
20351
20499
|
}
|
|
@@ -20531,7 +20679,7 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
20531
20679
|
const outsideFlatBaseline = zScore === null && comparisonMean !== baselineMean;
|
|
20532
20680
|
const significant = outsideFlatBaseline || zScore !== null && Math.abs(zScore) >= 2;
|
|
20533
20681
|
const direction = significant ? comparisonMean < baselineMean ? "drop" : "spike" : "normal";
|
|
20534
|
-
const relativeChange = percentChange(baselineMean, comparisonMean);
|
|
20682
|
+
const relativeChange = percentChange$1(baselineMean, comparisonMean);
|
|
20535
20683
|
return {
|
|
20536
20684
|
site: input.site,
|
|
20537
20685
|
metric,
|
|
@@ -21590,7 +21738,7 @@ function dateShift(date, days) {
|
|
|
21590
21738
|
}
|
|
21591
21739
|
//#endregion
|
|
21592
21740
|
//#region packages/core/src/analyze/experiments/measurement/window.ts
|
|
21593
|
-
const DAY_MS = 864e5;
|
|
21741
|
+
const DAY_MS$1 = 864e5;
|
|
21594
21742
|
function exactDate(value, label) {
|
|
21595
21743
|
const pattern = /^\d{4}-\d{2}-\d{2}$/;
|
|
21596
21744
|
const parsed = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -21605,7 +21753,7 @@ function boundedDays(value, fallback, label) {
|
|
|
21605
21753
|
function calendarDays(startDate, endDate) {
|
|
21606
21754
|
const start = (/* @__PURE__ */ new Date(`${startDate}T00:00:00.000Z`)).getTime();
|
|
21607
21755
|
const end = (/* @__PURE__ */ new Date(`${endDate}T00:00:00.000Z`)).getTime();
|
|
21608
|
-
return Math.floor((end - start) / DAY_MS) + 1;
|
|
21756
|
+
return Math.floor((end - start) / DAY_MS$1) + 1;
|
|
21609
21757
|
}
|
|
21610
21758
|
function subtractCalendarMonths(value, months) {
|
|
21611
21759
|
const source = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -22153,7 +22301,7 @@ function internalLinksVerdict(input) {
|
|
|
22153
22301
|
function uniqueInternalLinksWarnings(warnings) {
|
|
22154
22302
|
return [...new Map(warnings.map((warning) => [`${warning.stage}\u0000${warning.url}\u0000${warning.code}\u0000${warning.message}`, warning])).values()];
|
|
22155
22303
|
}
|
|
22156
|
-
function errorMessage(error) {
|
|
22304
|
+
function errorMessage$1(error) {
|
|
22157
22305
|
return error instanceof Error ? error.message : String(error);
|
|
22158
22306
|
}
|
|
22159
22307
|
function acceptedAlias(site, value) {
|
|
@@ -22203,7 +22351,7 @@ async function verifyInternalLinkTarget(input) {
|
|
|
22203
22351
|
stage: "target-fetch",
|
|
22204
22352
|
url: input.targetUrl,
|
|
22205
22353
|
code: "fetch-failed",
|
|
22206
|
-
message: errorMessage(error)
|
|
22354
|
+
message: errorMessage$1(error)
|
|
22207
22355
|
}]
|
|
22208
22356
|
};
|
|
22209
22357
|
}
|
|
@@ -22224,7 +22372,7 @@ async function verifyInternalLinkTarget(input) {
|
|
|
22224
22372
|
stage: "target-extract",
|
|
22225
22373
|
url: input.targetUrl,
|
|
22226
22374
|
code: "extract-failed",
|
|
22227
|
-
message: errorMessage(error)
|
|
22375
|
+
message: errorMessage$1(error)
|
|
22228
22376
|
}]
|
|
22229
22377
|
};
|
|
22230
22378
|
}
|
|
@@ -22287,7 +22435,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
22287
22435
|
stage: "source-fetch",
|
|
22288
22436
|
url: input.candidate.sourceUrl,
|
|
22289
22437
|
code: "fetch-failed",
|
|
22290
|
-
message: errorMessage(error)
|
|
22438
|
+
message: errorMessage$1(error)
|
|
22291
22439
|
}]
|
|
22292
22440
|
};
|
|
22293
22441
|
}
|
|
@@ -22305,7 +22453,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
22305
22453
|
stage: "source-fetch",
|
|
22306
22454
|
url: input.candidate.sourceUrl,
|
|
22307
22455
|
code: "fetch-failed",
|
|
22308
|
-
message: errorMessage(error)
|
|
22456
|
+
message: errorMessage$1(error)
|
|
22309
22457
|
}]
|
|
22310
22458
|
};
|
|
22311
22459
|
}
|
|
@@ -22319,7 +22467,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
22319
22467
|
stage: "source-extract",
|
|
22320
22468
|
url: input.candidate.sourceUrl,
|
|
22321
22469
|
code: "extract-failed",
|
|
22322
|
-
message: errorMessage(error)
|
|
22470
|
+
message: errorMessage$1(error)
|
|
22323
22471
|
}]
|
|
22324
22472
|
};
|
|
22325
22473
|
}
|
|
@@ -23130,7 +23278,7 @@ function compareText$3(left, right) {
|
|
|
23130
23278
|
}
|
|
23131
23279
|
return leftPoints.length - rightPoints.length;
|
|
23132
23280
|
}
|
|
23133
|
-
function boundedInteger$
|
|
23281
|
+
function boundedInteger$2(value, fallback, maximum) {
|
|
23134
23282
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
23135
23283
|
return Math.min(maximum, Math.max(1, Math.trunc(value)));
|
|
23136
23284
|
}
|
|
@@ -23323,10 +23471,10 @@ function caveats(input) {
|
|
|
23323
23471
|
return items;
|
|
23324
23472
|
}
|
|
23325
23473
|
function analyzeIndexCoverageSignals(input) {
|
|
23326
|
-
const itemLimit = boundedInteger$
|
|
23327
|
-
const clusterLimit = boundedInteger$
|
|
23328
|
-
const clusterSamples = boundedInteger$
|
|
23329
|
-
const minUrls = boundedInteger$
|
|
23474
|
+
const itemLimit = boundedInteger$2(input.limits?.itemsPerSection, DEFAULT_ITEMS_PER_SECTION, MAX_ITEMS_PER_SECTION);
|
|
23475
|
+
const clusterLimit = boundedInteger$2(input.limits?.templateClusters, DEFAULT_TEMPLATE_LIMIT, MAX_TEMPLATE_LIMIT);
|
|
23476
|
+
const clusterSamples = boundedInteger$2(input.limits?.templateSamples, DEFAULT_TEMPLATE_SAMPLES, MAX_TEMPLATE_SAMPLES);
|
|
23477
|
+
const minUrls = boundedInteger$2(input.templateThresholds?.minUrls, DEFAULT_TEMPLATE_MIN_URLS, Number.MAX_SAFE_INTEGER);
|
|
23330
23478
|
const requestedMinShare = input.templateThresholds?.minShare;
|
|
23331
23479
|
const minShare = Number.isFinite(requestedMinShare) ? Math.min(1, Math.max(0, requestedMinShare ?? DEFAULT_TEMPLATE_MIN_SHARE)) : DEFAULT_TEMPLATE_MIN_SHARE;
|
|
23332
23480
|
const crawl = crawlInput(input.crawl);
|
|
@@ -27101,16 +27249,16 @@ function benchmarkDetails(benchmark) {
|
|
|
27101
27249
|
positiveUrlSamples: benchmark.positiveUrlSamples
|
|
27102
27250
|
};
|
|
27103
27251
|
}
|
|
27104
|
-
function boundedInteger(input) {
|
|
27252
|
+
function boundedInteger$1(input) {
|
|
27105
27253
|
const value = Number.isFinite(input.value) ? input.value ?? 0 : input.fallback;
|
|
27106
27254
|
return Math.min(input.maximum ?? Number.POSITIVE_INFINITY, Math.max(1, Math.floor(value)));
|
|
27107
27255
|
}
|
|
27108
27256
|
function analyzeQueryClustersFromRows(input) {
|
|
27109
|
-
const minImpressions = boundedInteger({
|
|
27257
|
+
const minImpressions = boundedInteger$1({
|
|
27110
27258
|
value: input.minImpressions,
|
|
27111
27259
|
fallback: 25
|
|
27112
27260
|
});
|
|
27113
|
-
const limit = boundedInteger({
|
|
27261
|
+
const limit = boundedInteger$1({
|
|
27114
27262
|
value: input.limit,
|
|
27115
27263
|
fallback: 25,
|
|
27116
27264
|
maximum: 100
|
|
@@ -27752,8 +27900,8 @@ function reportPresentation(report) {
|
|
|
27752
27900
|
//#region packages/core/src/analyze/second-page-analysis.ts
|
|
27753
27901
|
const DEFAULT_MIN_IMPRESSIONS = 50;
|
|
27754
27902
|
const MAX_MIN_IMPRESSIONS = 1e9;
|
|
27755
|
-
const DEFAULT_LIMIT = 10;
|
|
27756
|
-
const MAX_LIMIT = 100;
|
|
27903
|
+
const DEFAULT_LIMIT$1 = 10;
|
|
27904
|
+
const MAX_LIMIT$1 = 100;
|
|
27757
27905
|
function integerInRange(value, fallback, minimum, maximum) {
|
|
27758
27906
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
27759
27907
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
@@ -27840,7 +27988,7 @@ function compareItems(left, right) {
|
|
|
27840
27988
|
}
|
|
27841
27989
|
function analyzeSecondPageRows(input) {
|
|
27842
27990
|
const minImpressions = integerInRange(input.minImpressions, DEFAULT_MIN_IMPRESSIONS, 0, MAX_MIN_IMPRESSIONS);
|
|
27843
|
-
const limit = integerInRange(input.limit, DEFAULT_LIMIT, 1, MAX_LIMIT);
|
|
27991
|
+
const limit = integerInRange(input.limit, DEFAULT_LIMIT$1, 1, MAX_LIMIT$1);
|
|
27844
27992
|
const selection = {
|
|
27845
27993
|
sourceRows: input.rows.length,
|
|
27846
27994
|
invalidRows: 0,
|
|
@@ -29375,98 +29523,1008 @@ async function updatePostmortemWorkflow(input, dependencies = {}) {
|
|
|
29375
29523
|
});
|
|
29376
29524
|
}
|
|
29377
29525
|
//#endregion
|
|
29378
|
-
//#region packages/core/src/
|
|
29379
|
-
|
|
29380
|
-
|
|
29381
|
-
|
|
29382
|
-
|
|
29383
|
-
|
|
29384
|
-
|
|
29385
|
-
|
|
29386
|
-
|
|
29387
|
-
}
|
|
29388
|
-
|
|
29389
|
-
|
|
29390
|
-
if (
|
|
29391
|
-
return
|
|
29526
|
+
//#region packages/core/src/bing/analysis.ts
|
|
29527
|
+
const DAY_MS = 864e5;
|
|
29528
|
+
const DEFAULT_COMPARISON_DAYS = 28;
|
|
29529
|
+
const WEEKLY_PERIODS = 4;
|
|
29530
|
+
const OPPORTUNITY_MIN_IMPRESSIONS = 20;
|
|
29531
|
+
const OPPORTUNITY_MIN_POSITION = 4;
|
|
29532
|
+
const OPPORTUNITY_MAX_POSITION = 20;
|
|
29533
|
+
const RESULT_LIMIT = 10;
|
|
29534
|
+
function dateOffset(date, days) {
|
|
29535
|
+
return new Date(Date.parse(`${date}T00:00:00Z`) + days * DAY_MS).toISOString().slice(0, 10);
|
|
29536
|
+
}
|
|
29537
|
+
function percentChange(previous, current) {
|
|
29538
|
+
if (previous === 0) return null;
|
|
29539
|
+
return round((current - previous) / previous * 100);
|
|
29540
|
+
}
|
|
29541
|
+
function round(value, digits = 3) {
|
|
29542
|
+
const factor = 10 ** digits;
|
|
29543
|
+
return Math.round(value * factor) / factor;
|
|
29544
|
+
}
|
|
29545
|
+
function trafficPeriod(byDate, startDate, endDate, expectedDays) {
|
|
29546
|
+
let clicks = 0;
|
|
29547
|
+
let impressions = 0;
|
|
29548
|
+
let observedDays = 0;
|
|
29549
|
+
for (const [date, row] of byDate) {
|
|
29550
|
+
if (date < startDate || date > endDate) continue;
|
|
29551
|
+
observedDays += 1;
|
|
29552
|
+
clicks += row.clicks;
|
|
29553
|
+
impressions += row.impressions;
|
|
29554
|
+
}
|
|
29555
|
+
return {
|
|
29556
|
+
startDate,
|
|
29557
|
+
endDate,
|
|
29558
|
+
expectedDays,
|
|
29559
|
+
observedDays,
|
|
29560
|
+
missingDays: Math.max(0, expectedDays - observedDays),
|
|
29561
|
+
clicks,
|
|
29562
|
+
impressions,
|
|
29563
|
+
ctr: impressions ? round(clicks / impressions * 100) : null
|
|
29564
|
+
};
|
|
29392
29565
|
}
|
|
29393
|
-
function
|
|
29394
|
-
|
|
29566
|
+
function analyzeBingTraffic(rows, comparisonDays = DEFAULT_COMPARISON_DAYS) {
|
|
29567
|
+
if (!rows.length) return void 0;
|
|
29568
|
+
const byDate = /* @__PURE__ */ new Map();
|
|
29569
|
+
for (const row of rows) {
|
|
29570
|
+
const existing = byDate.get(row.date) ?? {
|
|
29571
|
+
clicks: 0,
|
|
29572
|
+
impressions: 0
|
|
29573
|
+
};
|
|
29574
|
+
existing.clicks += row.clicks;
|
|
29575
|
+
existing.impressions += row.impressions;
|
|
29576
|
+
byDate.set(row.date, existing);
|
|
29577
|
+
}
|
|
29578
|
+
const latestDate = [...byDate.keys()].sort().at(-1);
|
|
29579
|
+
if (!latestDate) return void 0;
|
|
29580
|
+
const currentStart = dateOffset(latestDate, -(comparisonDays - 1));
|
|
29581
|
+
const previousEnd = dateOffset(currentStart, -1);
|
|
29582
|
+
const previousStart = dateOffset(previousEnd, -(comparisonDays - 1));
|
|
29583
|
+
const current = trafficPeriod(byDate, currentStart, latestDate, comparisonDays);
|
|
29584
|
+
const previous = trafficPeriod(byDate, previousStart, previousEnd, comparisonDays);
|
|
29585
|
+
return {
|
|
29586
|
+
comparisonDays,
|
|
29587
|
+
current,
|
|
29588
|
+
previous,
|
|
29589
|
+
changes: {
|
|
29590
|
+
clicks: current.clicks - previous.clicks,
|
|
29591
|
+
clicksPercent: percentChange(previous.clicks, current.clicks),
|
|
29592
|
+
impressions: current.impressions - previous.impressions,
|
|
29593
|
+
impressionsPercent: percentChange(previous.impressions, current.impressions),
|
|
29594
|
+
ctrPercentagePoints: current.ctr === null || previous.ctr === null ? null : round(current.ctr - previous.ctr)
|
|
29595
|
+
}
|
|
29596
|
+
};
|
|
29597
|
+
}
|
|
29598
|
+
const CRAWL_FIELDS = [
|
|
29599
|
+
"crawledPages",
|
|
29600
|
+
"inIndex",
|
|
29601
|
+
"inLinks",
|
|
29602
|
+
"crawlErrors",
|
|
29603
|
+
"code4xx",
|
|
29604
|
+
"code5xx",
|
|
29605
|
+
"blockedByRobotsTxt",
|
|
29606
|
+
"connectionTimeout",
|
|
29607
|
+
"dnsFailures",
|
|
29608
|
+
"containsMalware"
|
|
29609
|
+
];
|
|
29610
|
+
function analyzeBingCrawl(rows, comparisonDays = DEFAULT_COMPARISON_DAYS) {
|
|
29611
|
+
const ordered = [...rows].sort((a, b) => a.date.localeCompare(b.date, "en"));
|
|
29612
|
+
const current = ordered.at(-1);
|
|
29613
|
+
if (!current) return void 0;
|
|
29614
|
+
const targetDate = dateOffset(current.date, -comparisonDays);
|
|
29615
|
+
const previous = ordered.filter((row) => row.date <= targetDate).at(-1);
|
|
29616
|
+
const changes = {};
|
|
29617
|
+
if (!previous) return {
|
|
29618
|
+
comparisonDays,
|
|
29619
|
+
targetDate,
|
|
29620
|
+
current,
|
|
29621
|
+
previous: void 0,
|
|
29622
|
+
changes
|
|
29623
|
+
};
|
|
29624
|
+
for (const field of CRAWL_FIELDS) {
|
|
29625
|
+
const currentValue = current[field];
|
|
29626
|
+
const previousValue = previous[field];
|
|
29627
|
+
if (typeof currentValue !== "number" || typeof previousValue !== "number") continue;
|
|
29628
|
+
changes[field] = {
|
|
29629
|
+
previous: previousValue,
|
|
29630
|
+
current: currentValue,
|
|
29631
|
+
absolute: currentValue - previousValue,
|
|
29632
|
+
percent: percentChange(previousValue, currentValue)
|
|
29633
|
+
};
|
|
29634
|
+
}
|
|
29635
|
+
return {
|
|
29636
|
+
comparisonDays,
|
|
29637
|
+
targetDate,
|
|
29638
|
+
current,
|
|
29639
|
+
previous,
|
|
29640
|
+
changes
|
|
29641
|
+
};
|
|
29395
29642
|
}
|
|
29396
|
-
function
|
|
29397
|
-
const
|
|
29398
|
-
|
|
29399
|
-
|
|
29400
|
-
|
|
29643
|
+
function aggregateDimensions(rows, dates) {
|
|
29644
|
+
const values = /* @__PURE__ */ new Map();
|
|
29645
|
+
for (const row of rows) {
|
|
29646
|
+
if (!dates.has(row.date)) continue;
|
|
29647
|
+
const item = values.get(row.value) ?? {
|
|
29648
|
+
clicks: 0,
|
|
29649
|
+
impressions: 0,
|
|
29650
|
+
clickPositionTotal: 0,
|
|
29651
|
+
clickPositionWeight: 0,
|
|
29652
|
+
impressionPositionTotal: 0,
|
|
29653
|
+
impressionPositionWeight: 0,
|
|
29654
|
+
observedDates: /* @__PURE__ */ new Set()
|
|
29655
|
+
};
|
|
29656
|
+
item.clicks += row.clicks;
|
|
29657
|
+
item.impressions += row.impressions;
|
|
29658
|
+
item.observedDates.add(row.date);
|
|
29659
|
+
if (row.avgClickPosition !== void 0 && row.clicks > 0) {
|
|
29660
|
+
const weight = row.clicks;
|
|
29661
|
+
item.clickPositionTotal += row.avgClickPosition * weight;
|
|
29662
|
+
item.clickPositionWeight += weight;
|
|
29663
|
+
}
|
|
29664
|
+
if (row.avgImpressionPosition !== void 0 && row.impressions > 0) {
|
|
29665
|
+
const weight = row.impressions;
|
|
29666
|
+
item.impressionPositionTotal += row.avgImpressionPosition * weight;
|
|
29667
|
+
item.impressionPositionWeight += weight;
|
|
29668
|
+
}
|
|
29669
|
+
values.set(row.value, item);
|
|
29670
|
+
}
|
|
29671
|
+
return new Map([...values].map(([value, item]) => [value, {
|
|
29672
|
+
value,
|
|
29673
|
+
observedWeeks: item.observedDates.size,
|
|
29674
|
+
clicks: item.clicks,
|
|
29675
|
+
impressions: item.impressions,
|
|
29676
|
+
avgClickPosition: item.clickPositionWeight ? round(item.clickPositionTotal / item.clickPositionWeight) : null,
|
|
29677
|
+
avgImpressionPosition: item.impressionPositionWeight ? round(item.impressionPositionTotal / item.impressionPositionWeight) : null
|
|
29678
|
+
}]));
|
|
29679
|
+
}
|
|
29680
|
+
function compareValue(a, b) {
|
|
29681
|
+
return a.value.localeCompare(b.value, "en");
|
|
29682
|
+
}
|
|
29683
|
+
function analyzeBingDimensions(rows, kind) {
|
|
29684
|
+
const dates = [...new Set(rows.map((row) => row.date))].sort().reverse();
|
|
29685
|
+
const currentDates = dates.slice(0, WEEKLY_PERIODS);
|
|
29686
|
+
const previousDates = dates.slice(WEEKLY_PERIODS, WEEKLY_PERIODS * 2);
|
|
29687
|
+
const current = aggregateDimensions(rows, new Set(currentDates));
|
|
29688
|
+
const previous = aggregateDimensions(rows, new Set(previousDates));
|
|
29689
|
+
const matchedValues = [...current.keys()].filter((value) => previous.has(value));
|
|
29690
|
+
const comparableValues = currentDates.length === WEEKLY_PERIODS && previousDates.length === WEEKLY_PERIODS ? matchedValues.filter((value) => {
|
|
29691
|
+
const currentValue = current.get(value);
|
|
29692
|
+
const previousValue = previous.get(value);
|
|
29693
|
+
return currentValue?.observedWeeks === WEEKLY_PERIODS && previousValue?.observedWeeks === WEEKLY_PERIODS;
|
|
29694
|
+
}) : [];
|
|
29695
|
+
const movements = comparableValues.map((value) => {
|
|
29696
|
+
const currentValue = current.get(value);
|
|
29697
|
+
const previousValue = previous.get(value);
|
|
29698
|
+
return {
|
|
29699
|
+
value,
|
|
29700
|
+
current: currentValue,
|
|
29701
|
+
previous: previousValue,
|
|
29702
|
+
changes: {
|
|
29703
|
+
clicks: currentValue.clicks - previousValue.clicks,
|
|
29704
|
+
clicksPercent: percentChange(previousValue.clicks, currentValue.clicks),
|
|
29705
|
+
impressions: currentValue.impressions - previousValue.impressions,
|
|
29706
|
+
impressionsPercent: percentChange(previousValue.impressions, currentValue.impressions)
|
|
29707
|
+
}
|
|
29708
|
+
};
|
|
29709
|
+
}).sort((a, b) => Math.abs(b.changes.clicks) - Math.abs(a.changes.clicks) || Math.abs(b.changes.impressions) - Math.abs(a.changes.impressions) || compareValue(a, b)).slice(0, RESULT_LIMIT);
|
|
29710
|
+
const opportunities = [...current.values()].filter((row) => row.impressions >= OPPORTUNITY_MIN_IMPRESSIONS && row.avgImpressionPosition !== null && row.avgImpressionPosition >= OPPORTUNITY_MIN_POSITION && row.avgImpressionPosition <= OPPORTUNITY_MAX_POSITION).sort((a, b) => b.impressions - a.impressions || b.clicks - a.clicks || compareValue(a, b)).slice(0, RESULT_LIMIT);
|
|
29711
|
+
return {
|
|
29712
|
+
kind,
|
|
29713
|
+
sourceSemantics: "Bing weekly top-list evidence. Missing dimensions are unknown, not zero. Movements require observation in every week of both periods.",
|
|
29714
|
+
periods: {
|
|
29715
|
+
weeksPerPeriod: WEEKLY_PERIODS,
|
|
29716
|
+
currentDates,
|
|
29717
|
+
previousDates
|
|
29718
|
+
},
|
|
29719
|
+
coverage: {
|
|
29720
|
+
sourceRows: rows.length,
|
|
29721
|
+
currentDimensions: current.size,
|
|
29722
|
+
previousDimensions: previous.size,
|
|
29723
|
+
matchedDimensions: matchedValues.length,
|
|
29724
|
+
comparableDimensions: comparableValues.length,
|
|
29725
|
+
incompleteMatchedDimensions: matchedValues.length - comparableValues.length,
|
|
29726
|
+
currentOnlyDimensions: [...current.keys()].filter((value) => !previous.has(value)).length,
|
|
29727
|
+
previousOnlyDimensions: [...previous.keys()].filter((value) => !current.has(value)).length
|
|
29728
|
+
},
|
|
29729
|
+
thresholds: {
|
|
29730
|
+
heuristic: true,
|
|
29731
|
+
opportunityMinImpressions: OPPORTUNITY_MIN_IMPRESSIONS,
|
|
29732
|
+
opportunityPositionRange: [OPPORTUNITY_MIN_POSITION, OPPORTUNITY_MAX_POSITION],
|
|
29733
|
+
resultLimit: RESULT_LIMIT
|
|
29734
|
+
},
|
|
29735
|
+
movements,
|
|
29736
|
+
opportunities
|
|
29737
|
+
};
|
|
29401
29738
|
}
|
|
29402
|
-
|
|
29403
|
-
|
|
29404
|
-
|
|
29405
|
-
|
|
29406
|
-
|
|
29407
|
-
|
|
29408
|
-
|
|
29409
|
-
|
|
29410
|
-
|
|
29411
|
-
|
|
29412
|
-
|
|
29413
|
-
|
|
29414
|
-
|
|
29415
|
-
|
|
29416
|
-
|
|
29417
|
-
|
|
29418
|
-
|
|
29419
|
-
|
|
29420
|
-
|
|
29421
|
-
|
|
29422
|
-
|
|
29423
|
-
|
|
29739
|
+
const bingAnalysisDefaults = {
|
|
29740
|
+
comparisonDays: DEFAULT_COMPARISON_DAYS,
|
|
29741
|
+
weeklyPeriods: WEEKLY_PERIODS,
|
|
29742
|
+
resultLimit: RESULT_LIMIT
|
|
29743
|
+
};
|
|
29744
|
+
//#endregion
|
|
29745
|
+
//#region packages/core/src/bing/client.ts
|
|
29746
|
+
const DEFAULT_BASE_URL = "https://ssl.bing.com/webmaster/api.svc/json";
|
|
29747
|
+
const MAX_RESPONSE_BYTES$1 = 2e6;
|
|
29748
|
+
const MAX_STAT_ROWS = 400;
|
|
29749
|
+
const MAX_DIMENSION_ROWS = 8e3;
|
|
29750
|
+
const MAX_LINK_ROWS = 200;
|
|
29751
|
+
function bingDate(value) {
|
|
29752
|
+
if (typeof value !== "string") return void 0;
|
|
29753
|
+
const legacy = /^\/Date\((-?\d+)(?:[+-]\d{4})?\)\/$/.exec(value);
|
|
29754
|
+
const date = legacy ? new Date(Number(legacy[1])) : new Date(value);
|
|
29755
|
+
return Number.isNaN(date.getTime()) ? void 0 : date.toISOString().slice(0, 10);
|
|
29756
|
+
}
|
|
29757
|
+
function nonnegativeNumber(value) {
|
|
29758
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
|
|
29759
|
+
}
|
|
29760
|
+
function positionNumber(value) {
|
|
29761
|
+
if (value === void 0 || value === null || value === -1) return { valid: true };
|
|
29762
|
+
const number = nonnegativeNumber(value);
|
|
29763
|
+
return number === void 0 ? { valid: false } : {
|
|
29764
|
+
valid: true,
|
|
29765
|
+
value: number
|
|
29424
29766
|
};
|
|
29425
|
-
const clients = config.clients.filter((item) => item.id !== id);
|
|
29426
|
-
if (client.isDefault) {
|
|
29427
|
-
for (const item of clients) item.isDefault = false;
|
|
29428
|
-
config.defaultSite = client.siteUrl;
|
|
29429
|
-
}
|
|
29430
|
-
config.clients = [...clients, client];
|
|
29431
|
-
writeConfig(config);
|
|
29432
|
-
return client;
|
|
29433
29767
|
}
|
|
29434
|
-
function
|
|
29435
|
-
|
|
29436
|
-
|
|
29437
|
-
|
|
29438
|
-
|
|
29439
|
-
|
|
29440
|
-
|
|
29441
|
-
|
|
29768
|
+
function isHttpUrl(value) {
|
|
29769
|
+
if (typeof value !== "string" || value.length > 2e3) return false;
|
|
29770
|
+
try {
|
|
29771
|
+
const url = new URL(value);
|
|
29772
|
+
return url.protocol === "http:" || url.protocol === "https:";
|
|
29773
|
+
} catch {
|
|
29774
|
+
return false;
|
|
29775
|
+
}
|
|
29442
29776
|
}
|
|
29443
|
-
function
|
|
29444
|
-
const
|
|
29445
|
-
const client = getClient(idOrName);
|
|
29446
|
-
if (!client) throw new Error(`Client not found: ${idOrName}`);
|
|
29447
|
-
config.clients = config.clients.map((item) => ({
|
|
29448
|
-
...item,
|
|
29449
|
-
isDefault: item.id === client.id
|
|
29450
|
-
}));
|
|
29451
|
-
config.defaultSite = client.siteUrl;
|
|
29452
|
-
writeConfig(config);
|
|
29777
|
+
function boundedRows(rows, invalidRows, limit = MAX_STAT_ROWS) {
|
|
29778
|
+
const capped = rows.length > limit;
|
|
29453
29779
|
return {
|
|
29454
|
-
|
|
29455
|
-
|
|
29780
|
+
rows: capped ? rows.slice(-limit) : rows,
|
|
29781
|
+
invalidRows,
|
|
29782
|
+
capped,
|
|
29783
|
+
returnedRows: rows.length
|
|
29456
29784
|
};
|
|
29457
29785
|
}
|
|
29458
|
-
|
|
29459
|
-
|
|
29460
|
-
const
|
|
29461
|
-
|
|
29462
|
-
|
|
29463
|
-
|
|
29464
|
-
|
|
29465
|
-
|
|
29466
|
-
|
|
29467
|
-
|
|
29468
|
-
|
|
29469
|
-
|
|
29786
|
+
function dimensionValue(value, kind) {
|
|
29787
|
+
if (typeof value !== "string") return void 0;
|
|
29788
|
+
const normalized = value.trim();
|
|
29789
|
+
if (!normalized || normalized.length > 2e3) return void 0;
|
|
29790
|
+
if (kind === "page" && !isHttpUrl(normalized)) return void 0;
|
|
29791
|
+
return normalized;
|
|
29792
|
+
}
|
|
29793
|
+
function errorMessage(body) {
|
|
29794
|
+
try {
|
|
29795
|
+
const parsed = JSON.parse(body);
|
|
29796
|
+
return typeof parsed.Message === "string" ? parsed.Message : void 0;
|
|
29797
|
+
} catch {
|
|
29798
|
+
return;
|
|
29799
|
+
}
|
|
29800
|
+
}
|
|
29801
|
+
var BingWebmasterClient = class {
|
|
29802
|
+
credentials;
|
|
29803
|
+
authentication;
|
|
29804
|
+
baseUrl;
|
|
29805
|
+
fetchImpl;
|
|
29806
|
+
timeoutMs;
|
|
29807
|
+
constructor(credentials, options = {}) {
|
|
29808
|
+
this.credentials = credentials;
|
|
29809
|
+
if (!(credentials.apiKey?.trim() || credentials.accessToken?.trim())) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster credentials are empty.");
|
|
29810
|
+
this.authentication = credentials.apiKey ? "api-key" : "oauth";
|
|
29811
|
+
this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, "");
|
|
29812
|
+
this.fetchImpl = options.fetchImpl ?? publicHttpFetch;
|
|
29813
|
+
this.timeoutMs = options.timeoutMs ?? 3e4;
|
|
29814
|
+
}
|
|
29815
|
+
async request(method, params = {}) {
|
|
29816
|
+
const url = new URL(`${this.baseUrl}/${method}`);
|
|
29817
|
+
for (const [key, value] of Object.entries(params)) url.searchParams.set(key, value);
|
|
29818
|
+
if (this.credentials.apiKey) url.searchParams.set("apikey", this.credentials.apiKey);
|
|
29819
|
+
let response;
|
|
29820
|
+
try {
|
|
29821
|
+
response = await this.fetchImpl(url.toString(), {
|
|
29822
|
+
headers: {
|
|
29823
|
+
accept: "application/json",
|
|
29824
|
+
...this.credentials.accessToken ? { authorization: `Bearer ${this.credentials.accessToken}` } : {}
|
|
29825
|
+
},
|
|
29826
|
+
signal: AbortSignal.timeout(this.timeoutMs)
|
|
29827
|
+
});
|
|
29828
|
+
} catch (error) {
|
|
29829
|
+
if (error instanceof SeoError) throw error;
|
|
29830
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster did not respond: ${error instanceof Error ? error.message : String(error)}`);
|
|
29831
|
+
}
|
|
29832
|
+
const body = await readBoundedResponseText(response, MAX_RESPONSE_BYTES$1, "Bing Webmaster response");
|
|
29833
|
+
if (!response.ok) {
|
|
29834
|
+
const message = errorMessage(body);
|
|
29835
|
+
if (response.status === 401 || /invalidapikey|unauthorized/i.test(message ?? "")) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster rejected the saved credential. Run `seo providers bing connect` again.");
|
|
29836
|
+
if (response.status === 403) throw new SeoError("ACCESS_DENIED", "Bing Webmaster denied access to this site. Check that it is verified for the connected account.");
|
|
29837
|
+
if (response.status === 429) throw new SeoError("RATE_LIMITED", "Bing Webmaster rate limited this request. Try again later.");
|
|
29838
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster request failed with HTTP ${response.status}${message ? ` (${message})` : ""}.`);
|
|
29839
|
+
}
|
|
29840
|
+
try {
|
|
29841
|
+
return JSON.parse(body).d;
|
|
29842
|
+
} catch {
|
|
29843
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid JSON.");
|
|
29844
|
+
}
|
|
29845
|
+
}
|
|
29846
|
+
async listSites() {
|
|
29847
|
+
const data = await this.request("GetUserSites");
|
|
29848
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned an invalid site list.");
|
|
29849
|
+
const sites = [];
|
|
29850
|
+
let invalidRows = 0;
|
|
29851
|
+
for (const item of data) {
|
|
29852
|
+
if (!item || typeof item !== "object") {
|
|
29853
|
+
invalidRows += 1;
|
|
29854
|
+
continue;
|
|
29855
|
+
}
|
|
29856
|
+
const row = item;
|
|
29857
|
+
if (typeof row.Url !== "string" || typeof row.IsVerified !== "boolean") {
|
|
29858
|
+
invalidRows += 1;
|
|
29859
|
+
continue;
|
|
29860
|
+
}
|
|
29861
|
+
sites.push({
|
|
29862
|
+
url: row.Url,
|
|
29863
|
+
isVerified: row.IsVerified
|
|
29864
|
+
});
|
|
29865
|
+
}
|
|
29866
|
+
sites.sort((a, b) => a.url.localeCompare(b.url, "en"));
|
|
29867
|
+
return {
|
|
29868
|
+
sites,
|
|
29869
|
+
invalidRows
|
|
29870
|
+
};
|
|
29871
|
+
}
|
|
29872
|
+
async getTraffic(siteUrl) {
|
|
29873
|
+
const data = await this.request("GetRankAndTrafficStats", { siteUrl });
|
|
29874
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid traffic statistics.");
|
|
29875
|
+
const rows = [];
|
|
29876
|
+
let invalidRows = 0;
|
|
29877
|
+
for (const item of data) {
|
|
29878
|
+
if (!item || typeof item !== "object") {
|
|
29879
|
+
invalidRows += 1;
|
|
29880
|
+
continue;
|
|
29881
|
+
}
|
|
29882
|
+
const row = item;
|
|
29883
|
+
const date = bingDate(row.Date);
|
|
29884
|
+
const clicks = nonnegativeNumber(row.Clicks);
|
|
29885
|
+
const impressions = nonnegativeNumber(row.Impressions);
|
|
29886
|
+
if (!date || clicks === void 0 || impressions === void 0) {
|
|
29887
|
+
invalidRows += 1;
|
|
29888
|
+
continue;
|
|
29889
|
+
}
|
|
29890
|
+
rows.push({
|
|
29891
|
+
date,
|
|
29892
|
+
clicks,
|
|
29893
|
+
impressions
|
|
29894
|
+
});
|
|
29895
|
+
}
|
|
29896
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en") || a.clicks - b.clicks || a.impressions - b.impressions);
|
|
29897
|
+
return boundedRows(rows, invalidRows);
|
|
29898
|
+
}
|
|
29899
|
+
async getCrawlStats(siteUrl) {
|
|
29900
|
+
const data = await this.request("GetCrawlStats", { siteUrl });
|
|
29901
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid crawl statistics.");
|
|
29902
|
+
const fieldMap = {
|
|
29903
|
+
AllOtherCodes: "allOtherCodes",
|
|
29904
|
+
BlockedByRobotsTxt: "blockedByRobotsTxt",
|
|
29905
|
+
Code2xx: "code2xx",
|
|
29906
|
+
Code301: "code301",
|
|
29907
|
+
Code302: "code302",
|
|
29908
|
+
Code4xx: "code4xx",
|
|
29909
|
+
Code5xx: "code5xx",
|
|
29910
|
+
ConnectionTimeout: "connectionTimeout",
|
|
29911
|
+
ContainsMalware: "containsMalware",
|
|
29912
|
+
CrawledPages: "crawledPages",
|
|
29913
|
+
CrawlErrors: "crawlErrors",
|
|
29914
|
+
DnsFailures: "dnsFailures",
|
|
29915
|
+
InIndex: "inIndex",
|
|
29916
|
+
InLinks: "inLinks"
|
|
29917
|
+
};
|
|
29918
|
+
const rows = [];
|
|
29919
|
+
let invalidRows = 0;
|
|
29920
|
+
for (const item of data) {
|
|
29921
|
+
if (!item || typeof item !== "object") {
|
|
29922
|
+
invalidRows += 1;
|
|
29923
|
+
continue;
|
|
29924
|
+
}
|
|
29925
|
+
const source = item;
|
|
29926
|
+
const date = bingDate(source.Date);
|
|
29927
|
+
if (!date) {
|
|
29928
|
+
invalidRows += 1;
|
|
29929
|
+
continue;
|
|
29930
|
+
}
|
|
29931
|
+
const row = { date };
|
|
29932
|
+
let invalid = false;
|
|
29933
|
+
for (const [providerField, field] of Object.entries(fieldMap)) {
|
|
29934
|
+
const value = source[providerField];
|
|
29935
|
+
if (value === void 0 || value === null) continue;
|
|
29936
|
+
const number = nonnegativeNumber(value);
|
|
29937
|
+
if (number === void 0) {
|
|
29938
|
+
invalid = true;
|
|
29939
|
+
break;
|
|
29940
|
+
}
|
|
29941
|
+
row[field] = number;
|
|
29942
|
+
}
|
|
29943
|
+
if (invalid) {
|
|
29944
|
+
invalidRows += 1;
|
|
29945
|
+
continue;
|
|
29946
|
+
}
|
|
29947
|
+
rows.push(row);
|
|
29948
|
+
}
|
|
29949
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en"));
|
|
29950
|
+
return boundedRows(rows, invalidRows);
|
|
29951
|
+
}
|
|
29952
|
+
async getQueryStats(siteUrl) {
|
|
29953
|
+
return this.getDimensionStats("GetQueryStats", siteUrl, "query");
|
|
29954
|
+
}
|
|
29955
|
+
async getPageStats(siteUrl) {
|
|
29956
|
+
return this.getDimensionStats("GetPageStats", siteUrl, "page");
|
|
29957
|
+
}
|
|
29958
|
+
async getDimensionStats(method, siteUrl, kind) {
|
|
29959
|
+
const data = await this.request(method, { siteUrl });
|
|
29960
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster returned invalid ${kind} statistics.`);
|
|
29961
|
+
const rows = [];
|
|
29962
|
+
let invalidRows = 0;
|
|
29963
|
+
for (const item of data) {
|
|
29964
|
+
if (!item || typeof item !== "object") {
|
|
29965
|
+
invalidRows += 1;
|
|
29966
|
+
continue;
|
|
29967
|
+
}
|
|
29968
|
+
const source = item;
|
|
29969
|
+
const date = bingDate(source.Date);
|
|
29970
|
+
const value = dimensionValue(source.Query, kind);
|
|
29971
|
+
const clicks = nonnegativeNumber(source.Clicks);
|
|
29972
|
+
const impressions = nonnegativeNumber(source.Impressions);
|
|
29973
|
+
const clickPosition = positionNumber(source.AvgClickPosition);
|
|
29974
|
+
const impressionPosition = positionNumber(source.AvgImpressionPosition);
|
|
29975
|
+
if (!date || !value || clicks === void 0 || impressions === void 0 || !clickPosition.valid || !impressionPosition.valid) {
|
|
29976
|
+
invalidRows += 1;
|
|
29977
|
+
continue;
|
|
29978
|
+
}
|
|
29979
|
+
rows.push({
|
|
29980
|
+
date,
|
|
29981
|
+
value,
|
|
29982
|
+
clicks,
|
|
29983
|
+
impressions,
|
|
29984
|
+
avgClickPosition: clickPosition.value,
|
|
29985
|
+
avgImpressionPosition: impressionPosition.value
|
|
29986
|
+
});
|
|
29987
|
+
}
|
|
29988
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en") || a.value.localeCompare(b.value, "en"));
|
|
29989
|
+
return boundedRows(rows, invalidRows, MAX_DIMENSION_ROWS);
|
|
29990
|
+
}
|
|
29991
|
+
async getLinkCounts(siteUrl, page) {
|
|
29992
|
+
const data = await this.request("GetLinkCounts", {
|
|
29993
|
+
siteUrl,
|
|
29994
|
+
page: String(page)
|
|
29995
|
+
});
|
|
29996
|
+
if (!data || typeof data !== "object") throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link counts.");
|
|
29997
|
+
const response = data;
|
|
29998
|
+
if (!Array.isArray(response.Links)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link counts.");
|
|
29999
|
+
const rows = [];
|
|
30000
|
+
let invalidRows = 0;
|
|
30001
|
+
for (const item of response.Links) {
|
|
30002
|
+
if (!item || typeof item !== "object") {
|
|
30003
|
+
invalidRows += 1;
|
|
30004
|
+
continue;
|
|
30005
|
+
}
|
|
30006
|
+
const row = item;
|
|
30007
|
+
const count = nonnegativeNumber(row.Count);
|
|
30008
|
+
if (!isHttpUrl(row.Url) || count === void 0) {
|
|
30009
|
+
invalidRows += 1;
|
|
30010
|
+
continue;
|
|
30011
|
+
}
|
|
30012
|
+
rows.push({
|
|
30013
|
+
url: row.Url,
|
|
30014
|
+
count
|
|
30015
|
+
});
|
|
30016
|
+
}
|
|
30017
|
+
rows.sort((a, b) => b.count - a.count || a.url.localeCompare(b.url, "en"));
|
|
30018
|
+
return linkPage(rows, response.TotalPages, invalidRows);
|
|
30019
|
+
}
|
|
30020
|
+
async getUrlLinks(siteUrl, link, page) {
|
|
30021
|
+
const data = await this.request("GetUrlLinks", {
|
|
30022
|
+
siteUrl,
|
|
30023
|
+
link,
|
|
30024
|
+
page: String(page)
|
|
30025
|
+
});
|
|
30026
|
+
if (!data || typeof data !== "object") throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid referring links.");
|
|
30027
|
+
const response = data;
|
|
30028
|
+
if (!Array.isArray(response.Details)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid referring links.");
|
|
30029
|
+
const rows = [];
|
|
30030
|
+
let invalidRows = 0;
|
|
30031
|
+
for (const item of response.Details) {
|
|
30032
|
+
if (!item || typeof item !== "object") {
|
|
30033
|
+
invalidRows += 1;
|
|
30034
|
+
continue;
|
|
30035
|
+
}
|
|
30036
|
+
const row = item;
|
|
30037
|
+
if (!isHttpUrl(row.Url)) {
|
|
30038
|
+
invalidRows += 1;
|
|
30039
|
+
continue;
|
|
30040
|
+
}
|
|
30041
|
+
rows.push({
|
|
30042
|
+
url: row.Url,
|
|
30043
|
+
anchorText: typeof row.AnchorText === "string" && row.AnchorText.trim() ? row.AnchorText.trim() : void 0
|
|
30044
|
+
});
|
|
30045
|
+
}
|
|
30046
|
+
rows.sort((a, b) => a.url.localeCompare(b.url, "en") || (a.anchorText ?? "").localeCompare(b.anchorText ?? "", "en"));
|
|
30047
|
+
return linkPage(rows, response.TotalPages, invalidRows);
|
|
30048
|
+
}
|
|
30049
|
+
};
|
|
30050
|
+
function linkPage(rows, totalPagesValue, invalidRows) {
|
|
30051
|
+
const totalPages = nonnegativeNumber(totalPagesValue);
|
|
30052
|
+
if (totalPages === void 0 || !Number.isInteger(totalPages)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link pagination.");
|
|
30053
|
+
const capped = rows.length > MAX_LINK_ROWS;
|
|
30054
|
+
return {
|
|
30055
|
+
rows: capped ? rows.slice(0, MAX_LINK_ROWS) : rows,
|
|
30056
|
+
totalPages,
|
|
30057
|
+
invalidRows,
|
|
30058
|
+
capped,
|
|
30059
|
+
returnedRows: rows.length
|
|
30060
|
+
};
|
|
30061
|
+
}
|
|
30062
|
+
//#endregion
|
|
30063
|
+
//#region packages/core/src/storage/provider-secrets.ts
|
|
30064
|
+
const KEYRING_SERVICE = "seo";
|
|
30065
|
+
const PRIVATE_FILE_MODE = 384;
|
|
30066
|
+
function account(name) {
|
|
30067
|
+
return `provider:${name}`;
|
|
30068
|
+
}
|
|
30069
|
+
function readFile$1() {
|
|
30070
|
+
return readJsonFile(getSeoCliPaths().providerSecretsFile) ?? {
|
|
30071
|
+
version: 1,
|
|
30072
|
+
secrets: {}
|
|
30073
|
+
};
|
|
30074
|
+
}
|
|
30075
|
+
function writeFile$1(file) {
|
|
30076
|
+
const path = getSeoCliPaths().providerSecretsFile;
|
|
30077
|
+
if (Object.keys(file.secrets).length === 0) {
|
|
30078
|
+
safeRemove(path);
|
|
30079
|
+
return;
|
|
30080
|
+
}
|
|
30081
|
+
writeJsonAtomic(path, file, PRIVATE_FILE_MODE);
|
|
30082
|
+
}
|
|
30083
|
+
function deleteFileSecret(name) {
|
|
30084
|
+
const file = readFile$1();
|
|
30085
|
+
if (!(name in file.secrets)) return;
|
|
30086
|
+
delete file.secrets[name];
|
|
30087
|
+
writeFile$1(file);
|
|
30088
|
+
}
|
|
30089
|
+
async function readProviderSecret(input) {
|
|
30090
|
+
const environment = input.env ?? process.env;
|
|
30091
|
+
const environmentValue = input.envVar ? environment[input.envVar]?.trim() : void 0;
|
|
30092
|
+
if (environmentValue) return {
|
|
30093
|
+
value: environmentValue,
|
|
30094
|
+
source: "environment"
|
|
30095
|
+
};
|
|
30096
|
+
if (readConfig().security.useKeychain) try {
|
|
30097
|
+
const value = await getKeyringPassword(KEYRING_SERVICE, account(input.name));
|
|
30098
|
+
if (value) return {
|
|
30099
|
+
value,
|
|
30100
|
+
source: "keychain"
|
|
30101
|
+
};
|
|
30102
|
+
} catch {}
|
|
30103
|
+
const value = readFile$1().secrets[input.name];
|
|
30104
|
+
return value ? {
|
|
30105
|
+
value,
|
|
30106
|
+
source: "file"
|
|
30107
|
+
} : void 0;
|
|
30108
|
+
}
|
|
30109
|
+
async function writeProviderSecret(name, value) {
|
|
30110
|
+
const normalized = value.trim();
|
|
30111
|
+
if (!normalized) throw new Error("Provider secret cannot be empty.");
|
|
30112
|
+
if (readConfig().security.useKeychain) try {
|
|
30113
|
+
await setKeyringPassword(KEYRING_SERVICE, account(name), normalized);
|
|
30114
|
+
deleteFileSecret(name);
|
|
30115
|
+
return "keychain";
|
|
30116
|
+
} catch {}
|
|
30117
|
+
const file = readFile$1();
|
|
30118
|
+
file.secrets[name] = normalized;
|
|
30119
|
+
writeFile$1(file);
|
|
30120
|
+
return "file";
|
|
30121
|
+
}
|
|
30122
|
+
async function deleteProviderSecret(name) {
|
|
30123
|
+
await deleteKeyringPassword(KEYRING_SERVICE, account(name)).catch(() => false);
|
|
30124
|
+
deleteFileSecret(name);
|
|
30125
|
+
}
|
|
30126
|
+
//#endregion
|
|
30127
|
+
//#region packages/core/src/bing/credentials.ts
|
|
30128
|
+
const BING_API_KEY_SECRET = "bing-api-key";
|
|
30129
|
+
const BING_API_KEY_ENV = "SEO_BING_API_KEY";
|
|
30130
|
+
async function createBingWebmasterClient(input = {}) {
|
|
30131
|
+
const secret = await readProviderSecret({
|
|
30132
|
+
name: BING_API_KEY_SECRET,
|
|
30133
|
+
envVar: BING_API_KEY_ENV,
|
|
30134
|
+
env: input.env
|
|
30135
|
+
});
|
|
30136
|
+
if (!secret) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster is not connected. Run `seo providers bing connect`, or set SEO_BING_API_KEY for this process.");
|
|
30137
|
+
return {
|
|
30138
|
+
client: new BingWebmasterClient({ apiKey: secret.value }),
|
|
30139
|
+
credentialSource: secret.source
|
|
30140
|
+
};
|
|
30141
|
+
}
|
|
30142
|
+
//#endregion
|
|
30143
|
+
//#region packages/core/src/bing/overview.ts
|
|
30144
|
+
const DAILY_OUTPUT_ROWS = 14;
|
|
30145
|
+
function message(error) {
|
|
30146
|
+
return error instanceof Error ? error.message : String(error);
|
|
30147
|
+
}
|
|
30148
|
+
function range(rows) {
|
|
30149
|
+
return rows.length ? {
|
|
30150
|
+
startDate: rows[0]?.date ?? "",
|
|
30151
|
+
endDate: rows.at(-1)?.date ?? ""
|
|
30152
|
+
} : void 0;
|
|
30153
|
+
}
|
|
30154
|
+
function dailyOutput(data) {
|
|
30155
|
+
const rows = data.rows.slice(-14);
|
|
30156
|
+
return {
|
|
30157
|
+
...data,
|
|
30158
|
+
rows,
|
|
30159
|
+
retainedRows: data.rows.length,
|
|
30160
|
+
outputSelection: {
|
|
30161
|
+
strategy: "most-recent",
|
|
30162
|
+
availableRows: data.rows.length,
|
|
30163
|
+
returnedRows: rows.length,
|
|
30164
|
+
omittedRows: Math.max(0, data.rows.length - rows.length)
|
|
30165
|
+
}
|
|
30166
|
+
};
|
|
30167
|
+
}
|
|
30168
|
+
async function section(work, options) {
|
|
30169
|
+
try {
|
|
30170
|
+
const data = await work();
|
|
30171
|
+
const reasons = [
|
|
30172
|
+
...data.invalidRows ? [`${data.invalidRows} invalid provider rows were excluded.`] : [],
|
|
30173
|
+
...data.capped ? [`The provider response exceeded the ${options.rowLimit}-row limit.`] : [],
|
|
30174
|
+
...options.sourceWarning ? [options.sourceWarning] : []
|
|
30175
|
+
];
|
|
30176
|
+
return reasons.length ? {
|
|
30177
|
+
status: "partial",
|
|
30178
|
+
data,
|
|
30179
|
+
warning: reasons.join(" ")
|
|
30180
|
+
} : {
|
|
30181
|
+
status: "complete",
|
|
30182
|
+
data
|
|
30183
|
+
};
|
|
30184
|
+
} catch (error) {
|
|
30185
|
+
if (error instanceof SeoError && (error.code === "AUTH_REQUIRED" || error.code === "ACCESS_DENIED")) throw error;
|
|
30186
|
+
return {
|
|
30187
|
+
status: "unavailable",
|
|
30188
|
+
warning: message(error)
|
|
30189
|
+
};
|
|
30190
|
+
}
|
|
30191
|
+
}
|
|
30192
|
+
function changeFinding(input) {
|
|
30193
|
+
return {
|
|
30194
|
+
code: input.code,
|
|
30195
|
+
severity: "warning",
|
|
30196
|
+
title: input.title,
|
|
30197
|
+
evidence: {
|
|
30198
|
+
current: input.current,
|
|
30199
|
+
previous: input.previous,
|
|
30200
|
+
absoluteChange: input.absolute,
|
|
30201
|
+
percentChange: input.percent,
|
|
30202
|
+
currentDate: input.dates.current,
|
|
30203
|
+
previousDate: input.dates.previous
|
|
30204
|
+
},
|
|
30205
|
+
interpretation: input.interpretation,
|
|
30206
|
+
verification: input.verification
|
|
30207
|
+
};
|
|
30208
|
+
}
|
|
30209
|
+
function buildFindings(input) {
|
|
30210
|
+
const findings = [];
|
|
30211
|
+
const traffic = input.traffic;
|
|
30212
|
+
if (traffic && traffic.current.observedDays === traffic.current.expectedDays && traffic.previous.observedDays === traffic.previous.expectedDays && (traffic.changes.clicksPercent !== null && Math.abs(traffic.changes.clicksPercent) >= 10 || traffic.changes.impressionsPercent !== null && Math.abs(traffic.changes.impressionsPercent) >= 10)) {
|
|
30213
|
+
const metric = traffic.changes.clicksPercent !== null && Math.abs(traffic.changes.clicksPercent) >= 10 ? "clicks" : "impressions";
|
|
30214
|
+
const declined = traffic.changes[metric] < 0;
|
|
30215
|
+
findings.push({
|
|
30216
|
+
code: `bing_${metric}_${declined ? "declined" : "increased"}`,
|
|
30217
|
+
severity: declined ? "warning" : "info",
|
|
30218
|
+
title: `Bing ${metric} ${declined ? "declined" : "increased"} in the latest complete period`,
|
|
30219
|
+
evidence: {
|
|
30220
|
+
currentClicks: traffic.current.clicks,
|
|
30221
|
+
previousClicks: traffic.previous.clicks,
|
|
30222
|
+
clicksChange: traffic.changes.clicks,
|
|
30223
|
+
clicksPercentChange: traffic.changes.clicksPercent,
|
|
30224
|
+
currentImpressions: traffic.current.impressions,
|
|
30225
|
+
previousImpressions: traffic.previous.impressions,
|
|
30226
|
+
impressionsChange: traffic.changes.impressions,
|
|
30227
|
+
impressionsPercentChange: traffic.changes.impressionsPercent,
|
|
30228
|
+
ctrPercentagePointChange: traffic.changes.ctrPercentagePoints,
|
|
30229
|
+
currentRange: {
|
|
30230
|
+
startDate: traffic.current.startDate,
|
|
30231
|
+
endDate: traffic.current.endDate
|
|
30232
|
+
},
|
|
30233
|
+
previousRange: {
|
|
30234
|
+
startDate: traffic.previous.startDate,
|
|
30235
|
+
endDate: traffic.previous.endDate
|
|
30236
|
+
}
|
|
30237
|
+
},
|
|
30238
|
+
interpretation: "This is a Bing-observed traffic change. It does not establish the cause or describe Google traffic.",
|
|
30239
|
+
verification: "Review the query and page evidence for the same dates, then compare current pages and crawl health before changing content."
|
|
30240
|
+
});
|
|
30241
|
+
}
|
|
30242
|
+
const crawl = input.crawl;
|
|
30243
|
+
if (crawl?.previous) {
|
|
30244
|
+
const dates = {
|
|
30245
|
+
current: crawl.current.date,
|
|
30246
|
+
previous: crawl.previous.date
|
|
30247
|
+
};
|
|
30248
|
+
const errors = crawl.changes.crawlErrors;
|
|
30249
|
+
if (errors && errors.absolute >= Math.max(10, Math.ceil(errors.previous * .5))) findings.push(changeFinding({
|
|
30250
|
+
code: "bing_crawl_errors_increased",
|
|
30251
|
+
title: "Bing crawl errors increased",
|
|
30252
|
+
...errors,
|
|
30253
|
+
dates,
|
|
30254
|
+
interpretation: "Bing reported more crawl errors in its latest daily snapshot. The count may include URLs outside the current sitemap.",
|
|
30255
|
+
verification: "Run the fast sitemap health check, inspect Bing crawl issue samples, then audit any affected URLs directly."
|
|
30256
|
+
}));
|
|
30257
|
+
const code4xx = crawl.changes.code4xx;
|
|
30258
|
+
if (code4xx && code4xx.absolute >= Math.max(10, Math.ceil(code4xx.previous * .5))) findings.push(changeFinding({
|
|
30259
|
+
code: "bing_4xx_increased",
|
|
30260
|
+
title: "Bing-observed 4xx responses increased",
|
|
30261
|
+
...code4xx,
|
|
30262
|
+
dates,
|
|
30263
|
+
interpretation: "Bing encountered more 4xx responses. Intentional removals and obsolete discovered URLs can contribute to this count.",
|
|
30264
|
+
verification: "Inspect Bing crawl issue URLs and recover or redirect only URLs that should still exist or retain search value."
|
|
30265
|
+
}));
|
|
30266
|
+
const code5xx = crawl.changes.code5xx;
|
|
30267
|
+
if (code5xx && code5xx.current > 0 && code5xx.absolute > 0) findings.push(changeFinding({
|
|
30268
|
+
code: "bing_5xx_increased",
|
|
30269
|
+
title: "Bing-observed server errors increased",
|
|
30270
|
+
...code5xx,
|
|
30271
|
+
dates,
|
|
30272
|
+
interpretation: "Bing encountered more server errors in the latest snapshot. Even a small increase deserves a live check.",
|
|
30273
|
+
verification: "Check current server logs and run targeted URL audits before treating the provider count as an active outage."
|
|
30274
|
+
}));
|
|
30275
|
+
}
|
|
30276
|
+
for (const [kind, analysis] of [["query", input.queries], ["page", input.pages]]) {
|
|
30277
|
+
if (!analysis?.opportunities.length) continue;
|
|
30278
|
+
findings.push({
|
|
30279
|
+
code: `bing_${kind}_opportunities`,
|
|
30280
|
+
severity: "review",
|
|
30281
|
+
title: `${analysis.opportunities.length} Bing ${kind} opportunities need review`,
|
|
30282
|
+
evidence: {
|
|
30283
|
+
retainedOpportunities: analysis.opportunities.length,
|
|
30284
|
+
currentDimensions: analysis.coverage.currentDimensions,
|
|
30285
|
+
matchedDimensions: analysis.coverage.matchedDimensions,
|
|
30286
|
+
comparableDimensions: analysis.coverage.comparableDimensions,
|
|
30287
|
+
thresholds: analysis.thresholds
|
|
30288
|
+
},
|
|
30289
|
+
interpretation: "These are high-impression top-list entries currently observed around positions 4 to 20. The threshold is a prioritization heuristic, not a search-engine rule.",
|
|
30290
|
+
verification: kind === "query" ? "Check intent and the ranking page before changing copy or creating content." : "Audit the selected page and its matching queries before changing internal links, titles, or content."
|
|
30291
|
+
});
|
|
30292
|
+
}
|
|
30293
|
+
return findings;
|
|
30294
|
+
}
|
|
30295
|
+
async function bingWebmasterOverview(input) {
|
|
30296
|
+
if (!input.site.trim()) throw new SeoError("INVALID_INPUT", "Pass a Bing Webmaster site URL.");
|
|
30297
|
+
const resolved = input.client ? {
|
|
30298
|
+
client: input.client,
|
|
30299
|
+
credentialSource: input.credentialSource ?? "environment"
|
|
30300
|
+
} : await createBingWebmasterClient();
|
|
30301
|
+
const observedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
30302
|
+
const topListWarning = "Bing returns weekly top-list rows, so dimensions missing from a period remain unknown.";
|
|
30303
|
+
const [traffic, crawl, queries, pages] = await Promise.all([
|
|
30304
|
+
section(() => resolved.client.getTraffic(input.site), { rowLimit: 400 }),
|
|
30305
|
+
section(() => resolved.client.getCrawlStats(input.site), { rowLimit: 400 }),
|
|
30306
|
+
section(() => resolved.client.getQueryStats(input.site), {
|
|
30307
|
+
rowLimit: 8e3,
|
|
30308
|
+
sourceWarning: topListWarning
|
|
30309
|
+
}),
|
|
30310
|
+
section(() => resolved.client.getPageStats(input.site), {
|
|
30311
|
+
rowLimit: 8e3,
|
|
30312
|
+
sourceWarning: topListWarning
|
|
30313
|
+
})
|
|
30314
|
+
]);
|
|
30315
|
+
const statuses = [
|
|
30316
|
+
traffic.status,
|
|
30317
|
+
crawl.status,
|
|
30318
|
+
queries.status,
|
|
30319
|
+
pages.status
|
|
30320
|
+
];
|
|
30321
|
+
const dataStatus = statuses.every((status) => status === "complete") ? "complete" : statuses.every((status) => status === "unavailable") ? "unavailable" : "partial";
|
|
30322
|
+
const trafficAnalysis = traffic.status === "unavailable" ? void 0 : analyzeBingTraffic(traffic.data.rows);
|
|
30323
|
+
const crawlAnalysis = crawl.status === "unavailable" ? void 0 : analyzeBingCrawl(crawl.data.rows);
|
|
30324
|
+
const queryAnalysis = queries.status === "unavailable" ? void 0 : analyzeBingDimensions(queries.data.rows, "query");
|
|
30325
|
+
const pageAnalysis = pages.status === "unavailable" ? void 0 : analyzeBingDimensions(pages.data.rows, "page");
|
|
30326
|
+
const findings = buildFindings({
|
|
30327
|
+
traffic: trafficAnalysis,
|
|
30328
|
+
crawl: crawlAnalysis,
|
|
30329
|
+
queries: queryAnalysis,
|
|
30330
|
+
pages: pageAnalysis
|
|
30331
|
+
});
|
|
30332
|
+
return {
|
|
30333
|
+
schemaVersion: 2,
|
|
30334
|
+
site: input.site,
|
|
30335
|
+
generatedAt: observedAt,
|
|
30336
|
+
dataStatus,
|
|
30337
|
+
summary: {
|
|
30338
|
+
findings: findings.length,
|
|
30339
|
+
warnings: findings.filter((finding) => finding.severity === "warning").length,
|
|
30340
|
+
reviewItems: findings.filter((finding) => finding.severity === "review").length
|
|
30341
|
+
},
|
|
30342
|
+
findings,
|
|
30343
|
+
outputBudget: {
|
|
30344
|
+
maxFindings: 6,
|
|
30345
|
+
maxDailyRowsPerSection: DAILY_OUTPUT_ROWS,
|
|
30346
|
+
maxDailyRowsTotal: DAILY_OUTPUT_ROWS * 2,
|
|
30347
|
+
maxDimensionItemsPerList: 10,
|
|
30348
|
+
maxDimensionItemsTotal: 40
|
|
30349
|
+
},
|
|
30350
|
+
provenance: {
|
|
30351
|
+
provider: "bing-webmaster",
|
|
30352
|
+
authentication: resolved.client.authentication,
|
|
30353
|
+
credentialSource: resolved.credentialSource,
|
|
30354
|
+
observedAt,
|
|
30355
|
+
cached: false,
|
|
30356
|
+
rowLimits: {
|
|
30357
|
+
traffic: 400,
|
|
30358
|
+
crawl: 400,
|
|
30359
|
+
queries: 8e3,
|
|
30360
|
+
pages: 8e3
|
|
30361
|
+
},
|
|
30362
|
+
methods: [
|
|
30363
|
+
"GetRankAndTrafficStats",
|
|
30364
|
+
"GetCrawlStats",
|
|
30365
|
+
"GetQueryStats",
|
|
30366
|
+
"GetPageStats"
|
|
30367
|
+
]
|
|
30368
|
+
},
|
|
30369
|
+
traffic: traffic.status === "unavailable" ? traffic : {
|
|
30370
|
+
...traffic,
|
|
30371
|
+
data: {
|
|
30372
|
+
...dailyOutput(traffic.data),
|
|
30373
|
+
range: range(traffic.data.rows),
|
|
30374
|
+
clicks: traffic.data.rows.reduce((total, row) => total + row.clicks, 0),
|
|
30375
|
+
impressions: traffic.data.rows.reduce((total, row) => total + row.impressions, 0),
|
|
30376
|
+
analysis: trafficAnalysis
|
|
30377
|
+
}
|
|
30378
|
+
},
|
|
30379
|
+
crawl: crawl.status === "unavailable" ? crawl : {
|
|
30380
|
+
...crawl,
|
|
30381
|
+
data: {
|
|
30382
|
+
...dailyOutput(crawl.data),
|
|
30383
|
+
range: range(crawl.data.rows),
|
|
30384
|
+
latest: crawl.data.rows.at(-1),
|
|
30385
|
+
analysis: crawlAnalysis
|
|
30386
|
+
}
|
|
30387
|
+
},
|
|
30388
|
+
queries: queries.status === "unavailable" ? queries : {
|
|
30389
|
+
...queries,
|
|
30390
|
+
data: {
|
|
30391
|
+
returnedRows: queries.data.returnedRows,
|
|
30392
|
+
retainedRows: queries.data.rows.length,
|
|
30393
|
+
invalidRows: queries.data.invalidRows,
|
|
30394
|
+
capped: queries.data.capped,
|
|
30395
|
+
range: range(queries.data.rows),
|
|
30396
|
+
analysis: queryAnalysis
|
|
30397
|
+
}
|
|
30398
|
+
},
|
|
30399
|
+
pages: pages.status === "unavailable" ? pages : {
|
|
30400
|
+
...pages,
|
|
30401
|
+
data: {
|
|
30402
|
+
returnedRows: pages.data.returnedRows,
|
|
30403
|
+
retainedRows: pages.data.rows.length,
|
|
30404
|
+
invalidRows: pages.data.invalidRows,
|
|
30405
|
+
capped: pages.data.capped,
|
|
30406
|
+
range: range(pages.data.rows),
|
|
30407
|
+
analysis: pageAnalysis
|
|
30408
|
+
}
|
|
30409
|
+
},
|
|
30410
|
+
caveats: [
|
|
30411
|
+
"Bing reports its own observed search and crawl evidence. It is not a complete view of every search engine.",
|
|
30412
|
+
"The inIndex field is provider-reported crawl statistics, not independent proof that a URL is indexed.",
|
|
30413
|
+
"Query and page movements include only dimensions observed in every weekly top list in both periods. Missing entries are not treated as zero.",
|
|
30414
|
+
"Opportunity thresholds prioritize review. They are heuristics, not search-engine requirements."
|
|
30415
|
+
]
|
|
30416
|
+
};
|
|
30417
|
+
}
|
|
30418
|
+
//#endregion
|
|
30419
|
+
//#region packages/core/src/clients.ts
|
|
30420
|
+
function googleAnalyticsPropertyId(client) {
|
|
30421
|
+
return client?.analytics.google?.propertyId;
|
|
30422
|
+
}
|
|
30423
|
+
function bingWebmasterSiteUrl(client) {
|
|
30424
|
+
return client?.searchEngines?.bing?.siteUrl;
|
|
30425
|
+
}
|
|
30426
|
+
function slug(value) {
|
|
30427
|
+
return value.toLowerCase().replace(/^https?:\/\//, "").replace(/^sc-domain:/, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60);
|
|
30428
|
+
}
|
|
30429
|
+
function uniqueStrings(values = []) {
|
|
30430
|
+
return [...new Set(values.map((value) => value.trim()).filter(Boolean))];
|
|
30431
|
+
}
|
|
30432
|
+
function deriveId(input) {
|
|
30433
|
+
const id = slug(input.id ?? input.name ?? input.siteUrl);
|
|
30434
|
+
if (!id) throw new Error("Client id could not be derived.");
|
|
30435
|
+
return id;
|
|
30436
|
+
}
|
|
30437
|
+
function listClients() {
|
|
30438
|
+
return [...readConfig().clients].sort((a, b) => a.name.localeCompare(b.name));
|
|
30439
|
+
}
|
|
30440
|
+
function getClient(idOrName) {
|
|
30441
|
+
const clients = readConfig().clients;
|
|
30442
|
+
if (!idOrName) return clients.find((client) => client.isDefault);
|
|
30443
|
+
const normalized = idOrName.toLowerCase();
|
|
30444
|
+
return clients.find((client) => client.id.toLowerCase() === normalized || client.name.toLowerCase() === normalized);
|
|
30445
|
+
}
|
|
30446
|
+
function saveClient(input) {
|
|
30447
|
+
const config = readConfig();
|
|
30448
|
+
const id = deriveId(input);
|
|
30449
|
+
const existing = config.clients.find((client) => client.id === id);
|
|
30450
|
+
const now = Date.now();
|
|
30451
|
+
const client = {
|
|
30452
|
+
id,
|
|
30453
|
+
name: input.name ?? existing?.name ?? id,
|
|
30454
|
+
siteUrl: input.siteUrl,
|
|
30455
|
+
startUrl: input.startUrl ?? existing?.startUrl,
|
|
30456
|
+
watchUrls: uniqueStrings(input.watchUrls ?? existing?.watchUrls),
|
|
30457
|
+
brandTerms: uniqueStrings(input.brandTerms ?? existing?.brandTerms ?? deriveBrandTerms({
|
|
30458
|
+
id,
|
|
30459
|
+
name: input.name ?? existing?.name,
|
|
30460
|
+
siteUrl: input.siteUrl
|
|
30461
|
+
})),
|
|
30462
|
+
analytics: input.analytics ?? existing?.analytics ?? {},
|
|
30463
|
+
searchEngines: input.searchEngines ?? existing?.searchEngines,
|
|
30464
|
+
reportDay: input.reportDay ?? existing?.reportDay,
|
|
30465
|
+
technicalWeekday: input.technicalWeekday ?? existing?.technicalWeekday,
|
|
30466
|
+
isDefault: input.isDefault ?? existing?.isDefault,
|
|
30467
|
+
createdAt: existing?.createdAt ?? now,
|
|
30468
|
+
updatedAt: now
|
|
30469
|
+
};
|
|
30470
|
+
const clients = config.clients.filter((item) => item.id !== id);
|
|
30471
|
+
if (client.isDefault) {
|
|
30472
|
+
for (const item of clients) item.isDefault = false;
|
|
30473
|
+
config.defaultSite = client.siteUrl;
|
|
30474
|
+
}
|
|
30475
|
+
config.clients = [...clients, client];
|
|
30476
|
+
writeConfig(config);
|
|
30477
|
+
return client;
|
|
30478
|
+
}
|
|
30479
|
+
function setClientBingSite(idOrName, siteUrl) {
|
|
30480
|
+
const client = getClient(idOrName);
|
|
30481
|
+
if (!client) throw new Error(`Client not found: ${idOrName}`);
|
|
30482
|
+
return saveClient({
|
|
30483
|
+
id: client.id,
|
|
30484
|
+
name: client.name,
|
|
30485
|
+
siteUrl: client.siteUrl,
|
|
30486
|
+
searchEngines: {
|
|
30487
|
+
...client.searchEngines,
|
|
30488
|
+
bing: { siteUrl }
|
|
30489
|
+
}
|
|
30490
|
+
});
|
|
30491
|
+
}
|
|
30492
|
+
function deleteClient(idOrName) {
|
|
30493
|
+
const config = readConfig();
|
|
30494
|
+
const client = getClient(idOrName);
|
|
30495
|
+
if (!client) return false;
|
|
30496
|
+
config.clients = config.clients.filter((item) => item.id !== client.id);
|
|
30497
|
+
if (config.defaultSite === client.siteUrl && !config.clients.some((item) => item.siteUrl === client.siteUrl) && !config.sites.some((site) => site.siteUrl === client.siteUrl)) config.defaultSite = void 0;
|
|
30498
|
+
writeConfig(config);
|
|
30499
|
+
return true;
|
|
30500
|
+
}
|
|
30501
|
+
function setDefaultClient(idOrName) {
|
|
30502
|
+
const config = readConfig();
|
|
30503
|
+
const client = getClient(idOrName);
|
|
30504
|
+
if (!client) throw new Error(`Client not found: ${idOrName}`);
|
|
30505
|
+
config.clients = config.clients.map((item) => ({
|
|
30506
|
+
...item,
|
|
30507
|
+
isDefault: item.id === client.id
|
|
30508
|
+
}));
|
|
30509
|
+
config.defaultSite = client.siteUrl;
|
|
30510
|
+
writeConfig(config);
|
|
30511
|
+
return {
|
|
30512
|
+
...client,
|
|
30513
|
+
isDefault: true
|
|
30514
|
+
};
|
|
30515
|
+
}
|
|
30516
|
+
//#endregion
|
|
30517
|
+
//#region packages/core/src/doctor.ts
|
|
30518
|
+
const REQUIRED_SCOPES = ["https://www.googleapis.com/auth/webmasters.readonly", "https://www.googleapis.com/auth/analytics.readonly"];
|
|
30519
|
+
async function runDoctor(dependencies = {}) {
|
|
30520
|
+
const paths = getSeoCliPaths();
|
|
30521
|
+
const config = readConfig();
|
|
30522
|
+
const authMode = getAuthModeStatus();
|
|
30523
|
+
const serviceAccount = getServiceAccountStatus();
|
|
30524
|
+
const tokens = await readTokens();
|
|
30525
|
+
const oauthClient = readOauthClient();
|
|
30526
|
+
const tokenClientConfigured = tokens ? Boolean(getClientConfig(tokens.client_source)) : void 0;
|
|
30527
|
+
const checks = [];
|
|
29470
30528
|
try {
|
|
29471
30529
|
const database = (dependencies.checkDatabase ?? checkDatabaseReadiness)();
|
|
29472
30530
|
checks.push({
|
|
@@ -29538,6 +30596,151 @@ async function runDoctor(dependencies = {}) {
|
|
|
29538
30596
|
};
|
|
29539
30597
|
}
|
|
29540
30598
|
//#endregion
|
|
30599
|
+
//#region packages/core/src/export/report-html.ts
|
|
30600
|
+
function escapeHtml(value) {
|
|
30601
|
+
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll("\"", """).replaceAll("'", "'");
|
|
30602
|
+
}
|
|
30603
|
+
function safeHttpUrl(value) {
|
|
30604
|
+
try {
|
|
30605
|
+
const url = new URL(value);
|
|
30606
|
+
return url.protocol === "http:" || url.protocol === "https:" ? url.toString() : void 0;
|
|
30607
|
+
} catch {
|
|
30608
|
+
return;
|
|
30609
|
+
}
|
|
30610
|
+
}
|
|
30611
|
+
function siteMarkup(site) {
|
|
30612
|
+
const safeUrl = safeHttpUrl(site);
|
|
30613
|
+
const label = escapeHtml(site);
|
|
30614
|
+
return safeUrl ? `<a href="${escapeHtml(safeUrl)}" rel="noreferrer">${label}</a>` : label;
|
|
30615
|
+
}
|
|
30616
|
+
function list(items) {
|
|
30617
|
+
if (items.length === 0) return "<p class=\"empty\">No items to report.</p>";
|
|
30618
|
+
return `<ul>${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul>`;
|
|
30619
|
+
}
|
|
30620
|
+
function reportSections(report) {
|
|
30621
|
+
return report.sections.map((section) => `<section class="panel">
|
|
30622
|
+
<h2>${escapeHtml(section.title)}</h2>
|
|
30623
|
+
${list(section.bullets)}
|
|
30624
|
+
</section>`).join("");
|
|
30625
|
+
}
|
|
30626
|
+
function additionalSections(sections) {
|
|
30627
|
+
return sections.map((section) => `<section class="panel">
|
|
30628
|
+
<h2>${escapeHtml(section.title)}</h2>
|
|
30629
|
+
${section.summary ? `<p>${escapeHtml(section.summary)}</p>` : ""}
|
|
30630
|
+
${list(section.items)}
|
|
30631
|
+
</section>`).join("");
|
|
30632
|
+
}
|
|
30633
|
+
function priorities(report) {
|
|
30634
|
+
if (report.priorities.length === 0) return "<p class=\"empty\">No evidence-backed priorities were found.</p>";
|
|
30635
|
+
return `<ol class="priorities">${report.priorities.map((priority) => `<li>
|
|
30636
|
+
<div class="priority-heading">
|
|
30637
|
+
<strong>${escapeHtml(priority.title)}</strong>
|
|
30638
|
+
<span class="confidence">${escapeHtml(priority.confidence)} confidence</span>
|
|
30639
|
+
</div>
|
|
30640
|
+
<p>${escapeHtml(priority.action)}</p>
|
|
30641
|
+
</li>`).join("")}</ol>`;
|
|
30642
|
+
}
|
|
30643
|
+
function analystEvidence(report) {
|
|
30644
|
+
const skipped = report.diagnosis.skippedSections ?? [];
|
|
30645
|
+
const attempts = report.changeMeasurementAttempts;
|
|
30646
|
+
return `<section class="panel analyst-only">
|
|
30647
|
+
<h2>Evidence coverage</h2>
|
|
30648
|
+
<dl class="facts">
|
|
30649
|
+
<div><dt>Data status</dt><dd>${escapeHtml(report.dataStatus)}</dd></div>
|
|
30650
|
+
<div><dt>Generated</dt><dd>${escapeHtml(report.generatedAt)}</dd></div>
|
|
30651
|
+
<div><dt>Saved changes checked</dt><dd>${attempts.length}</dd></div>
|
|
30652
|
+
<div><dt>Unavailable sections</dt><dd>${skipped.length}</dd></div>
|
|
30653
|
+
</dl>
|
|
30654
|
+
${skipped.length ? `<h3>Unavailable sections</h3>${list(skipped.map((section) => "reason" in section && typeof section.reason === "string" ? `${section.section}: ${section.reason}` : section.section))}` : ""}
|
|
30655
|
+
</section>`;
|
|
30656
|
+
}
|
|
30657
|
+
const styles = `
|
|
30658
|
+
:root { color-scheme: light; font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; color: #17211b; background: #f2f4ef; }
|
|
30659
|
+
* { box-sizing: border-box; }
|
|
30660
|
+
body { margin: 0; line-height: 1.55; }
|
|
30661
|
+
a { color: inherit; text-decoration-color: #ff5b35; text-underline-offset: 0.18em; }
|
|
30662
|
+
main { width: min(980px, calc(100% - 32px)); margin: 32px auto 64px; }
|
|
30663
|
+
header { padding: clamp(28px, 6vw, 64px); color: #f9fbf7; background: #17211b; border-radius: 20px; }
|
|
30664
|
+
.eyebrow { margin: 0 0 12px; color: #ff7958; font-size: 0.78rem; font-weight: 800; letter-spacing: 0.12em; text-transform: uppercase; }
|
|
30665
|
+
h1 { max-width: 760px; margin: 0; font-size: clamp(2rem, 5vw, 4rem); line-height: 1.02; letter-spacing: -0.04em; }
|
|
30666
|
+
header .headline { max-width: 760px; margin: 24px 0 0; font-size: clamp(1.05rem, 2vw, 1.3rem); color: #d9e1d9; }
|
|
30667
|
+
.meta { display: flex; flex-wrap: wrap; gap: 10px 22px; margin-top: 28px; color: #b9c5ba; font-size: 0.9rem; }
|
|
30668
|
+
.status { display: inline-flex; width: fit-content; margin-top: 22px; padding: 6px 10px; border: 1px solid #4f6153; border-radius: 999px; font-size: 0.8rem; font-weight: 700; text-transform: capitalize; }
|
|
30669
|
+
.panel { margin-top: 20px; padding: clamp(22px, 4vw, 38px); background: #fff; border: 1px solid #dfe4dc; border-radius: 16px; box-shadow: 0 8px 30px rgba(23, 33, 27, 0.05); }
|
|
30670
|
+
h2 { margin: 0 0 16px; font-size: clamp(1.25rem, 3vw, 1.65rem); letter-spacing: -0.02em; }
|
|
30671
|
+
h3 { margin: 26px 0 10px; font-size: 1rem; }
|
|
30672
|
+
p { margin: 0 0 14px; }
|
|
30673
|
+
ul, ol { margin: 0; padding-left: 1.25rem; }
|
|
30674
|
+
li + li { margin-top: 10px; }
|
|
30675
|
+
.priorities { list-style: none; padding: 0; counter-reset: priority; }
|
|
30676
|
+
.priorities > li { counter-increment: priority; padding: 18px 0 18px 52px; position: relative; border-top: 1px solid #e6eae4; }
|
|
30677
|
+
.priorities > li:first-child { border-top: 0; padding-top: 4px; }
|
|
30678
|
+
.priorities > li::before { content: counter(priority); position: absolute; left: 0; top: 16px; display: grid; width: 34px; height: 34px; place-items: center; border-radius: 50%; color: #17211b; background: #ff7958; font-weight: 800; }
|
|
30679
|
+
.priorities > li:first-child::before { top: 2px; }
|
|
30680
|
+
.priority-heading { display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 8px 20px; }
|
|
30681
|
+
.confidence { color: #647067; font-size: 0.82rem; text-transform: capitalize; }
|
|
30682
|
+
.priorities p { margin: 8px 0 0; color: #3e4941; }
|
|
30683
|
+
.facts { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 12px; margin: 0; }
|
|
30684
|
+
.facts div { padding: 14px; background: #f5f7f3; border-radius: 10px; }
|
|
30685
|
+
dt { color: #647067; font-size: 0.78rem; font-weight: 700; text-transform: uppercase; }
|
|
30686
|
+
dd { margin: 4px 0 0; font-weight: 700; }
|
|
30687
|
+
details summary { cursor: pointer; font-weight: 750; }
|
|
30688
|
+
details ul { margin-top: 14px; }
|
|
30689
|
+
.empty { color: #647067; font-style: italic; }
|
|
30690
|
+
footer { margin-top: 26px; color: #647067; font-size: 0.8rem; text-align: center; }
|
|
30691
|
+
@media print { :root { background: #fff; } main { width: 100%; margin: 0; } header, .panel { break-inside: avoid; box-shadow: none; } header { border-radius: 0; } }
|
|
30692
|
+
`;
|
|
30693
|
+
function renderReportHtml(input) {
|
|
30694
|
+
const report = input.report;
|
|
30695
|
+
const view = input.view ?? "client";
|
|
30696
|
+
const title = input.title ?? "SEO report";
|
|
30697
|
+
const evidenceNotesOpen = report.dataStatus !== "complete" ? " open" : "";
|
|
30698
|
+
const analyst = view === "analyst" ? analystEvidence(report) : "";
|
|
30699
|
+
return `<!doctype html>
|
|
30700
|
+
<html lang="en">
|
|
30701
|
+
<head>
|
|
30702
|
+
<meta charset="utf-8">
|
|
30703
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
30704
|
+
<meta name="robots" content="noindex,nofollow">
|
|
30705
|
+
<title>${escapeHtml(title)}: ${escapeHtml(report.site)}</title>
|
|
30706
|
+
<style>${styles}</style>
|
|
30707
|
+
</head>
|
|
30708
|
+
<body>
|
|
30709
|
+
<main>
|
|
30710
|
+
<header>
|
|
30711
|
+
<p class="eyebrow">${escapeHtml(title)}</p>
|
|
30712
|
+
<h1>${siteMarkup(report.site)}</h1>
|
|
30713
|
+
<p class="headline">${escapeHtml(report.headline)}</p>
|
|
30714
|
+
<div class="meta">
|
|
30715
|
+
<span>${escapeHtml(report.period.startDate)} to ${escapeHtml(report.period.endDate)}</span>
|
|
30716
|
+
<span>Generated ${escapeHtml(report.generatedAt.slice(0, 10))}</span>
|
|
30717
|
+
<span>${escapeHtml(view)} view</span>
|
|
30718
|
+
</div>
|
|
30719
|
+
<span class="status">${escapeHtml(report.dataStatus)} data</span>
|
|
30720
|
+
</header>
|
|
30721
|
+
|
|
30722
|
+
<section class="panel">
|
|
30723
|
+
<h2>Priorities</h2>
|
|
30724
|
+
${priorities(report)}
|
|
30725
|
+
</section>
|
|
30726
|
+
|
|
30727
|
+
${reportSections(report)}
|
|
30728
|
+
${additionalSections(input.additionalSections ?? [])}
|
|
30729
|
+
${analyst}
|
|
30730
|
+
|
|
30731
|
+
<section class="panel">
|
|
30732
|
+
<details${evidenceNotesOpen}>
|
|
30733
|
+
<summary>Evidence notes and limitations</summary>
|
|
30734
|
+
${list(report.caveats)}
|
|
30735
|
+
</details>
|
|
30736
|
+
</section>
|
|
30737
|
+
|
|
30738
|
+
<footer>Generated locally from observed report evidence.</footer>
|
|
30739
|
+
</main>
|
|
30740
|
+
</body>
|
|
30741
|
+
</html>`;
|
|
30742
|
+
}
|
|
30743
|
+
//#endregion
|
|
29541
30744
|
//#region packages/core/src/export/update-postmortem.ts
|
|
29542
30745
|
const HEADERS = {
|
|
29543
30746
|
summary: [
|
|
@@ -29848,60 +31051,779 @@ function updatePostmortemCsvFiles(report) {
|
|
|
29848
31051
|
headers: [...HEADERS.segment],
|
|
29849
31052
|
rows: postmortemSegmentRows(report, dimension)
|
|
29850
31053
|
})),
|
|
29851
|
-
...[
|
|
29852
|
-
"page",
|
|
29853
|
-
"query",
|
|
29854
|
-
"device",
|
|
29855
|
-
"country"
|
|
29856
|
-
].map((dimension) => ({
|
|
29857
|
-
filename: `postmortem-segment-${dimension}-unmatched.csv`,
|
|
29858
|
-
headers: [...HEADERS.unmatchedSegment],
|
|
29859
|
-
rows: postmortemUnmatchedSegmentRows(report, dimension)
|
|
29860
|
-
}))
|
|
29861
|
-
];
|
|
31054
|
+
...[
|
|
31055
|
+
"page",
|
|
31056
|
+
"query",
|
|
31057
|
+
"device",
|
|
31058
|
+
"country"
|
|
31059
|
+
].map((dimension) => ({
|
|
31060
|
+
filename: `postmortem-segment-${dimension}-unmatched.csv`,
|
|
31061
|
+
headers: [...HEADERS.unmatchedSegment],
|
|
31062
|
+
rows: postmortemUnmatchedSegmentRows(report, dimension)
|
|
31063
|
+
}))
|
|
31064
|
+
];
|
|
31065
|
+
}
|
|
31066
|
+
//#endregion
|
|
31067
|
+
//#region packages/core/src/ga4/matching.ts
|
|
31068
|
+
function hostnameFromUrl(value) {
|
|
31069
|
+
try {
|
|
31070
|
+
const url = new URL(value);
|
|
31071
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
31072
|
+
return url.hostname.toLowerCase().replace(/\.$/, "") || void 0;
|
|
31073
|
+
} catch {
|
|
31074
|
+
return;
|
|
31075
|
+
}
|
|
31076
|
+
}
|
|
31077
|
+
function domainFromProperty(value) {
|
|
31078
|
+
if (!value.startsWith("sc-domain:")) return void 0;
|
|
31079
|
+
const domain = value.slice(10).trim().toLowerCase();
|
|
31080
|
+
if (!domain || /[/:]/.test(domain)) return void 0;
|
|
31081
|
+
return domain.replace(/\.$/, "") || void 0;
|
|
31082
|
+
}
|
|
31083
|
+
function matchForSite(site, streamHostname) {
|
|
31084
|
+
const domain = domainFromProperty(site);
|
|
31085
|
+
if (domain) return streamHostname === domain || streamHostname.endsWith(`.${domain}`) ? "domain" : void 0;
|
|
31086
|
+
return hostnameFromUrl(site) === streamHostname ? "hostname" : void 0;
|
|
31087
|
+
}
|
|
31088
|
+
function matchGa4WebStreams(site, candidates) {
|
|
31089
|
+
return candidates.flatMap((candidate) => {
|
|
31090
|
+
const defaultUri = candidate.stream.webStreamData?.defaultUri;
|
|
31091
|
+
if (!defaultUri) return [];
|
|
31092
|
+
const hostname = hostnameFromUrl(defaultUri);
|
|
31093
|
+
if (!hostname) return [];
|
|
31094
|
+
const match = matchForSite(site, hostname);
|
|
31095
|
+
if (!match) return [];
|
|
31096
|
+
return [{
|
|
31097
|
+
...candidate,
|
|
31098
|
+
defaultUri,
|
|
31099
|
+
hostname,
|
|
31100
|
+
match
|
|
31101
|
+
}];
|
|
31102
|
+
});
|
|
31103
|
+
}
|
|
31104
|
+
function ga4MatchReason(match, site) {
|
|
31105
|
+
const stream = match.stream.displayName ?? match.defaultUri;
|
|
31106
|
+
const location = match.match === "domain" ? "is inside" : "matches";
|
|
31107
|
+
return `Matched Google Analytics web stream ${stream} (${match.defaultUri}): its hostname ${location} ${site}.`;
|
|
31108
|
+
}
|
|
31109
|
+
//#endregion
|
|
31110
|
+
//#region packages/core/src/indexnow/types.ts
|
|
31111
|
+
const INDEXNOW_ENDPOINT = "https://api.indexnow.org/indexnow";
|
|
31112
|
+
const INDEXNOW_MAX_URLS = 1e3;
|
|
31113
|
+
const INDEXNOW_KEY_ENV = "SEO_INDEXNOW_KEY";
|
|
31114
|
+
const INDEXNOW_KEYS_SECRET = "indexnow-keys";
|
|
31115
|
+
//#endregion
|
|
31116
|
+
//#region packages/core/src/indexnow/keys.ts
|
|
31117
|
+
const KEY_PATTERN = /^[A-Za-z0-9-]{8,128}$/;
|
|
31118
|
+
function isPrivateHost(hostname) {
|
|
31119
|
+
const host = hostname.toLowerCase().replace(/^\[|\]$/g, "");
|
|
31120
|
+
if (host === "localhost" || host.endsWith(".localhost") || host.includes(":")) return true;
|
|
31121
|
+
const octets = host.split(".").map(Number);
|
|
31122
|
+
if (octets.length !== 4 || octets.some((value) => !Number.isInteger(value))) return false;
|
|
31123
|
+
const [first, second] = octets;
|
|
31124
|
+
return first === 0 || first === 10 || first === 127 || first === 100 && second !== void 0 && second >= 64 && second <= 127 || first === 169 && second === 254 || first === 172 && second !== void 0 && second >= 16 && second <= 31 || first === 192 && second === 0 || first === 192 && second === 168 || first === 198 && (second === 18 || second === 19) || first !== void 0 && first >= 224;
|
|
31125
|
+
}
|
|
31126
|
+
function parseIndexNowSite(value) {
|
|
31127
|
+
let url;
|
|
31128
|
+
try {
|
|
31129
|
+
url = new URL(value);
|
|
31130
|
+
} catch {
|
|
31131
|
+
throw new SeoError("INVALID_INPUT", "IndexNow site must be a valid URL.");
|
|
31132
|
+
}
|
|
31133
|
+
if (!["http:", "https:"].includes(url.protocol) || !url.hostname) throw new SeoError("INVALID_INPUT", "IndexNow site must use HTTP or HTTPS.");
|
|
31134
|
+
if (url.username || url.password) throw new SeoError("INVALID_INPUT", "IndexNow site URLs cannot contain credentials.");
|
|
31135
|
+
if (isPrivateHost(url.hostname)) throw new SeoError("INVALID_INPUT", "IndexNow requires a public site host.");
|
|
31136
|
+
return url;
|
|
31137
|
+
}
|
|
31138
|
+
function validateIndexNowKey(value) {
|
|
31139
|
+
const key = value.trim();
|
|
31140
|
+
if (!KEY_PATTERN.test(key)) throw new SeoError("INVALID_INPUT", "IndexNow keys must contain 8 to 128 letters, numbers, or hyphens.");
|
|
31141
|
+
return key;
|
|
31142
|
+
}
|
|
31143
|
+
function createIndexNowKey() {
|
|
31144
|
+
return randomBytes(16).toString("hex");
|
|
31145
|
+
}
|
|
31146
|
+
function createIndexNowKeyRecord(input) {
|
|
31147
|
+
const site = parseIndexNowSite(input.site);
|
|
31148
|
+
const key = validateIndexNowKey(input.key ?? createIndexNowKey());
|
|
31149
|
+
const keyLocation = input.keyLocation ? parseIndexNowSite(input.keyLocation) : new URL(`/${key}.txt`, site.origin);
|
|
31150
|
+
if (keyLocation.hostname.toLowerCase() !== site.hostname.toLowerCase()) throw new SeoError("INVALID_INPUT", "IndexNow key location must use the same host as the site.");
|
|
31151
|
+
return {
|
|
31152
|
+
host: site.hostname.toLowerCase(),
|
|
31153
|
+
key,
|
|
31154
|
+
keyLocation: keyLocation.toString(),
|
|
31155
|
+
createdAt: (input.now ?? /* @__PURE__ */ new Date()).toISOString()
|
|
31156
|
+
};
|
|
31157
|
+
}
|
|
31158
|
+
function validateIndexNowKeyRecord(record) {
|
|
31159
|
+
const createdAt = new Date(record.createdAt);
|
|
31160
|
+
if (Number.isNaN(createdAt.getTime())) throw new SeoError("INVALID_INPUT", "IndexNow key record has an invalid creation date.");
|
|
31161
|
+
const normalized = createIndexNowKeyRecord({
|
|
31162
|
+
site: record.keyLocation,
|
|
31163
|
+
key: record.key,
|
|
31164
|
+
keyLocation: record.keyLocation,
|
|
31165
|
+
now: createdAt
|
|
31166
|
+
});
|
|
31167
|
+
if (normalized.host !== record.host.toLowerCase()) throw new SeoError("INVALID_INPUT", "IndexNow key record host does not match its key location.");
|
|
31168
|
+
return normalized;
|
|
31169
|
+
}
|
|
31170
|
+
function emptyKeys() {
|
|
31171
|
+
return {
|
|
31172
|
+
version: 1,
|
|
31173
|
+
sites: {}
|
|
31174
|
+
};
|
|
31175
|
+
}
|
|
31176
|
+
function parseSavedKeys(value) {
|
|
31177
|
+
try {
|
|
31178
|
+
const parsed = JSON.parse(value);
|
|
31179
|
+
if (parsed.version !== 1 || !parsed.sites || typeof parsed.sites !== "object" || Array.isArray(parsed.sites)) throw new Error("invalid shape");
|
|
31180
|
+
const sites = {};
|
|
31181
|
+
for (const [host, value] of Object.entries(parsed.sites)) {
|
|
31182
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("invalid key record");
|
|
31183
|
+
const record = validateIndexNowKeyRecord(value);
|
|
31184
|
+
if (record.host !== host) throw new Error("invalid host mapping");
|
|
31185
|
+
sites[host] = record;
|
|
31186
|
+
}
|
|
31187
|
+
return {
|
|
31188
|
+
version: 1,
|
|
31189
|
+
sites
|
|
31190
|
+
};
|
|
31191
|
+
} catch {
|
|
31192
|
+
throw new SeoError("AUTH_REQUIRED", "Saved IndexNow keys are invalid. Run `seo indexnow setup` again.");
|
|
31193
|
+
}
|
|
31194
|
+
}
|
|
31195
|
+
async function readSavedKeys() {
|
|
31196
|
+
const saved = await readProviderSecret({ name: INDEXNOW_KEYS_SECRET });
|
|
31197
|
+
if (!saved) return { keys: emptyKeys() };
|
|
31198
|
+
return {
|
|
31199
|
+
keys: parseSavedKeys(saved.value),
|
|
31200
|
+
source: saved.source
|
|
31201
|
+
};
|
|
31202
|
+
}
|
|
31203
|
+
async function saveIndexNowKey(record) {
|
|
31204
|
+
const normalized = validateIndexNowKeyRecord(record);
|
|
31205
|
+
const { keys } = await readSavedKeys();
|
|
31206
|
+
keys.sites[normalized.host] = normalized;
|
|
31207
|
+
return writeProviderSecret(INDEXNOW_KEYS_SECRET, JSON.stringify(keys));
|
|
31208
|
+
}
|
|
31209
|
+
async function resolveIndexNowKey(input) {
|
|
31210
|
+
const site = parseIndexNowSite(input.site);
|
|
31211
|
+
const environmentKey = (input.env ?? process.env)[INDEXNOW_KEY_ENV]?.trim();
|
|
31212
|
+
if (environmentKey) return {
|
|
31213
|
+
record: createIndexNowKeyRecord({
|
|
31214
|
+
site: site.origin,
|
|
31215
|
+
key: environmentKey
|
|
31216
|
+
}),
|
|
31217
|
+
source: "environment"
|
|
31218
|
+
};
|
|
31219
|
+
const { keys, source } = await readSavedKeys();
|
|
31220
|
+
const record = keys.sites[site.hostname.toLowerCase()];
|
|
31221
|
+
if (!record || !source) throw new SeoError("AUTH_REQUIRED", `IndexNow is not set up for ${site.hostname}. Run \`seo indexnow setup --site ${site.origin} --output <public-directory>\`, or set SEO_INDEXNOW_KEY for this process.`);
|
|
31222
|
+
return {
|
|
31223
|
+
record: validateIndexNowKeyRecord(record),
|
|
31224
|
+
source
|
|
31225
|
+
};
|
|
31226
|
+
}
|
|
31227
|
+
async function listIndexNowKeys() {
|
|
31228
|
+
const { keys } = await readSavedKeys();
|
|
31229
|
+
return Object.values(keys.sites).map(({ key: _key, ...record }) => record).sort((a, b) => a.host < b.host ? -1 : a.host > b.host ? 1 : 0);
|
|
31230
|
+
}
|
|
31231
|
+
async function removeIndexNowKey(siteValue) {
|
|
31232
|
+
const site = parseIndexNowSite(siteValue);
|
|
31233
|
+
const { keys } = await readSavedKeys();
|
|
31234
|
+
const host = site.hostname.toLowerCase();
|
|
31235
|
+
if (!keys.sites[host]) return false;
|
|
31236
|
+
delete keys.sites[host];
|
|
31237
|
+
if (Object.keys(keys.sites).length === 0) await deleteProviderSecret(INDEXNOW_KEYS_SECRET);
|
|
31238
|
+
else await writeProviderSecret(INDEXNOW_KEYS_SECRET, JSON.stringify(keys));
|
|
31239
|
+
return true;
|
|
31240
|
+
}
|
|
31241
|
+
//#endregion
|
|
31242
|
+
//#region packages/core/src/indexnow/client.ts
|
|
31243
|
+
function uniqueUrls(values, host) {
|
|
31244
|
+
if (values.length === 0) throw new SeoError("INVALID_INPUT", "Pass at least one URL to IndexNow.");
|
|
31245
|
+
if (values.length > 1e3) throw new SeoError("INVALID_INPUT", `IndexNow submissions are limited to ${INDEXNOW_MAX_URLS} URLs per run.`);
|
|
31246
|
+
const urls = /* @__PURE__ */ new Set();
|
|
31247
|
+
for (const value of values) {
|
|
31248
|
+
const url = parseIndexNowSite(value);
|
|
31249
|
+
if (url.toString().length > 2e3) throw new SeoError("INVALID_INPUT", "IndexNow URLs cannot exceed 2,000 characters.");
|
|
31250
|
+
if (url.hostname.toLowerCase() !== host) throw new SeoError("INVALID_INPUT", `Every IndexNow URL must belong to ${host}.`);
|
|
31251
|
+
url.hash = "";
|
|
31252
|
+
urls.add(url.toString());
|
|
31253
|
+
}
|
|
31254
|
+
return [...urls].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
31255
|
+
}
|
|
31256
|
+
async function verifyIndexNowKey(input) {
|
|
31257
|
+
const record = validateIndexNowKeyRecord(input.record);
|
|
31258
|
+
const keyLocation = parseIndexNowSite(record.keyLocation);
|
|
31259
|
+
let response;
|
|
31260
|
+
try {
|
|
31261
|
+
response = await (input.fetchImpl ?? publicHttpFetch)(keyLocation.toString(), {
|
|
31262
|
+
headers: { accept: "text/plain" },
|
|
31263
|
+
redirect: "manual",
|
|
31264
|
+
signal: AbortSignal.timeout(input.timeoutMs ?? 15e3)
|
|
31265
|
+
});
|
|
31266
|
+
} catch (error) {
|
|
31267
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `IndexNow key verification failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
31268
|
+
}
|
|
31269
|
+
const body = await readBoundedResponseText(response, 512, "IndexNow key file");
|
|
31270
|
+
return {
|
|
31271
|
+
verified: response.ok && body.trim() === record.key,
|
|
31272
|
+
status: response.status,
|
|
31273
|
+
keyLocation: keyLocation.toString()
|
|
31274
|
+
};
|
|
31275
|
+
}
|
|
31276
|
+
function indexNowFailure(status) {
|
|
31277
|
+
if (status === 403) return new SeoError("ACCESS_DENIED", "IndexNow could not validate the key. Confirm the public key file contains the saved key.");
|
|
31278
|
+
if (status === 429) return new SeoError("RATE_LIMITED", "IndexNow rate limited this submission. Wait before trying again.");
|
|
31279
|
+
if (status === 400 || status === 422) return new SeoError("INVALID_INPUT", `IndexNow rejected the submission with HTTP ${status}. Check the URL host and key configuration.`);
|
|
31280
|
+
return new SeoError("PROVIDER_UNAVAILABLE", `IndexNow submission failed with HTTP ${status}.`);
|
|
31281
|
+
}
|
|
31282
|
+
async function submitIndexNow(input) {
|
|
31283
|
+
const record = validateIndexNowKeyRecord(input.record);
|
|
31284
|
+
const urls = uniqueUrls(input.urls, record.host);
|
|
31285
|
+
const keyLocation = parseIndexNowSite(record.keyLocation);
|
|
31286
|
+
const base = {
|
|
31287
|
+
schemaVersion: 1,
|
|
31288
|
+
generatedAt: (input.now ?? /* @__PURE__ */ new Date()).toISOString(),
|
|
31289
|
+
endpoint: INDEXNOW_ENDPOINT,
|
|
31290
|
+
host: record.host,
|
|
31291
|
+
keyLocation: keyLocation.toString(),
|
|
31292
|
+
submittedUrls: urls.length,
|
|
31293
|
+
caveats: ["IndexNow confirms receipt, not crawling, indexing, ranking, or traffic.", "Submit URLs only after they are added, updated, or deleted."]
|
|
31294
|
+
};
|
|
31295
|
+
if (input.dryRun) return {
|
|
31296
|
+
...base,
|
|
31297
|
+
dryRun: true,
|
|
31298
|
+
status: "validated"
|
|
31299
|
+
};
|
|
31300
|
+
const verification = await verifyIndexNowKey({
|
|
31301
|
+
record,
|
|
31302
|
+
fetchImpl: input.fetchImpl
|
|
31303
|
+
});
|
|
31304
|
+
if (!verification.verified) throw new SeoError("ACCESS_DENIED", `IndexNow key verification failed at ${verification.keyLocation} with HTTP ${verification.status}. Deploy the generated key file before submitting URLs.`);
|
|
31305
|
+
let response;
|
|
31306
|
+
try {
|
|
31307
|
+
response = await (input.fetchImpl ?? publicHttpFetch)(INDEXNOW_ENDPOINT, {
|
|
31308
|
+
method: "POST",
|
|
31309
|
+
headers: {
|
|
31310
|
+
accept: "application/json",
|
|
31311
|
+
"content-type": "application/json; charset=utf-8"
|
|
31312
|
+
},
|
|
31313
|
+
body: JSON.stringify({
|
|
31314
|
+
host: record.host,
|
|
31315
|
+
key: record.key,
|
|
31316
|
+
keyLocation: keyLocation.toString(),
|
|
31317
|
+
urlList: urls
|
|
31318
|
+
}),
|
|
31319
|
+
signal: AbortSignal.timeout(3e4)
|
|
31320
|
+
});
|
|
31321
|
+
} catch (error) {
|
|
31322
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `IndexNow did not respond: ${error instanceof Error ? error.message : String(error)}`);
|
|
31323
|
+
}
|
|
31324
|
+
await readBoundedResponseText(response, 64e3, "IndexNow response");
|
|
31325
|
+
if (response.status !== 200 && response.status !== 202) throw indexNowFailure(response.status);
|
|
31326
|
+
return {
|
|
31327
|
+
...base,
|
|
31328
|
+
dryRun: false,
|
|
31329
|
+
status: response.status === 200 ? "submitted" : "pending-validation",
|
|
31330
|
+
responseStatus: response.status
|
|
31331
|
+
};
|
|
31332
|
+
}
|
|
31333
|
+
//#endregion
|
|
31334
|
+
//#region packages/core/src/links/normalize.ts
|
|
31335
|
+
const FIELD_ALIASES = {
|
|
31336
|
+
sourceUrl: [
|
|
31337
|
+
"sourceurl",
|
|
31338
|
+
"source",
|
|
31339
|
+
"referringurl",
|
|
31340
|
+
"referringpage",
|
|
31341
|
+
"fromurl"
|
|
31342
|
+
],
|
|
31343
|
+
targetUrl: [
|
|
31344
|
+
"targeturl",
|
|
31345
|
+
"target",
|
|
31346
|
+
"destinationurl",
|
|
31347
|
+
"destination",
|
|
31348
|
+
"tourl"
|
|
31349
|
+
],
|
|
31350
|
+
anchorText: ["anchortext", "anchor"],
|
|
31351
|
+
firstSeenAt: ["firstseenat", "firstseen"],
|
|
31352
|
+
lastSeenAt: ["lastseenat", "lastseen"],
|
|
31353
|
+
nofollow: ["nofollow", "isnofollow"]
|
|
31354
|
+
};
|
|
31355
|
+
function key(value) {
|
|
31356
|
+
return value.toLowerCase().replaceAll(/[^a-z0-9]/g, "");
|
|
31357
|
+
}
|
|
31358
|
+
function valueFor(row, field) {
|
|
31359
|
+
const aliases = new Set(FIELD_ALIASES[field]);
|
|
31360
|
+
for (const [name, value] of Object.entries(row)) if (aliases.has(key(name))) return value;
|
|
31361
|
+
}
|
|
31362
|
+
function httpUrl(value) {
|
|
31363
|
+
if (typeof value !== "string" || !value.trim() || value.trim().length > 2e3) return;
|
|
31364
|
+
try {
|
|
31365
|
+
const url = new URL(value.trim());
|
|
31366
|
+
return url.protocol === "http:" || url.protocol === "https:" ? url : void 0;
|
|
31367
|
+
} catch {
|
|
31368
|
+
return;
|
|
31369
|
+
}
|
|
31370
|
+
}
|
|
31371
|
+
function optionalString(value) {
|
|
31372
|
+
return typeof value === "string" && value.trim() ? value.trim().slice(0, 1e3) : void 0;
|
|
31373
|
+
}
|
|
31374
|
+
function optionalBoolean(value) {
|
|
31375
|
+
if (typeof value === "boolean") return value;
|
|
31376
|
+
if (typeof value !== "string") return void 0;
|
|
31377
|
+
if (/^(true|yes|1)$/i.test(value.trim())) return true;
|
|
31378
|
+
if (/^(false|no|0)$/i.test(value.trim())) return false;
|
|
31379
|
+
}
|
|
31380
|
+
function normalizeLinkEvidenceRow(raw) {
|
|
31381
|
+
const source = httpUrl(valueFor(raw, "sourceUrl"));
|
|
31382
|
+
const target = httpUrl(valueFor(raw, "targetUrl"));
|
|
31383
|
+
if (!source || !target) return void 0;
|
|
31384
|
+
return {
|
|
31385
|
+
sourceUrl: source.toString(),
|
|
31386
|
+
sourceDomain: source.hostname.toLowerCase(),
|
|
31387
|
+
targetUrl: target.toString(),
|
|
31388
|
+
anchorText: optionalString(valueFor(raw, "anchorText")),
|
|
31389
|
+
firstSeenAt: optionalString(valueFor(raw, "firstSeenAt")),
|
|
31390
|
+
lastSeenAt: optionalString(valueFor(raw, "lastSeenAt")),
|
|
31391
|
+
nofollow: optionalBoolean(valueFor(raw, "nofollow"))
|
|
31392
|
+
};
|
|
31393
|
+
}
|
|
31394
|
+
function linkEvidenceKey(row) {
|
|
31395
|
+
return `${row.sourceUrl}\u0000${row.targetUrl}\u0000${row.anchorText ?? ""}`;
|
|
31396
|
+
}
|
|
31397
|
+
function compareLinkEvidence(a, b) {
|
|
31398
|
+
return a.targetUrl.localeCompare(b.targetUrl, "en") || a.sourceUrl.localeCompare(b.sourceUrl, "en") || (a.anchorText ?? "").localeCompare(b.anchorText ?? "", "en");
|
|
31399
|
+
}
|
|
31400
|
+
//#endregion
|
|
31401
|
+
//#region packages/core/src/links/bing.ts
|
|
31402
|
+
const MAX_COUNT_PAGES = 5;
|
|
31403
|
+
const DEFAULT_TARGET_LIMIT = 20;
|
|
31404
|
+
const MAX_TARGET_LIMIT = 50;
|
|
31405
|
+
const DEFAULT_DETAIL_PAGES = 1;
|
|
31406
|
+
const MAX_DETAIL_PAGES = 3;
|
|
31407
|
+
const DEFAULT_ROW_LIMIT$2 = 500;
|
|
31408
|
+
const MAX_ROW_LIMIT$2 = 1e3;
|
|
31409
|
+
const MAX_CONCURRENCY = 3;
|
|
31410
|
+
function integer(value, fallback, max, name) {
|
|
31411
|
+
const resolved = value ?? fallback;
|
|
31412
|
+
if (!Number.isInteger(resolved) || resolved < 1 || resolved > max) throw new SeoError("INVALID_INPUT", `${name} must be between 1 and ${max}.`);
|
|
31413
|
+
return resolved;
|
|
31414
|
+
}
|
|
31415
|
+
async function mapConcurrent(values, concurrency, work) {
|
|
31416
|
+
const output = new Array(values.length);
|
|
31417
|
+
let next = 0;
|
|
31418
|
+
async function worker() {
|
|
31419
|
+
while (next < values.length) {
|
|
31420
|
+
const index = next;
|
|
31421
|
+
next += 1;
|
|
31422
|
+
const value = values[index];
|
|
31423
|
+
if (value !== void 0) output[index] = await work(value);
|
|
31424
|
+
}
|
|
31425
|
+
}
|
|
31426
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, () => worker()));
|
|
31427
|
+
return output;
|
|
31428
|
+
}
|
|
31429
|
+
async function targetDetails(input) {
|
|
31430
|
+
const rows = [];
|
|
31431
|
+
let requestedPages = 0;
|
|
31432
|
+
let invalidRows = 0;
|
|
31433
|
+
let returnedRows = 0;
|
|
31434
|
+
let partial = false;
|
|
31435
|
+
let totalPages = 0;
|
|
31436
|
+
for (let page = 0; page < input.pages; page += 1) {
|
|
31437
|
+
requestedPages += 1;
|
|
31438
|
+
let response;
|
|
31439
|
+
try {
|
|
31440
|
+
response = await input.client.getUrlLinks(input.site, input.target.url, page);
|
|
31441
|
+
} catch (error) {
|
|
31442
|
+
if (error instanceof SeoError && (error.code === "AUTH_REQUIRED" || error.code === "ACCESS_DENIED")) throw error;
|
|
31443
|
+
return {
|
|
31444
|
+
rows,
|
|
31445
|
+
requestedPages,
|
|
31446
|
+
invalidRows,
|
|
31447
|
+
returnedRows,
|
|
31448
|
+
partial: true,
|
|
31449
|
+
failed: true
|
|
31450
|
+
};
|
|
31451
|
+
}
|
|
31452
|
+
invalidRows += response.invalidRows;
|
|
31453
|
+
returnedRows += response.returnedRows;
|
|
31454
|
+
rows.push(...response.rows);
|
|
31455
|
+
totalPages = response.totalPages;
|
|
31456
|
+
partial ||= response.capped;
|
|
31457
|
+
if (page + 1 >= response.totalPages || response.rows.length === 0) break;
|
|
31458
|
+
}
|
|
31459
|
+
partial ||= requestedPages < totalPages;
|
|
31460
|
+
return {
|
|
31461
|
+
rows,
|
|
31462
|
+
requestedPages,
|
|
31463
|
+
invalidRows,
|
|
31464
|
+
returnedRows,
|
|
31465
|
+
partial,
|
|
31466
|
+
failed: false
|
|
31467
|
+
};
|
|
31468
|
+
}
|
|
31469
|
+
async function collectBingLinkEvidence(input) {
|
|
31470
|
+
if (!input.site.trim()) throw new SeoError("INVALID_INPUT", "Pass a Bing Webmaster site URL.");
|
|
31471
|
+
const targetLimit = integer(input.targetLimit, DEFAULT_TARGET_LIMIT, MAX_TARGET_LIMIT, "Bing link target limit");
|
|
31472
|
+
const detailPages = integer(input.detailPagesPerTarget, DEFAULT_DETAIL_PAGES, MAX_DETAIL_PAGES, "Bing detail page limit");
|
|
31473
|
+
const rowLimit = integer(input.rowLimit, DEFAULT_ROW_LIMIT$2, MAX_ROW_LIMIT$2, "Bing link row limit");
|
|
31474
|
+
const client = input.client ?? (await createBingWebmasterClient()).client;
|
|
31475
|
+
const countRows = [];
|
|
31476
|
+
let countPagesRequested = 0;
|
|
31477
|
+
let invalidRows = 0;
|
|
31478
|
+
let countInvalidRows = 0;
|
|
31479
|
+
let countRowsReturned = 0;
|
|
31480
|
+
let partial = false;
|
|
31481
|
+
let totalCountPages = 0;
|
|
31482
|
+
for (let page = 0; page < MAX_COUNT_PAGES; page += 1) {
|
|
31483
|
+
const response = await client.getLinkCounts(input.site, page);
|
|
31484
|
+
countPagesRequested += 1;
|
|
31485
|
+
invalidRows += response.invalidRows;
|
|
31486
|
+
countInvalidRows += response.invalidRows;
|
|
31487
|
+
countRowsReturned += response.returnedRows;
|
|
31488
|
+
countRows.push(...response.rows);
|
|
31489
|
+
totalCountPages = response.totalPages;
|
|
31490
|
+
partial ||= response.capped;
|
|
31491
|
+
if (countRows.length >= targetLimit || page + 1 >= response.totalPages || response.rows.length === 0) break;
|
|
31492
|
+
}
|
|
31493
|
+
partial ||= countPagesRequested < totalCountPages;
|
|
31494
|
+
const countsByUrl = /* @__PURE__ */ new Map();
|
|
31495
|
+
for (const row of countRows) countsByUrl.set(row.url, Math.max(row.count, countsByUrl.get(row.url) ?? 0));
|
|
31496
|
+
const counts = [...countsByUrl].map(([url, count]) => ({
|
|
31497
|
+
url,
|
|
31498
|
+
count
|
|
31499
|
+
}));
|
|
31500
|
+
counts.sort((a, b) => b.count - a.count || a.url.localeCompare(b.url, "en"));
|
|
31501
|
+
const selectedTargets = counts.slice(0, targetLimit);
|
|
31502
|
+
partial ||= counts.length > targetLimit;
|
|
31503
|
+
const details = await mapConcurrent(selectedTargets, MAX_CONCURRENCY, async (target) => targetDetails({
|
|
31504
|
+
client,
|
|
31505
|
+
site: input.site,
|
|
31506
|
+
target,
|
|
31507
|
+
pages: detailPages
|
|
31508
|
+
}));
|
|
31509
|
+
const rows = [];
|
|
31510
|
+
const keys = /* @__PURE__ */ new Set();
|
|
31511
|
+
let duplicateRows = 0;
|
|
31512
|
+
let detailPagesRequested = 0;
|
|
31513
|
+
let detailInvalidRows = 0;
|
|
31514
|
+
let detailRowsReturned = 0;
|
|
31515
|
+
let failedTargets = 0;
|
|
31516
|
+
for (const [index, detail] of details.entries()) {
|
|
31517
|
+
const target = selectedTargets[index];
|
|
31518
|
+
if (!target) continue;
|
|
31519
|
+
detailPagesRequested += detail.requestedPages;
|
|
31520
|
+
if (detail.failed) failedTargets += 1;
|
|
31521
|
+
invalidRows += detail.invalidRows;
|
|
31522
|
+
detailInvalidRows += detail.invalidRows;
|
|
31523
|
+
detailRowsReturned += detail.returnedRows;
|
|
31524
|
+
partial ||= detail.partial;
|
|
31525
|
+
for (const source of detail.rows) {
|
|
31526
|
+
if (rows.length >= rowLimit) {
|
|
31527
|
+
partial = true;
|
|
31528
|
+
break;
|
|
31529
|
+
}
|
|
31530
|
+
const row = normalizeLinkEvidenceRow({
|
|
31531
|
+
sourceUrl: source.url,
|
|
31532
|
+
targetUrl: target.url,
|
|
31533
|
+
anchorText: source.anchorText
|
|
31534
|
+
});
|
|
31535
|
+
if (!row) {
|
|
31536
|
+
invalidRows += 1;
|
|
31537
|
+
continue;
|
|
31538
|
+
}
|
|
31539
|
+
const key = linkEvidenceKey(row);
|
|
31540
|
+
if (keys.has(key)) {
|
|
31541
|
+
duplicateRows += 1;
|
|
31542
|
+
continue;
|
|
31543
|
+
}
|
|
31544
|
+
keys.add(key);
|
|
31545
|
+
rows.push(row);
|
|
31546
|
+
}
|
|
31547
|
+
}
|
|
31548
|
+
rows.sort(compareLinkEvidence);
|
|
31549
|
+
return {
|
|
31550
|
+
rows,
|
|
31551
|
+
targetCounts: selectedTargets.map((target) => ({
|
|
31552
|
+
targetUrl: target.url,
|
|
31553
|
+
providerReportedLinks: target.count,
|
|
31554
|
+
observedLinks: rows.filter((row) => row.targetUrl === target.url).length
|
|
31555
|
+
})),
|
|
31556
|
+
provenance: {
|
|
31557
|
+
provider: "bing-webmaster",
|
|
31558
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
31559
|
+
cached: false,
|
|
31560
|
+
suppliedRows: detailRowsReturned + detailInvalidRows,
|
|
31561
|
+
validRows: rows.length,
|
|
31562
|
+
invalidRows,
|
|
31563
|
+
duplicateRows,
|
|
31564
|
+
capped: partial,
|
|
31565
|
+
rowLimit,
|
|
31566
|
+
completeness: partial ? "partial" : "unknown",
|
|
31567
|
+
providerRequests: {
|
|
31568
|
+
methods: ["GetLinkCounts", "GetUrlLinks"],
|
|
31569
|
+
targetPagesRequested: countPagesRequested,
|
|
31570
|
+
detailPagesRequested,
|
|
31571
|
+
maxConcurrentRequests: MAX_CONCURRENCY
|
|
31572
|
+
},
|
|
31573
|
+
providerCoverage: {
|
|
31574
|
+
targetCountRows: {
|
|
31575
|
+
returnedRows: countRowsReturned,
|
|
31576
|
+
retainedRows: counts.length,
|
|
31577
|
+
invalidRows: countInvalidRows
|
|
31578
|
+
},
|
|
31579
|
+
detailRows: {
|
|
31580
|
+
returnedRows: detailRowsReturned,
|
|
31581
|
+
retainedRows: rows.length,
|
|
31582
|
+
invalidRows: detailInvalidRows
|
|
31583
|
+
}
|
|
31584
|
+
}
|
|
31585
|
+
},
|
|
31586
|
+
warnings: [
|
|
31587
|
+
...countRowsReturned === 0 ? ["Bing returned no target-link rows. This does not prove that the site has no links."] : [],
|
|
31588
|
+
...partial ? ["Bing link evidence was bounded by provider pagination, failed requests, or report limits."] : [],
|
|
31589
|
+
...failedTargets ? [`${failedTargets} target detail request${failedTargets === 1 ? "" : "s"} failed while other evidence was retained.`] : []
|
|
31590
|
+
]
|
|
31591
|
+
};
|
|
29862
31592
|
}
|
|
29863
31593
|
//#endregion
|
|
29864
|
-
//#region packages/core/src/
|
|
29865
|
-
|
|
31594
|
+
//#region packages/core/src/links/import.ts
|
|
31595
|
+
const DEFAULT_ROW_LIMIT$1 = 1e4;
|
|
31596
|
+
const MAX_ROW_LIMIT$1 = 1e5;
|
|
31597
|
+
const MAX_STREAM_BYTES = 5e7;
|
|
31598
|
+
const MAX_JSON_BYTES = 1e7;
|
|
31599
|
+
const MAX_CELL_BYTES = 1e6;
|
|
31600
|
+
function formatFor$1(path, explicit) {
|
|
31601
|
+
if (explicit) return explicit;
|
|
31602
|
+
const extension = extname(path).toLowerCase();
|
|
31603
|
+
if (extension === ".csv") return "csv";
|
|
31604
|
+
if (extension === ".jsonl" || extension === ".ndjson") return "jsonl";
|
|
31605
|
+
if (extension === ".json") return "json";
|
|
31606
|
+
throw new SeoError("INVALID_INPUT", "Link imports must be CSV, JSON, JSONL, or NDJSON.");
|
|
31607
|
+
}
|
|
31608
|
+
function boundedRowLimit(value) {
|
|
31609
|
+
const limit = value ?? DEFAULT_ROW_LIMIT$1;
|
|
31610
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_ROW_LIMIT$1) throw new SeoError("INVALID_INPUT", `Link import row limit must be between 1 and ${MAX_ROW_LIMIT$1}.`);
|
|
31611
|
+
return limit;
|
|
31612
|
+
}
|
|
31613
|
+
async function* csvRecords(path, byteLimit, progress) {
|
|
31614
|
+
const stream = createReadStream(path, { encoding: "utf8" });
|
|
31615
|
+
let record = [];
|
|
31616
|
+
let field = "";
|
|
31617
|
+
let quoted = false;
|
|
31618
|
+
let pendingQuote = false;
|
|
31619
|
+
for await (const chunk of stream) {
|
|
31620
|
+
progress.bytesRead += Buffer.byteLength(chunk);
|
|
31621
|
+
if (progress.bytesRead > byteLimit) {
|
|
31622
|
+
stream.destroy();
|
|
31623
|
+
throw new SeoError("INVALID_INPUT", `Link import exceeds the ${byteLimit}-byte streaming limit.`);
|
|
31624
|
+
}
|
|
31625
|
+
for (const character of chunk) {
|
|
31626
|
+
if (pendingQuote) {
|
|
31627
|
+
if (character === "\"") {
|
|
31628
|
+
field += "\"";
|
|
31629
|
+
pendingQuote = false;
|
|
31630
|
+
continue;
|
|
31631
|
+
}
|
|
31632
|
+
quoted = false;
|
|
31633
|
+
pendingQuote = false;
|
|
31634
|
+
}
|
|
31635
|
+
if (quoted) if (character === "\"") pendingQuote = true;
|
|
31636
|
+
else field += character;
|
|
31637
|
+
else if (character === "\"" && field.length === 0) quoted = true;
|
|
31638
|
+
else if (character === ",") {
|
|
31639
|
+
record.push(field);
|
|
31640
|
+
field = "";
|
|
31641
|
+
} else if (character === "\n") {
|
|
31642
|
+
record.push(field.replace(/\r$/, ""));
|
|
31643
|
+
field = "";
|
|
31644
|
+
if (record.some((value) => value.length > 0)) yield record;
|
|
31645
|
+
record = [];
|
|
31646
|
+
} else field += character;
|
|
31647
|
+
if (Buffer.byteLength(field) > MAX_CELL_BYTES) {
|
|
31648
|
+
stream.destroy();
|
|
31649
|
+
throw new SeoError("INVALID_INPUT", `Link import contains a cell larger than ${MAX_CELL_BYTES} bytes.`);
|
|
31650
|
+
}
|
|
31651
|
+
}
|
|
31652
|
+
}
|
|
31653
|
+
if (quoted && !pendingQuote) throw new SeoError("INVALID_INPUT", "Link CSV contains an unclosed quote.");
|
|
31654
|
+
if (field || record.length) {
|
|
31655
|
+
record.push(field.replace(/\r$/, ""));
|
|
31656
|
+
if (record.some((value) => value.length > 0)) yield record;
|
|
31657
|
+
}
|
|
31658
|
+
}
|
|
31659
|
+
async function* jsonlRecords(path, byteLimit, progress) {
|
|
31660
|
+
const stream = createReadStream(path, { encoding: "utf8" });
|
|
31661
|
+
let remainder = "";
|
|
31662
|
+
for await (const chunk of stream) {
|
|
31663
|
+
progress.bytesRead += Buffer.byteLength(chunk);
|
|
31664
|
+
if (progress.bytesRead > byteLimit) {
|
|
31665
|
+
stream.destroy();
|
|
31666
|
+
throw new SeoError("INVALID_INPUT", `Link import exceeds the ${byteLimit}-byte streaming limit.`);
|
|
31667
|
+
}
|
|
31668
|
+
remainder += chunk;
|
|
31669
|
+
let newline = remainder.indexOf("\n");
|
|
31670
|
+
while (newline >= 0) {
|
|
31671
|
+
const line = remainder.slice(0, newline).trim();
|
|
31672
|
+
remainder = remainder.slice(newline + 1);
|
|
31673
|
+
if (line) yield parseObject(line);
|
|
31674
|
+
newline = remainder.indexOf("\n");
|
|
31675
|
+
}
|
|
31676
|
+
if (Buffer.byteLength(remainder) > MAX_CELL_BYTES) {
|
|
31677
|
+
stream.destroy();
|
|
31678
|
+
throw new SeoError("INVALID_INPUT", `Link import contains a line larger than ${MAX_CELL_BYTES} bytes.`);
|
|
31679
|
+
}
|
|
31680
|
+
}
|
|
31681
|
+
if (remainder.trim()) yield parseObject(remainder);
|
|
31682
|
+
}
|
|
31683
|
+
function parseObject(value) {
|
|
29866
31684
|
try {
|
|
29867
|
-
const
|
|
29868
|
-
if (
|
|
29869
|
-
return
|
|
31685
|
+
const parsed = JSON.parse(value);
|
|
31686
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("not an object");
|
|
31687
|
+
return parsed;
|
|
29870
31688
|
} catch {
|
|
29871
|
-
|
|
31689
|
+
throw new SeoError("INVALID_INPUT", "Link JSONL must contain one JSON object per line.");
|
|
31690
|
+
}
|
|
31691
|
+
}
|
|
31692
|
+
function rowFromCsv(headers, values) {
|
|
31693
|
+
return Object.fromEntries(headers.map((header, index) => [header, values[index]]));
|
|
31694
|
+
}
|
|
31695
|
+
async function importLinkEvidence(input) {
|
|
31696
|
+
const path = resolve(input.file);
|
|
31697
|
+
const file = await stat(path).catch(() => void 0);
|
|
31698
|
+
if (!file?.isFile()) throw new SeoError("INVALID_INPUT", `Link import file was not found: ${path}`);
|
|
31699
|
+
const format = formatFor$1(path, input.format);
|
|
31700
|
+
const rowLimit = boundedRowLimit(input.rowLimit);
|
|
31701
|
+
if (format === "json" && file.size > MAX_JSON_BYTES) throw new SeoError("INVALID_INPUT", `JSON link imports are limited to ${MAX_JSON_BYTES} bytes. Use CSV or JSONL for larger files.`);
|
|
31702
|
+
const progress = { bytesRead: 0 };
|
|
31703
|
+
let records;
|
|
31704
|
+
if (format === "csv") {
|
|
31705
|
+
const csv = csvRecords(path, MAX_STREAM_BYTES, progress);
|
|
31706
|
+
const first = await csv.next();
|
|
31707
|
+
const headers = (Array.isArray(first.value) ? first.value : []).map((value, index) => index === 0 ? value.replace(/^\uFEFF/, "") : value);
|
|
31708
|
+
if (!headers.length) throw new SeoError("INVALID_INPUT", "Link CSV has no header row.");
|
|
31709
|
+
records = (async function* () {
|
|
31710
|
+
for await (const values of csv) yield rowFromCsv(headers, values);
|
|
31711
|
+
})();
|
|
31712
|
+
} else if (format === "jsonl") records = jsonlRecords(path, MAX_STREAM_BYTES, progress);
|
|
31713
|
+
else {
|
|
31714
|
+
const body = await readFile(path, "utf8");
|
|
31715
|
+
progress.bytesRead = Buffer.byteLength(body);
|
|
31716
|
+
let parsed;
|
|
31717
|
+
try {
|
|
31718
|
+
parsed = JSON.parse(body);
|
|
31719
|
+
} catch {
|
|
31720
|
+
throw new SeoError("INVALID_INPUT", "Link JSON is not valid JSON.");
|
|
31721
|
+
}
|
|
31722
|
+
if (!Array.isArray(parsed)) throw new SeoError("INVALID_INPUT", "Link JSON must be an array of objects.");
|
|
31723
|
+
records = (async function* () {
|
|
31724
|
+
for (const row of parsed) yield row && typeof row === "object" && !Array.isArray(row) ? row : {};
|
|
31725
|
+
})();
|
|
29872
31726
|
}
|
|
31727
|
+
const rows = [];
|
|
31728
|
+
const keys = /* @__PURE__ */ new Set();
|
|
31729
|
+
let suppliedRows = 0;
|
|
31730
|
+
let invalidRows = 0;
|
|
31731
|
+
let duplicateRows = 0;
|
|
31732
|
+
let capped = false;
|
|
31733
|
+
for await (const raw of records) {
|
|
31734
|
+
if (suppliedRows >= rowLimit) {
|
|
31735
|
+
capped = true;
|
|
31736
|
+
break;
|
|
31737
|
+
}
|
|
31738
|
+
suppliedRows += 1;
|
|
31739
|
+
const row = normalizeLinkEvidenceRow(raw);
|
|
31740
|
+
if (!row) {
|
|
31741
|
+
invalidRows += 1;
|
|
31742
|
+
continue;
|
|
31743
|
+
}
|
|
31744
|
+
const key = linkEvidenceKey(row);
|
|
31745
|
+
if (keys.has(key)) {
|
|
31746
|
+
duplicateRows += 1;
|
|
31747
|
+
continue;
|
|
31748
|
+
}
|
|
31749
|
+
keys.add(key);
|
|
31750
|
+
rows.push(row);
|
|
31751
|
+
}
|
|
31752
|
+
rows.sort(compareLinkEvidence);
|
|
31753
|
+
const observedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
31754
|
+
return {
|
|
31755
|
+
rows,
|
|
31756
|
+
targetCounts: [],
|
|
31757
|
+
provenance: {
|
|
31758
|
+
provider: `${format}-import`,
|
|
31759
|
+
observedAt,
|
|
31760
|
+
cached: false,
|
|
31761
|
+
suppliedRows,
|
|
31762
|
+
validRows: rows.length,
|
|
31763
|
+
invalidRows,
|
|
31764
|
+
duplicateRows,
|
|
31765
|
+
capped,
|
|
31766
|
+
rowLimit,
|
|
31767
|
+
completeness: capped ? "partial" : "unknown",
|
|
31768
|
+
file: {
|
|
31769
|
+
path,
|
|
31770
|
+
format,
|
|
31771
|
+
bytesRead: progress.bytesRead,
|
|
31772
|
+
fileBytes: file.size
|
|
31773
|
+
}
|
|
31774
|
+
},
|
|
31775
|
+
warnings: capped ? [`Import stopped after ${rowLimit} rows. The file contains more data.`] : []
|
|
31776
|
+
};
|
|
29873
31777
|
}
|
|
29874
|
-
|
|
29875
|
-
|
|
29876
|
-
|
|
29877
|
-
|
|
29878
|
-
|
|
29879
|
-
|
|
29880
|
-
|
|
29881
|
-
|
|
29882
|
-
if (domain) return streamHostname === domain || streamHostname.endsWith(`.${domain}`) ? "domain" : void 0;
|
|
29883
|
-
return hostnameFromUrl(site) === streamHostname ? "hostname" : void 0;
|
|
31778
|
+
//#endregion
|
|
31779
|
+
//#region packages/core/src/links/report.ts
|
|
31780
|
+
const DEFAULT_OUTPUT_LIMIT = 100;
|
|
31781
|
+
const MAX_OUTPUT_LIMIT = 500;
|
|
31782
|
+
function boundedLimit(value) {
|
|
31783
|
+
const limit = value ?? DEFAULT_OUTPUT_LIMIT;
|
|
31784
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_OUTPUT_LIMIT) throw new SeoError("INVALID_INPUT", `Link report limit must be between 1 and ${MAX_OUTPUT_LIMIT}.`);
|
|
31785
|
+
return limit;
|
|
29884
31786
|
}
|
|
29885
|
-
function
|
|
29886
|
-
|
|
29887
|
-
|
|
29888
|
-
|
|
29889
|
-
|
|
29890
|
-
|
|
29891
|
-
|
|
29892
|
-
if (!match) return [];
|
|
29893
|
-
return [{
|
|
29894
|
-
...candidate,
|
|
29895
|
-
defaultUri,
|
|
29896
|
-
hostname,
|
|
29897
|
-
match
|
|
29898
|
-
}];
|
|
29899
|
-
});
|
|
31787
|
+
function targetCounts(rows) {
|
|
31788
|
+
const counts = /* @__PURE__ */ new Map();
|
|
31789
|
+
for (const row of rows) counts.set(row.targetUrl, (counts.get(row.targetUrl) ?? 0) + 1);
|
|
31790
|
+
return [...counts].map(([targetUrl, observedLinks]) => ({
|
|
31791
|
+
targetUrl,
|
|
31792
|
+
observedLinks
|
|
31793
|
+
})).sort((a, b) => b.observedLinks - a.observedLinks || a.targetUrl.localeCompare(b.targetUrl, "en"));
|
|
29900
31794
|
}
|
|
29901
|
-
function
|
|
29902
|
-
const
|
|
29903
|
-
const
|
|
29904
|
-
|
|
31795
|
+
function linkEvidenceReport(input) {
|
|
31796
|
+
const limit = boundedLimit(input.limit);
|
|
31797
|
+
const rows = input.evidence.rows.slice(0, limit);
|
|
31798
|
+
const domains = new Set(input.evidence.rows.map((row) => row.sourceDomain));
|
|
31799
|
+
const observedTargets = targetCounts(input.evidence.rows);
|
|
31800
|
+
const counts = input.evidence.targetCounts.length ? input.evidence.targetCounts : observedTargets;
|
|
31801
|
+
return {
|
|
31802
|
+
schemaVersion: 1,
|
|
31803
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
31804
|
+
dataStatus: input.evidence.provenance.completeness === "complete" ? "complete" : "partial",
|
|
31805
|
+
summary: {
|
|
31806
|
+
observedLinks: input.evidence.rows.length,
|
|
31807
|
+
referringDomains: domains.size,
|
|
31808
|
+
targetPages: new Set(input.evidence.rows.map((row) => row.targetUrl)).size,
|
|
31809
|
+
providerTargetPages: counts.length
|
|
31810
|
+
},
|
|
31811
|
+
provenance: input.evidence.provenance,
|
|
31812
|
+
selection: {
|
|
31813
|
+
availableRows: input.evidence.rows.length,
|
|
31814
|
+
returnedRows: rows.length,
|
|
31815
|
+
omittedRows: Math.max(0, input.evidence.rows.length - rows.length),
|
|
31816
|
+
limit
|
|
31817
|
+
},
|
|
31818
|
+
targetCounts: counts.slice(0, 100),
|
|
31819
|
+
links: rows,
|
|
31820
|
+
warnings: input.evidence.warnings,
|
|
31821
|
+
caveats: [
|
|
31822
|
+
"This is bounded provider or imported evidence, not a complete backlink index.",
|
|
31823
|
+
"A missing link is not proof that no link exists, and an observed link is not a quality or ranking verdict.",
|
|
31824
|
+
"Referring-domain counts describe the retained rows only."
|
|
31825
|
+
]
|
|
31826
|
+
};
|
|
29905
31827
|
}
|
|
29906
31828
|
//#endregion
|
|
29907
31829
|
//#region packages/core/src/storage/log-retention.ts
|
|
@@ -30278,6 +32200,472 @@ var JsonFileStorageAdapter = class {
|
|
|
30278
32200
|
};
|
|
30279
32201
|
const localCredentialsProvider = new LocalCredentialsProvider();
|
|
30280
32202
|
//#endregion
|
|
32203
|
+
//#region packages/core/src/server-logs/parse.ts
|
|
32204
|
+
const MONTHS = {
|
|
32205
|
+
Jan: 0,
|
|
32206
|
+
Feb: 1,
|
|
32207
|
+
Mar: 2,
|
|
32208
|
+
Apr: 3,
|
|
32209
|
+
May: 4,
|
|
32210
|
+
Jun: 5,
|
|
32211
|
+
Jul: 6,
|
|
32212
|
+
Aug: 7,
|
|
32213
|
+
Sep: 8,
|
|
32214
|
+
Oct: 9,
|
|
32215
|
+
Nov: 10,
|
|
32216
|
+
Dec: 11
|
|
32217
|
+
};
|
|
32218
|
+
const CRAWLERS = [
|
|
32219
|
+
{
|
|
32220
|
+
family: "Googlebot",
|
|
32221
|
+
category: "search",
|
|
32222
|
+
patterns: ["googlebot", "google-inspectiontool"]
|
|
32223
|
+
},
|
|
32224
|
+
{
|
|
32225
|
+
family: "Bingbot",
|
|
32226
|
+
category: "search",
|
|
32227
|
+
patterns: ["bingbot", "bingpreview"]
|
|
32228
|
+
},
|
|
32229
|
+
{
|
|
32230
|
+
family: "Applebot",
|
|
32231
|
+
category: "search",
|
|
32232
|
+
patterns: ["applebot"]
|
|
32233
|
+
},
|
|
32234
|
+
{
|
|
32235
|
+
family: "DuckDuckBot",
|
|
32236
|
+
category: "search",
|
|
32237
|
+
patterns: ["duckduckbot"]
|
|
32238
|
+
},
|
|
32239
|
+
{
|
|
32240
|
+
family: "YandexBot",
|
|
32241
|
+
category: "search",
|
|
32242
|
+
patterns: ["yandexbot"]
|
|
32243
|
+
},
|
|
32244
|
+
{
|
|
32245
|
+
family: "Baiduspider",
|
|
32246
|
+
category: "search",
|
|
32247
|
+
patterns: ["baiduspider"]
|
|
32248
|
+
},
|
|
32249
|
+
{
|
|
32250
|
+
family: "PetalBot",
|
|
32251
|
+
category: "search",
|
|
32252
|
+
patterns: ["petalbot"]
|
|
32253
|
+
},
|
|
32254
|
+
{
|
|
32255
|
+
family: "OpenAI",
|
|
32256
|
+
category: "ai",
|
|
32257
|
+
patterns: [
|
|
32258
|
+
"gptbot",
|
|
32259
|
+
"chatgpt-user",
|
|
32260
|
+
"oai-searchbot"
|
|
32261
|
+
]
|
|
32262
|
+
},
|
|
32263
|
+
{
|
|
32264
|
+
family: "Anthropic",
|
|
32265
|
+
category: "ai",
|
|
32266
|
+
patterns: [
|
|
32267
|
+
"claudebot",
|
|
32268
|
+
"claude-searchbot",
|
|
32269
|
+
"claude-user"
|
|
32270
|
+
]
|
|
32271
|
+
},
|
|
32272
|
+
{
|
|
32273
|
+
family: "Perplexity",
|
|
32274
|
+
category: "ai",
|
|
32275
|
+
patterns: ["perplexitybot", "perplexity-user"]
|
|
32276
|
+
},
|
|
32277
|
+
{
|
|
32278
|
+
family: "Amazonbot",
|
|
32279
|
+
category: "ai",
|
|
32280
|
+
patterns: ["amazonbot"]
|
|
32281
|
+
},
|
|
32282
|
+
{
|
|
32283
|
+
family: "Bytespider",
|
|
32284
|
+
category: "ai",
|
|
32285
|
+
patterns: ["bytespider"]
|
|
32286
|
+
},
|
|
32287
|
+
{
|
|
32288
|
+
family: "CCBot",
|
|
32289
|
+
category: "ai",
|
|
32290
|
+
patterns: ["ccbot"]
|
|
32291
|
+
},
|
|
32292
|
+
{
|
|
32293
|
+
family: "Cohere",
|
|
32294
|
+
category: "ai",
|
|
32295
|
+
patterns: ["cohere-ai"]
|
|
32296
|
+
}
|
|
32297
|
+
];
|
|
32298
|
+
const COMBINED_LOG = /^\S+\s+\S+\s+\S+\s+\[([^\]]+)]\s+"([^"\s]+)\s+([^"\s]+)(?:\s+[^"\s]+)?"\s+(\d{3})\s+(\d+|-)\s*(?:"[^"]*"\s+"([^"]*)")?\s*$/;
|
|
32299
|
+
function classifyCrawler(userAgent) {
|
|
32300
|
+
if (!userAgent) return void 0;
|
|
32301
|
+
const value = userAgent.toLowerCase();
|
|
32302
|
+
const match = CRAWLERS.find((crawler) => crawler.patterns.some((pattern) => value.includes(pattern)));
|
|
32303
|
+
return match ? {
|
|
32304
|
+
family: match.family,
|
|
32305
|
+
category: match.category
|
|
32306
|
+
} : void 0;
|
|
32307
|
+
}
|
|
32308
|
+
function logTimestamp(value) {
|
|
32309
|
+
const match = /^(\d{2})\/([A-Z][a-z]{2})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})$/.exec(value);
|
|
32310
|
+
if (!match) return void 0;
|
|
32311
|
+
const month = MONTHS[match[2] ?? ""];
|
|
32312
|
+
if (month === void 0) return void 0;
|
|
32313
|
+
const utc = Date.UTC(Number(match[3]), month, Number(match[1]), Number(match[4]), Number(match[5]), Number(match[6]));
|
|
32314
|
+
const offsetMinutes = Number(match[8]) * 60 + Number(match[9]);
|
|
32315
|
+
const adjusted = utc + (match[7] === "+" ? -1 : 1) * offsetMinutes * 6e4;
|
|
32316
|
+
const date = new Date(adjusted);
|
|
32317
|
+
return Number.isNaN(date.getTime()) ? void 0 : date.toISOString();
|
|
32318
|
+
}
|
|
32319
|
+
function normalizedPath(value) {
|
|
32320
|
+
if (typeof value !== "string" || value.length < 1 || value.length > 2e3) return;
|
|
32321
|
+
try {
|
|
32322
|
+
if (value.startsWith("/")) return new URL(value, "https://server-log.invalid").pathname;
|
|
32323
|
+
const url = new URL(value);
|
|
32324
|
+
return ["http:", "https:"].includes(url.protocol) ? url.pathname : void 0;
|
|
32325
|
+
} catch {
|
|
32326
|
+
return;
|
|
32327
|
+
}
|
|
32328
|
+
}
|
|
32329
|
+
function normalizedMethod(value) {
|
|
32330
|
+
if (typeof value !== "string" || !/^[A-Za-z]{1,16}$/.test(value)) return;
|
|
32331
|
+
return value.toUpperCase();
|
|
32332
|
+
}
|
|
32333
|
+
function normalizedStatus(value) {
|
|
32334
|
+
const status = typeof value === "number" ? value : Number(value);
|
|
32335
|
+
return Number.isInteger(status) && status >= 100 && status <= 599 ? status : void 0;
|
|
32336
|
+
}
|
|
32337
|
+
function normalizedBytes(value) {
|
|
32338
|
+
if (value === void 0 || value === null || value === "" || value === "-") return;
|
|
32339
|
+
const bytes = typeof value === "number" ? value : Number(value);
|
|
32340
|
+
return Number.isSafeInteger(bytes) && bytes >= 0 ? bytes : void 0;
|
|
32341
|
+
}
|
|
32342
|
+
function record(input) {
|
|
32343
|
+
let timestamp;
|
|
32344
|
+
if (typeof input.timestamp === "string") {
|
|
32345
|
+
const date = new Date(input.timestamp);
|
|
32346
|
+
if (!Number.isNaN(date.getTime())) timestamp = date.toISOString();
|
|
32347
|
+
}
|
|
32348
|
+
const method = normalizedMethod(input.method);
|
|
32349
|
+
const path = normalizedPath(input.path);
|
|
32350
|
+
const status = normalizedStatus(input.status);
|
|
32351
|
+
if (!timestamp || !method || !path || status === void 0) return void 0;
|
|
32352
|
+
const userAgent = typeof input.userAgent === "string" && input.userAgent.length <= 2e3 ? input.userAgent : void 0;
|
|
32353
|
+
return {
|
|
32354
|
+
timestamp,
|
|
32355
|
+
method,
|
|
32356
|
+
path,
|
|
32357
|
+
status,
|
|
32358
|
+
bytes: normalizedBytes(input.bytes),
|
|
32359
|
+
userAgent,
|
|
32360
|
+
crawler: classifyCrawler(userAgent)
|
|
32361
|
+
};
|
|
32362
|
+
}
|
|
32363
|
+
function parseCombinedLogLine(line) {
|
|
32364
|
+
const match = COMBINED_LOG.exec(line);
|
|
32365
|
+
if (!match) return void 0;
|
|
32366
|
+
const timestamp = logTimestamp(match[1] ?? "");
|
|
32367
|
+
if (!timestamp) return void 0;
|
|
32368
|
+
return record({
|
|
32369
|
+
timestamp,
|
|
32370
|
+
method: match[2],
|
|
32371
|
+
path: match[3],
|
|
32372
|
+
status: match[4],
|
|
32373
|
+
bytes: match[5],
|
|
32374
|
+
userAgent: match[6]
|
|
32375
|
+
});
|
|
32376
|
+
}
|
|
32377
|
+
function firstValue(value, names) {
|
|
32378
|
+
for (const name of names) if (value[name] !== void 0) return value[name];
|
|
32379
|
+
}
|
|
32380
|
+
function parseJsonLogLine(line) {
|
|
32381
|
+
let value;
|
|
32382
|
+
try {
|
|
32383
|
+
value = JSON.parse(line);
|
|
32384
|
+
} catch {
|
|
32385
|
+
return;
|
|
32386
|
+
}
|
|
32387
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return;
|
|
32388
|
+
const row = value;
|
|
32389
|
+
return record({
|
|
32390
|
+
timestamp: firstValue(row, [
|
|
32391
|
+
"timestamp",
|
|
32392
|
+
"time",
|
|
32393
|
+
"datetime",
|
|
32394
|
+
"@timestamp"
|
|
32395
|
+
]),
|
|
32396
|
+
method: firstValue(row, [
|
|
32397
|
+
"method",
|
|
32398
|
+
"requestMethod",
|
|
32399
|
+
"http_method"
|
|
32400
|
+
]),
|
|
32401
|
+
path: firstValue(row, [
|
|
32402
|
+
"path",
|
|
32403
|
+
"url",
|
|
32404
|
+
"requestUri",
|
|
32405
|
+
"request_uri"
|
|
32406
|
+
]),
|
|
32407
|
+
status: firstValue(row, [
|
|
32408
|
+
"status",
|
|
32409
|
+
"statusCode",
|
|
32410
|
+
"status_code"
|
|
32411
|
+
]),
|
|
32412
|
+
bytes: firstValue(row, [
|
|
32413
|
+
"bytes",
|
|
32414
|
+
"bodyBytesSent",
|
|
32415
|
+
"body_bytes_sent"
|
|
32416
|
+
]),
|
|
32417
|
+
userAgent: firstValue(row, [
|
|
32418
|
+
"userAgent",
|
|
32419
|
+
"user_agent",
|
|
32420
|
+
"httpUserAgent",
|
|
32421
|
+
"http_user_agent"
|
|
32422
|
+
])
|
|
32423
|
+
});
|
|
32424
|
+
}
|
|
32425
|
+
function emptyStatusBreakdown() {
|
|
32426
|
+
return {
|
|
32427
|
+
success: 0,
|
|
32428
|
+
redirect: 0,
|
|
32429
|
+
clientError: 0,
|
|
32430
|
+
serverError: 0,
|
|
32431
|
+
other: 0
|
|
32432
|
+
};
|
|
32433
|
+
}
|
|
32434
|
+
function addStatus(breakdown, status) {
|
|
32435
|
+
if (status >= 200 && status < 300) breakdown.success += 1;
|
|
32436
|
+
else if (status >= 300 && status < 400) breakdown.redirect += 1;
|
|
32437
|
+
else if (status >= 400 && status < 500) breakdown.clientError += 1;
|
|
32438
|
+
else if (status >= 500) breakdown.serverError += 1;
|
|
32439
|
+
else breakdown.other += 1;
|
|
32440
|
+
}
|
|
32441
|
+
//#endregion
|
|
32442
|
+
//#region packages/core/src/server-logs/import.ts
|
|
32443
|
+
const DEFAULT_ROW_LIMIT = 1e6;
|
|
32444
|
+
const MAX_ROW_LIMIT = 1e7;
|
|
32445
|
+
const DEFAULT_PATH_LIMIT = 25e3;
|
|
32446
|
+
const MAX_PATH_LIMIT = 1e5;
|
|
32447
|
+
const BYTE_LIMIT = 1e9;
|
|
32448
|
+
const MAX_LINE_BYTES = 65536;
|
|
32449
|
+
function boundedInteger(value, fallback, maximum, label) {
|
|
32450
|
+
const result = value ?? fallback;
|
|
32451
|
+
if (!Number.isInteger(result) || result < 1 || result > maximum) throw new SeoError("INVALID_INPUT", `${label} must be between 1 and ${maximum}.`);
|
|
32452
|
+
return result;
|
|
32453
|
+
}
|
|
32454
|
+
function formatFor(path, explicit) {
|
|
32455
|
+
if (explicit) return explicit;
|
|
32456
|
+
const extension = extname(path).toLowerCase();
|
|
32457
|
+
return [".jsonl", ".ndjson"].includes(extension) ? "jsonl" : "combined";
|
|
32458
|
+
}
|
|
32459
|
+
async function* lines(path, progress) {
|
|
32460
|
+
const stream = createReadStream(path, {
|
|
32461
|
+
encoding: "utf8",
|
|
32462
|
+
highWaterMark: 64 * 1024
|
|
32463
|
+
});
|
|
32464
|
+
let remainder = "";
|
|
32465
|
+
let discarding = false;
|
|
32466
|
+
for await (const chunk of stream) {
|
|
32467
|
+
const remainingBytes = BYTE_LIMIT - progress.bytesRead;
|
|
32468
|
+
if (remainingBytes <= 0) {
|
|
32469
|
+
progress.bytesCapped = true;
|
|
32470
|
+
break;
|
|
32471
|
+
}
|
|
32472
|
+
const chunkBytes = Buffer.byteLength(chunk);
|
|
32473
|
+
const accepted = chunkBytes > remainingBytes ? Buffer.from(chunk).subarray(0, remainingBytes).toString("utf8") : chunk;
|
|
32474
|
+
progress.bytesRead += Buffer.byteLength(accepted);
|
|
32475
|
+
if (chunkBytes > remainingBytes) progress.bytesCapped = true;
|
|
32476
|
+
remainder += accepted;
|
|
32477
|
+
let newline = remainder.indexOf("\n");
|
|
32478
|
+
while (newline >= 0) {
|
|
32479
|
+
const line = remainder.slice(0, newline).replace(/\r$/, "");
|
|
32480
|
+
remainder = remainder.slice(newline + 1);
|
|
32481
|
+
if (discarding || Buffer.byteLength(line) > MAX_LINE_BYTES) {
|
|
32482
|
+
yield { tooLong: true };
|
|
32483
|
+
discarding = false;
|
|
32484
|
+
} else if (line.trim()) yield { line };
|
|
32485
|
+
newline = remainder.indexOf("\n");
|
|
32486
|
+
}
|
|
32487
|
+
if (Buffer.byteLength(remainder) > MAX_LINE_BYTES) {
|
|
32488
|
+
remainder = "";
|
|
32489
|
+
discarding = true;
|
|
32490
|
+
}
|
|
32491
|
+
if (progress.bytesCapped) break;
|
|
32492
|
+
}
|
|
32493
|
+
if (discarding || Buffer.byteLength(remainder) > MAX_LINE_BYTES) yield { tooLong: true };
|
|
32494
|
+
else if (remainder.trim()) yield { line: remainder.replace(/\r$/, "") };
|
|
32495
|
+
}
|
|
32496
|
+
function compareCrawler(a, b) {
|
|
32497
|
+
return b.requests - a.requests || a.family.localeCompare(b.family, "en");
|
|
32498
|
+
}
|
|
32499
|
+
function compareCrawlerPath(a, b) {
|
|
32500
|
+
return b.requests - a.requests || a.family.localeCompare(b.family, "en") || a.path.localeCompare(b.path, "en");
|
|
32501
|
+
}
|
|
32502
|
+
function updatePeriod(timestamp, period) {
|
|
32503
|
+
if (!period.firstSeenAt || timestamp < period.firstSeenAt) period.firstSeenAt = timestamp;
|
|
32504
|
+
if (!period.lastSeenAt || timestamp > period.lastSeenAt) period.lastSeenAt = timestamp;
|
|
32505
|
+
}
|
|
32506
|
+
function newCrawler(record) {
|
|
32507
|
+
const crawler = record.crawler;
|
|
32508
|
+
if (!crawler) throw new Error("Crawler evidence is required.");
|
|
32509
|
+
return {
|
|
32510
|
+
family: crawler.family,
|
|
32511
|
+
category: crawler.category,
|
|
32512
|
+
requests: 0,
|
|
32513
|
+
lastSeenAt: record.timestamp,
|
|
32514
|
+
...emptyStatusBreakdown()
|
|
32515
|
+
};
|
|
32516
|
+
}
|
|
32517
|
+
function newCrawlerPath(record) {
|
|
32518
|
+
return {
|
|
32519
|
+
...newCrawler(record),
|
|
32520
|
+
path: record.path
|
|
32521
|
+
};
|
|
32522
|
+
}
|
|
32523
|
+
async function importServerLog(input) {
|
|
32524
|
+
const path = resolve(input.file);
|
|
32525
|
+
const file = await stat(path).catch(() => void 0);
|
|
32526
|
+
if (!file?.isFile()) throw new SeoError("INVALID_INPUT", `Server log was not found: ${path}`);
|
|
32527
|
+
const format = formatFor(path, input.format);
|
|
32528
|
+
const rowLimit = boundedInteger(input.rowLimit, DEFAULT_ROW_LIMIT, MAX_ROW_LIMIT, "Server log row limit");
|
|
32529
|
+
const pathLimit = boundedInteger(input.pathLimit, DEFAULT_PATH_LIMIT, MAX_PATH_LIMIT, "Server log path limit");
|
|
32530
|
+
const parse = format === "jsonl" ? parseJsonLogLine : parseCombinedLogLine;
|
|
32531
|
+
const progress = {
|
|
32532
|
+
bytesRead: 0,
|
|
32533
|
+
bytesCapped: false
|
|
32534
|
+
};
|
|
32535
|
+
const period = {};
|
|
32536
|
+
const statusCodes = /* @__PURE__ */ new Map();
|
|
32537
|
+
const crawlers = /* @__PURE__ */ new Map();
|
|
32538
|
+
const crawlerPaths = /* @__PURE__ */ new Map();
|
|
32539
|
+
let suppliedRows = 0;
|
|
32540
|
+
let parsedRows = 0;
|
|
32541
|
+
let invalidRows = 0;
|
|
32542
|
+
let crawlerRows = 0;
|
|
32543
|
+
let responseBytes = 0;
|
|
32544
|
+
let rowsCapped = false;
|
|
32545
|
+
let pathsCapped = false;
|
|
32546
|
+
let untrackedCrawlerPathRows = 0;
|
|
32547
|
+
for await (const item of lines(path, progress)) {
|
|
32548
|
+
if (suppliedRows >= rowLimit) {
|
|
32549
|
+
rowsCapped = true;
|
|
32550
|
+
break;
|
|
32551
|
+
}
|
|
32552
|
+
suppliedRows += 1;
|
|
32553
|
+
const parsed = item.line ? parse(item.line) : void 0;
|
|
32554
|
+
if (!parsed) {
|
|
32555
|
+
invalidRows += 1;
|
|
32556
|
+
continue;
|
|
32557
|
+
}
|
|
32558
|
+
parsedRows += 1;
|
|
32559
|
+
responseBytes += parsed.bytes ?? 0;
|
|
32560
|
+
updatePeriod(parsed.timestamp, period);
|
|
32561
|
+
statusCodes.set(parsed.status, (statusCodes.get(parsed.status) ?? 0) + 1);
|
|
32562
|
+
if (!parsed.crawler) continue;
|
|
32563
|
+
crawlerRows += 1;
|
|
32564
|
+
const crawler = crawlers.get(parsed.crawler.family) ?? newCrawler(parsed);
|
|
32565
|
+
crawler.requests += 1;
|
|
32566
|
+
if (parsed.timestamp > crawler.lastSeenAt) crawler.lastSeenAt = parsed.timestamp;
|
|
32567
|
+
addStatus(crawler, parsed.status);
|
|
32568
|
+
crawlers.set(parsed.crawler.family, crawler);
|
|
32569
|
+
const key = `${parsed.crawler.family}\u0000${parsed.path}`;
|
|
32570
|
+
let crawlerPath = crawlerPaths.get(key);
|
|
32571
|
+
if (!crawlerPath && crawlerPaths.size >= pathLimit) {
|
|
32572
|
+
pathsCapped = true;
|
|
32573
|
+
untrackedCrawlerPathRows += 1;
|
|
32574
|
+
continue;
|
|
32575
|
+
}
|
|
32576
|
+
crawlerPath ??= newCrawlerPath(parsed);
|
|
32577
|
+
crawlerPath.requests += 1;
|
|
32578
|
+
if (parsed.timestamp > crawlerPath.lastSeenAt) crawlerPath.lastSeenAt = parsed.timestamp;
|
|
32579
|
+
addStatus(crawlerPath, parsed.status);
|
|
32580
|
+
crawlerPaths.set(key, crawlerPath);
|
|
32581
|
+
}
|
|
32582
|
+
const fileReadCompletely = !rowsCapped && !progress.bytesCapped && progress.bytesRead >= file.size;
|
|
32583
|
+
const completeness = fileReadCompletely && invalidRows === 0 && !pathsCapped ? "complete" : "partial";
|
|
32584
|
+
const warnings = [];
|
|
32585
|
+
if (rowsCapped) warnings.push(`Analysis stopped after ${rowLimit} rows.`);
|
|
32586
|
+
if (progress.bytesCapped) warnings.push(`Analysis stopped after ${BYTE_LIMIT} input bytes.`);
|
|
32587
|
+
if (invalidRows) warnings.push(`${invalidRows} malformed or unsupported rows were skipped.`);
|
|
32588
|
+
if (pathsCapped) warnings.push(`Crawler path tracking stopped at ${pathLimit} unique crawler and path pairs. Overall crawler and status totals still include later valid rows.`);
|
|
32589
|
+
return {
|
|
32590
|
+
summary: {
|
|
32591
|
+
suppliedRows,
|
|
32592
|
+
parsedRows,
|
|
32593
|
+
invalidRows,
|
|
32594
|
+
crawlerRows,
|
|
32595
|
+
nonCrawlerRows: parsedRows - crawlerRows,
|
|
32596
|
+
responseBytes,
|
|
32597
|
+
...period
|
|
32598
|
+
},
|
|
32599
|
+
statusCodes: [...statusCodes].map(([status, requests]) => ({
|
|
32600
|
+
status,
|
|
32601
|
+
requests
|
|
32602
|
+
})).sort((a, b) => a.status - b.status),
|
|
32603
|
+
crawlers: [...crawlers.values()].sort(compareCrawler),
|
|
32604
|
+
crawlerPaths: [...crawlerPaths.values()].sort(compareCrawlerPath),
|
|
32605
|
+
provenance: {
|
|
32606
|
+
source: "local-server-log",
|
|
32607
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
32608
|
+
cached: false,
|
|
32609
|
+
file: {
|
|
32610
|
+
path,
|
|
32611
|
+
format,
|
|
32612
|
+
bytesRead: progress.bytesRead,
|
|
32613
|
+
fileBytes: file.size
|
|
32614
|
+
},
|
|
32615
|
+
limits: {
|
|
32616
|
+
rowLimit,
|
|
32617
|
+
pathLimit,
|
|
32618
|
+
byteLimit: BYTE_LIMIT,
|
|
32619
|
+
maxLineBytes: MAX_LINE_BYTES
|
|
32620
|
+
},
|
|
32621
|
+
coverage: {
|
|
32622
|
+
fileReadCompletely,
|
|
32623
|
+
rowsCapped,
|
|
32624
|
+
bytesCapped: progress.bytesCapped,
|
|
32625
|
+
pathsCapped,
|
|
32626
|
+
untrackedCrawlerPathRows
|
|
32627
|
+
},
|
|
32628
|
+
completeness
|
|
32629
|
+
},
|
|
32630
|
+
warnings
|
|
32631
|
+
};
|
|
32632
|
+
}
|
|
32633
|
+
//#endregion
|
|
32634
|
+
//#region packages/core/src/server-logs/report.ts
|
|
32635
|
+
const DEFAULT_LIMIT = 100;
|
|
32636
|
+
const MAX_LIMIT = 500;
|
|
32637
|
+
function outputLimit(value) {
|
|
32638
|
+
const limit = value ?? DEFAULT_LIMIT;
|
|
32639
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) throw new SeoError("INVALID_INPUT", `Server log output limit must be between 1 and ${MAX_LIMIT}.`);
|
|
32640
|
+
return limit;
|
|
32641
|
+
}
|
|
32642
|
+
function serverLogReport(input) {
|
|
32643
|
+
const limit = outputLimit(input.limit);
|
|
32644
|
+
const crawlerPaths = input.evidence.crawlerPaths.slice(0, limit);
|
|
32645
|
+
return {
|
|
32646
|
+
schemaVersion: 1,
|
|
32647
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
32648
|
+
dataStatus: input.evidence.provenance.completeness,
|
|
32649
|
+
summary: input.evidence.summary,
|
|
32650
|
+
statusCodes: input.evidence.statusCodes,
|
|
32651
|
+
crawlers: input.evidence.crawlers,
|
|
32652
|
+
crawlerPaths,
|
|
32653
|
+
selection: {
|
|
32654
|
+
availableCrawlerPaths: input.evidence.crawlerPaths.length,
|
|
32655
|
+
returnedCrawlerPaths: crawlerPaths.length,
|
|
32656
|
+
omittedCrawlerPaths: Math.max(0, input.evidence.crawlerPaths.length - crawlerPaths.length),
|
|
32657
|
+
limit
|
|
32658
|
+
},
|
|
32659
|
+
provenance: input.evidence.provenance,
|
|
32660
|
+
warnings: input.evidence.warnings,
|
|
32661
|
+
caveats: [
|
|
32662
|
+
"Crawler families are identified from user-agent strings, which can be missing or spoofed.",
|
|
32663
|
+
"This report describes only the supplied log file and cannot prove that every server or time period is represented.",
|
|
32664
|
+
"Crawler path rankings describe retained path aggregates when the path limit is reached."
|
|
32665
|
+
]
|
|
32666
|
+
};
|
|
32667
|
+
}
|
|
32668
|
+
//#endregion
|
|
30281
32669
|
//#region packages/core/src/telemetry.ts
|
|
30282
32670
|
const TELEMETRY_ENDPOINT = "https://seoskill.dev/api/t";
|
|
30283
32671
|
const TELEMETRY_SCHEMA_VERSION = 1;
|
|
@@ -30315,6 +32703,7 @@ const TELEMETRY_REPORTS = [
|
|
|
30315
32703
|
"ai-search-scorecard",
|
|
30316
32704
|
"audit-page",
|
|
30317
32705
|
"audit-urls",
|
|
32706
|
+
"bing-webmaster-overview",
|
|
30318
32707
|
"cannibalisation",
|
|
30319
32708
|
"community-intent",
|
|
30320
32709
|
"compare-crawls",
|
|
@@ -30333,6 +32722,8 @@ const TELEMETRY_REPORTS = [
|
|
|
30333
32722
|
"index-monitor",
|
|
30334
32723
|
"index-watch",
|
|
30335
32724
|
"internal-links",
|
|
32725
|
+
"link-evidence",
|
|
32726
|
+
"server-log-analysis",
|
|
30336
32727
|
"link-recovery",
|
|
30337
32728
|
"crawl-history",
|
|
30338
32729
|
"crawler-rules",
|
|
@@ -30674,6 +33065,6 @@ function isoDate(daysAgo = 0) {
|
|
|
30674
33065
|
return date.toISOString().slice(0, 10);
|
|
30675
33066
|
}
|
|
30676
33067
|
//#endregion
|
|
30677
|
-
export { AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, AI_SEARCH_SCORECARD_METHODOLOGY_ID, AI_SEARCH_SCORECARD_METHODOLOGY_VERSION, AUDIT_PAGE_RULE_IDS, AUTOMATIC_BASELINE_SITE_RETENTION, AUTOMATIC_BASELINE_TOTAL_RETENTION, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CRAWL_REPORT_STORAGE_VERSION, CRAWL_RUN_RETENTION, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_DEFAULT_MIN_IMPRESSIONS, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DataForSeoProvider, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, INDEX_WATCH_ATTEMPT_RETENTION, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LINK_RECOVER_RUN_RETENTION, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, OKF_DEFAULT_CONCEPTS, OKF_MAX_CONCEPTS, REPORT_BASELINE_MAX_AGE_MS, REPORT_BASELINE_MAX_DEPTH, REPORT_BASELINE_MAX_PAGES, ResponseSizeLimitError, SCORECARD_CHECK_WEIGHTS, SCORECARD_STATUS_CREDIT, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SemrushProvider, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_NOTICE, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, affectedUrls, agentReadiness, aiPromptsForQuery, aiReadiness, aiReferralSourceForValue, aiReferralsReport, aiSearchScorecard, allocateIndexUrlsToProperties, analyticsConnectionsSchema, analyzeAiReferralRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, attachCrawlRecommendations, auditCrawlPages, auditCrawlRequests, auditLlmsTxt, auditPage, authStatus, boundedSitemapInventory, buildOkfBundle, buildPseoAuditReportFromRows, buildTelemetryPayload, cannibalReport, changeMeasurementCsvRows, checkDatabaseReadiness, classifyCommunityIntent, clearCache, clientProfileSchema, clusterPseoTemplates, clusterQueryRows, collectAgentDiscovery, combineRobotsValues, communityIntentReport, compareCrawlPages, compareCrawlReports, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlDiff, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportStore, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createContentGroup, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, decayClusterDrafts, decayComparisonRange, decayingReport, deleteChange, deleteClient, deleteContentGroup, deleteCrawlReport, deleteTokens, deriveBrandTerms, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensureSeoCliDirs, entityReadiness, estimateSerpTitleWidth, expectedCtrForPosition, explainOkfValidation, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fetchText, fileSize, finalGscDateRange, findOverlappingSearchUpdates, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, generateLlmsTxt, geoGaps, geoGapsReport, getAuthModeStatus, getCacheStats, getChange, getClient, getClientConfig, getContentGroup, getDb, getKeywordProvider, getPreviousRun, getPrivacySnapshot, getRepeatedLinkRecoverUrls, getRunPages, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleAnalyticsPropertyId, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoveragePlan, indexCoverageSignals, indexMonitor, indexWatch, inferPseoTemplate, inferTemplateMovement, initializeTelemetry, insertCrawlRun, insertLinkRecoverRun, inspectUrl, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isCompatibleTechnicalBaseline, isCurrentTechnicalBaseline, isEntityEvidenceSchemaType, isSiteIdentitySchemaType, isSkippableReportError, isTelemetryCi, isTelemetryReport, isoDate, isoWeek, latestCrawlReport, latestCrawlSummaries, latestFinalGscDate, latestIndexWatchSummary, latestLinkRecoverSummary, linkEntries, linkRecover, linkRecoverRecommendation, listChanges, listClients, listContentGroups, listCrawlReports, listGa4AccountSummaries, listGa4DataStreams, listRules, listSearchUpdates, listSites, loadCrawlReport, localCredentialsProvider, loginWithLoopback, makeRunId, matchGa4WebStreams, measureChange, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, monitoringStatus, monthlyReport, monthlyReportWorkflow, narrativeCsvFiles, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, noteCacheWrite, okfConceptLimit, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, parsePseoPath, performanceAudit, performanceReportIsCacheable, planDueIndexUrls, planIndexCoverageFromUrls, plural, propertyMatchesUrl, providerPreferenceSchema, pruneIndexWatchSnapshots, pruneLogs, pseoAuditOptions, pseoAuditReport, pseoCsvFiles, pseoIndexStatus, pseoPresentation, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, readBoundedResponseText, readConfig, readOauthClient, readTokens, recommendCrawlDiffItem, recommendationForRule, recordChange, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderPseoMarkdown, renderWorkflowMarkdown, reportNarrative, reportPresentation, requestHeaders, resolveSegmentRanges, resolveSeoCliPaths, resolveTechnicalBaseline, reviewObservations, roundedPosition, runDoctor, runGa4Report, safeError, saveClient, saveCrawlReport, secondPage, segmentComparisonRange, segmentImpact, segmentRangeDays, selectAffectedUrls, selectDueIndexUrls, seoErrorEnvelope, seoToAiQueryReport, sessionSourceInListFilter, setDefaultClient, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteSchema, sqliteCrawlReportStore, strikingDistance, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, technicalWatchWorkflow, telemetryErrorCategory, templateForUrl, toSeoError, tokenSchema, topCrawlRecommendations, topFixes, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, unavailableSegmentImpactReport, unicodeTokens, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateOkfFiles, verifyQueryContent, workflowPresentation, writeConfig, writeOauthClient, writeTokens, xRobotsDirectives };
|
|
33068
|
+
export { AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, AI_SEARCH_SCORECARD_METHODOLOGY_ID, AI_SEARCH_SCORECARD_METHODOLOGY_VERSION, AUDIT_PAGE_RULE_IDS, AUTOMATIC_BASELINE_SITE_RETENTION, AUTOMATIC_BASELINE_TOTAL_RETENTION, BING_API_KEY_ENV, BING_API_KEY_SECRET, BROWSER_USER_AGENT, BingWebmasterClient, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CRAWL_REPORT_STORAGE_VERSION, CRAWL_RUN_RETENTION, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_DEFAULT_MIN_IMPRESSIONS, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DataForSeoProvider, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, INDEXNOW_ENDPOINT, INDEXNOW_KEYS_SECRET, INDEXNOW_KEY_ENV, INDEXNOW_MAX_URLS, INDEX_WATCH_ATTEMPT_RETENTION, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LINK_RECOVER_RUN_RETENTION, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, OKF_DEFAULT_CONCEPTS, OKF_MAX_CONCEPTS, REPORT_BASELINE_MAX_AGE_MS, REPORT_BASELINE_MAX_DEPTH, REPORT_BASELINE_MAX_PAGES, ResponseSizeLimitError, SCORECARD_CHECK_WEIGHTS, SCORECARD_STATUS_CREDIT, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SemrushProvider, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_NOTICE, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, addStatus, affectedUrls, agentReadiness, aiPromptsForQuery, aiReadiness, aiReferralSourceForValue, aiReferralsReport, aiSearchScorecard, allocateIndexUrlsToProperties, analyticsConnectionsSchema, analyzeAiReferralRows, analyzeBingCrawl, analyzeBingDimensions, analyzeBingTraffic, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, attachCrawlRecommendations, auditCrawlPages, auditCrawlRequests, auditLlmsTxt, auditPage, authStatus, bingAnalysisDefaults, bingWebmasterOverview, bingWebmasterSiteUrl, boundedSitemapInventory, buildOkfBundle, buildPseoAuditReportFromRows, buildTelemetryPayload, cannibalReport, changeMeasurementCsvRows, checkDatabaseReadiness, classifyCommunityIntent, classifyCrawler, clearCache, clientProfileSchema, clusterPseoTemplates, clusterQueryRows, collectAgentDiscovery, collectBingLinkEvidence, combineRobotsValues, communityIntentReport, compareCrawlPages, compareCrawlReports, compareLinkEvidence, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlDiff, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportStore, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createBingWebmasterClient, createContentGroup, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createIndexNowKey, createIndexNowKeyRecord, createLogger, createPageRenderer, ctrUnderperformersReport, decayClusterDrafts, decayComparisonRange, decayingReport, deleteChange, deleteClient, deleteContentGroup, deleteCrawlReport, deleteProviderSecret, deleteTokens, deriveBrandTerms, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, emptyStatusBreakdown, ensureSeoCliDirs, entityReadiness, estimateSerpTitleWidth, expectedCtrForPosition, explainOkfValidation, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fetchText, fileSize, finalGscDateRange, findOverlappingSearchUpdates, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, generateLlmsTxt, geoGaps, geoGapsReport, getAuthModeStatus, getCacheStats, getChange, getClient, getClientConfig, getContentGroup, getDb, getKeywordProvider, getPreviousRun, getPrivacySnapshot, getRepeatedLinkRecoverUrls, getRunPages, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleAnalyticsPropertyId, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, importLinkEvidence, importServerLog, indexCoveragePlan, indexCoverageSignals, indexMonitor, indexWatch, inferPseoTemplate, inferTemplateMovement, initializeTelemetry, insertCrawlRun, insertLinkRecoverRun, inspectUrl, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isCompatibleTechnicalBaseline, isCurrentTechnicalBaseline, isEntityEvidenceSchemaType, isSiteIdentitySchemaType, isSkippableReportError, isTelemetryCi, isTelemetryReport, isoDate, isoWeek, latestCrawlReport, latestCrawlSummaries, latestFinalGscDate, latestIndexWatchSummary, latestLinkRecoverSummary, linkEntries, linkEvidenceKey, linkEvidenceReport, linkRecover, linkRecoverRecommendation, listChanges, listClients, listContentGroups, listCrawlReports, listGa4AccountSummaries, listGa4DataStreams, listIndexNowKeys, listRules, listSearchUpdates, listSites, loadCrawlReport, localCredentialsProvider, loginWithLoopback, makeRunId, matchGa4WebStreams, measureChange, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, monitoringStatus, monthlyReport, monthlyReportWorkflow, narrativeCsvFiles, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLinkEvidenceRow, normalizeLoadedCrawlReport, noteCacheWrite, okfConceptLimit, pageEvidenceRecommendation, pageOpportunitiesReport, parseCombinedLogLine, parseIndexNowSite, parseJsonLogLine, parseOlderThan, parsePseoPath, performanceAudit, performanceReportIsCacheable, planDueIndexUrls, planIndexCoverageFromUrls, plural, propertyMatchesUrl, providerPreferenceSchema, pruneIndexWatchSnapshots, pruneLogs, pseoAuditOptions, pseoAuditReport, pseoCsvFiles, pseoIndexStatus, pseoPresentation, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, readBoundedResponseText, readConfig, readOauthClient, readProviderSecret, readTokens, recommendCrawlDiffItem, recommendationForRule, recordChange, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, removeIndexNowKey, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderPseoMarkdown, renderReportHtml, renderWorkflowMarkdown, reportNarrative, reportPresentation, requestHeaders, resolveIndexNowKey, resolveSegmentRanges, resolveSeoCliPaths, resolveTechnicalBaseline, reviewObservations, roundedPosition, runDoctor, runGa4Report, safeError, saveClient, saveCrawlReport, saveIndexNowKey, secondPage, segmentComparisonRange, segmentImpact, segmentRangeDays, selectAffectedUrls, selectDueIndexUrls, seoErrorEnvelope, seoToAiQueryReport, serverLogReport, sessionSourceInListFilter, setClientBingSite, setDefaultClient, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteSchema, sqliteCrawlReportStore, strikingDistance, submitIndexNow, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, technicalWatchWorkflow, telemetryErrorCategory, templateForUrl, toSeoError, tokenSchema, topCrawlRecommendations, topFixes, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, unavailableSegmentImpactReport, unicodeTokens, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateIndexNowKey, validateIndexNowKeyRecord, validateOkfFiles, verifyIndexNowKey, verifyQueryContent, workflowPresentation, writeConfig, writeOauthClient, writeProviderSecret, writeTokens, xRobotsDirectives };
|
|
30678
33069
|
|
|
30679
33070
|
//# sourceMappingURL=index.js.map
|