seo 0.2.10 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -1
- package/dist/cli.js +212 -130
- package/dist/cli.js.map +1 -1
- package/dist/{dist-CBY8QOnv.js → dist-BB0yH7qK.js} +1 -1
- package/dist/{dist-w2XtKwjW.js → dist-Dwdogl4j.js} +85 -80
- package/dist/dist-Dwdogl4j.js.map +1 -0
- package/dist/index.d.ts +1984 -71
- package/dist/index.js +2900 -247
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +2647 -197
- package/dist/mcp.js.map +1 -1
- package/dist/{package-92W1Zc-o.js → package-B_swz2Xd.js} +2 -2
- package/dist/{package-92W1Zc-o.js.map → package-B_swz2Xd.js.map} +1 -1
- package/evals/bing-webmaster-overview.json +31 -0
- package/evals/link-evidence.json +29 -0
- package/evals/seo.json +16 -0
- package/evals/server-log-analysis.json +29 -0
- package/package.json +3 -3
- package/skills/seo/SKILL.md +56 -67
- package/dist/dist-w2XtKwjW.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { n as load, t as version } from "./package-
|
|
2
|
+
import { n as load, t as version } from "./package-B_swz2Xd.js";
|
|
3
3
|
import { Headers, fetch as fetch$1 } from "undici";
|
|
4
|
-
import { chmodSync, createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { chmodSync, createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
5
5
|
import { arch, homedir, platform, totalmem } from "node:os";
|
|
6
|
-
import { basename, dirname, join } from "node:path";
|
|
6
|
+
import { basename, dirname, extname, join, resolve } from "node:path";
|
|
7
7
|
import { z } from "zod";
|
|
8
8
|
import lockfile from "proper-lockfile";
|
|
9
|
-
import crypto, { createHash, randomUUID, sign } from "node:crypto";
|
|
9
|
+
import crypto, { createHash, randomBytes, randomUUID, sign } from "node:crypto";
|
|
10
10
|
import http from "node:http";
|
|
11
11
|
import open from "open";
|
|
12
12
|
import Libsql from "libsql";
|
|
@@ -21,9 +21,9 @@ import { gunzipSync } from "node:zlib";
|
|
|
21
21
|
import { execFile } from "node:child_process";
|
|
22
22
|
import { fileURLToPath } from "node:url";
|
|
23
23
|
import { promisify } from "node:util";
|
|
24
|
+
import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
24
25
|
import pino from "pino";
|
|
25
26
|
import pretty from "pino-pretty";
|
|
26
|
-
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
27
27
|
//#region packages/core/src/errors.ts
|
|
28
28
|
const errorMetadata = {
|
|
29
29
|
ACCESS_DENIED: {
|
|
@@ -83,12 +83,12 @@ var SeoError = class extends Error {
|
|
|
83
83
|
this.retryable = errorMetadata[code].retryable;
|
|
84
84
|
}
|
|
85
85
|
};
|
|
86
|
-
function errorMessage$
|
|
86
|
+
function errorMessage$3(error) {
|
|
87
87
|
return error instanceof Error ? error.message : String(error);
|
|
88
88
|
}
|
|
89
89
|
function toSeoError(error) {
|
|
90
90
|
if (error instanceof SeoError) return error;
|
|
91
|
-
const message = errorMessage$
|
|
91
|
+
const message = errorMessage$3(error);
|
|
92
92
|
if (/not logged in/i.test(message)) return new SeoError("AUTH_REQUIRED", message);
|
|
93
93
|
if (/oauth client config.*missing|shared seo google app/i.test(message)) return new SeoError("AUTH_CONFIG_REQUIRED", message);
|
|
94
94
|
if (/invalid_grant|refresh token.*no longer valid/i.test(message)) return new SeoError("AUTH_EXPIRED", message);
|
|
@@ -145,6 +145,7 @@ function resolveSeoCliPaths(input = {}) {
|
|
|
145
145
|
configFile: join(configDir, "config.json"),
|
|
146
146
|
tokensFile: join(configDir, "tokens.json"),
|
|
147
147
|
oauthClientFile: join(configDir, "oauth-client.json"),
|
|
148
|
+
providerSecretsFile: join(configDir, "provider-secrets.json"),
|
|
148
149
|
telemetryStateFile: join(configDir, "telemetry.json"),
|
|
149
150
|
cacheDbFile: join(cacheDir, "cache.db")
|
|
150
151
|
};
|
|
@@ -231,6 +232,7 @@ const clientProfileSchema = z.object({
|
|
|
231
232
|
watchUrls: z.array(z.string()).default([]),
|
|
232
233
|
brandTerms: z.array(z.string()).default([]),
|
|
233
234
|
analytics: analyticsConnectionsSchema,
|
|
235
|
+
searchEngines: z.object({ bing: z.object({ siteUrl: z.string() }).optional() }).optional(),
|
|
234
236
|
reportDay: z.number().int().min(1).max(31).optional(),
|
|
235
237
|
technicalWeekday: z.number().int().min(0).max(7).optional(),
|
|
236
238
|
isDefault: z.boolean().optional(),
|
|
@@ -316,11 +318,11 @@ async function deleteKeyringPassword(service, account) {
|
|
|
316
318
|
}
|
|
317
319
|
//#endregion
|
|
318
320
|
//#region packages/core/src/storage/config.ts
|
|
319
|
-
const KEYRING_SERVICE = "seo";
|
|
320
|
-
const PRIVATE_FILE_MODE = 384;
|
|
321
|
+
const KEYRING_SERVICE$1 = "seo";
|
|
322
|
+
const PRIVATE_FILE_MODE$1 = 384;
|
|
321
323
|
function tightenConfigPermissions(path) {
|
|
322
324
|
try {
|
|
323
|
-
chmodSync(path, PRIVATE_FILE_MODE);
|
|
325
|
+
chmodSync(path, PRIVATE_FILE_MODE$1);
|
|
324
326
|
} catch (error) {
|
|
325
327
|
if (error.code !== "ENOENT") throw error;
|
|
326
328
|
}
|
|
@@ -334,7 +336,7 @@ function readConfig() {
|
|
|
334
336
|
}
|
|
335
337
|
function writeConfig(config) {
|
|
336
338
|
ensureSeoCliDirs();
|
|
337
|
-
writeJsonAtomic(getSeoCliPaths().configFile, configSchema.parse(config), PRIVATE_FILE_MODE);
|
|
339
|
+
writeJsonAtomic(getSeoCliPaths().configFile, configSchema.parse(config), PRIVATE_FILE_MODE$1);
|
|
338
340
|
}
|
|
339
341
|
function getKeyringAccount(tokens) {
|
|
340
342
|
return `google:${tokens.account_email}`;
|
|
@@ -354,7 +356,7 @@ function tokenMetadata(tokens) {
|
|
|
354
356
|
}
|
|
355
357
|
async function readKeyringTokens(tokens) {
|
|
356
358
|
const account = getKeyringAccount(tokens);
|
|
357
|
-
const [accessToken, refreshToken] = await Promise.all([getKeyringPassword(KEYRING_SERVICE, `${account}:access`), getKeyringPassword(KEYRING_SERVICE, `${account}:refresh`)]);
|
|
359
|
+
const [accessToken, refreshToken] = await Promise.all([getKeyringPassword(KEYRING_SERVICE$1, `${account}:access`), getKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`)]);
|
|
358
360
|
return {
|
|
359
361
|
accessToken: accessToken ?? void 0,
|
|
360
362
|
refreshToken: refreshToken ?? void 0
|
|
@@ -362,14 +364,14 @@ async function readKeyringTokens(tokens) {
|
|
|
362
364
|
}
|
|
363
365
|
async function writeKeyringTokens(tokens) {
|
|
364
366
|
const account = getKeyringAccount(tokens);
|
|
365
|
-
if (tokens.access_token) await setKeyringPassword(KEYRING_SERVICE, `${account}:access`, tokens.access_token);
|
|
366
|
-
else await deleteKeyringPassword(KEYRING_SERVICE, `${account}:access`);
|
|
367
|
-
if (tokens.refresh_token) await setKeyringPassword(KEYRING_SERVICE, `${account}:refresh`, tokens.refresh_token);
|
|
368
|
-
else await deleteKeyringPassword(KEYRING_SERVICE, `${account}:refresh`);
|
|
367
|
+
if (tokens.access_token) await setKeyringPassword(KEYRING_SERVICE$1, `${account}:access`, tokens.access_token);
|
|
368
|
+
else await deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:access`);
|
|
369
|
+
if (tokens.refresh_token) await setKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`, tokens.refresh_token);
|
|
370
|
+
else await deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`);
|
|
369
371
|
}
|
|
370
372
|
async function deleteKeyringTokens(tokens) {
|
|
371
373
|
const account = getKeyringAccount(tokens);
|
|
372
|
-
await Promise.all([deleteKeyringPassword(KEYRING_SERVICE, `${account}:access`), deleteKeyringPassword(KEYRING_SERVICE, `${account}:refresh`)]);
|
|
374
|
+
await Promise.all([deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:access`), deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`)]);
|
|
373
375
|
}
|
|
374
376
|
async function readTokens() {
|
|
375
377
|
ensureSeoCliDirs();
|
|
@@ -381,7 +383,7 @@ async function readTokens() {
|
|
|
381
383
|
const keyringTokens = await readKeyringTokens(parsed);
|
|
382
384
|
if (hasTokenSecrets(parsed)) {
|
|
383
385
|
await writeKeyringTokens(parsed);
|
|
384
|
-
writeJsonAtomic(getSeoCliPaths().tokensFile, tokenMetadata(parsed), PRIVATE_FILE_MODE);
|
|
386
|
+
writeJsonAtomic(getSeoCliPaths().tokensFile, tokenMetadata(parsed), PRIVATE_FILE_MODE$1);
|
|
385
387
|
return parsed;
|
|
386
388
|
}
|
|
387
389
|
return {
|
|
@@ -398,10 +400,10 @@ async function writeTokens(tokens) {
|
|
|
398
400
|
const parsed = tokenSchema.parse(tokens);
|
|
399
401
|
if (readConfig().security.useKeychain) try {
|
|
400
402
|
await writeKeyringTokens(parsed);
|
|
401
|
-
writeJsonAtomic(getSeoCliPaths().tokensFile, tokenMetadata(parsed), PRIVATE_FILE_MODE);
|
|
403
|
+
writeJsonAtomic(getSeoCliPaths().tokensFile, tokenMetadata(parsed), PRIVATE_FILE_MODE$1);
|
|
402
404
|
return;
|
|
403
405
|
} catch {}
|
|
404
|
-
writeJsonAtomic(getSeoCliPaths().tokensFile, parsed, PRIVATE_FILE_MODE);
|
|
406
|
+
writeJsonAtomic(getSeoCliPaths().tokensFile, parsed, PRIVATE_FILE_MODE$1);
|
|
405
407
|
}
|
|
406
408
|
async function deleteTokens() {
|
|
407
409
|
const tokens = await readTokens();
|
|
@@ -493,6 +495,12 @@ function getPrivacySnapshot() {
|
|
|
493
495
|
mode: fileMode(paths.oauthClientFile),
|
|
494
496
|
sizeBytes: sizeOf(paths.oauthClientFile)
|
|
495
497
|
},
|
|
498
|
+
{
|
|
499
|
+
label: "Provider secrets",
|
|
500
|
+
path: paths.providerSecretsFile,
|
|
501
|
+
mode: fileMode(paths.providerSecretsFile),
|
|
502
|
+
sizeBytes: sizeOf(paths.providerSecretsFile)
|
|
503
|
+
},
|
|
496
504
|
{
|
|
497
505
|
label: "Telemetry state",
|
|
498
506
|
path: paths.telemetryStateFile,
|
|
@@ -2524,6 +2532,43 @@ function extractCanonicalEvidence($, headers, baseUrl) {
|
|
|
2524
2532
|
return evidenceFromCandidates([...htmlCandidates($, baseUrl), ...headerCandidates(headers, baseUrl)]);
|
|
2525
2533
|
}
|
|
2526
2534
|
//#endregion
|
|
2535
|
+
//#region packages/core/src/extract/content-sketch.ts
|
|
2536
|
+
const SHINGLE_SIZE = 5;
|
|
2537
|
+
const MAX_HASHES = 32;
|
|
2538
|
+
function words(value) {
|
|
2539
|
+
return value.normalize("NFKC").toLocaleLowerCase("en").match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
2540
|
+
}
|
|
2541
|
+
function sampleIndexes(total, limit) {
|
|
2542
|
+
if (total <= limit) return Array.from({ length: total }, (_, index) => index);
|
|
2543
|
+
return Array.from({ length: limit }, (_, index) => Math.round(index * (total - 1) / (limit - 1)));
|
|
2544
|
+
}
|
|
2545
|
+
function digest(value) {
|
|
2546
|
+
return createHash("sha256").update(value).digest("hex").slice(0, 16);
|
|
2547
|
+
}
|
|
2548
|
+
function contentSketch(value) {
|
|
2549
|
+
const tokens = words(value);
|
|
2550
|
+
const hashes = sampleIndexes(Math.max(0, tokens.length - SHINGLE_SIZE + 1), MAX_HASHES).map((index) => digest(tokens.slice(index, index + SHINGLE_SIZE).join(" ")));
|
|
2551
|
+
return {
|
|
2552
|
+
version: 1,
|
|
2553
|
+
wordCount: tokens.length,
|
|
2554
|
+
shingleSize: SHINGLE_SIZE,
|
|
2555
|
+
sampledShingles: hashes.length,
|
|
2556
|
+
hashes
|
|
2557
|
+
};
|
|
2558
|
+
}
|
|
2559
|
+
function contentSketchCoverage(source, candidate) {
|
|
2560
|
+
if (!source || source.hashes.length < 4) return null;
|
|
2561
|
+
const targets = new Set(source.hashes);
|
|
2562
|
+
const matched = /* @__PURE__ */ new Set();
|
|
2563
|
+
const tokens = words(candidate);
|
|
2564
|
+
for (let index = 0; index <= tokens.length - source.shingleSize; index += 1) {
|
|
2565
|
+
const hash = digest(tokens.slice(index, index + source.shingleSize).join(" "));
|
|
2566
|
+
if (targets.has(hash)) matched.add(hash);
|
|
2567
|
+
if (matched.size === targets.size) break;
|
|
2568
|
+
}
|
|
2569
|
+
return Math.round(matched.size / targets.size * 1e3) / 1e3;
|
|
2570
|
+
}
|
|
2571
|
+
//#endregion
|
|
2527
2572
|
//#region packages/core/src/extract/main-content.ts
|
|
2528
2573
|
function readableError(error) {
|
|
2529
2574
|
return error instanceof Error ? error.message : String(error);
|
|
@@ -3141,6 +3186,48 @@ const SOCIAL_PROFILE_HOSTS = [
|
|
|
3141
3186
|
"x.com",
|
|
3142
3187
|
"youtube.com"
|
|
3143
3188
|
];
|
|
3189
|
+
const LOGIN_PATH_PATTERN = /(?:^|\/)(?:auth|login|log-in|signin|sign-in|sso)(?:\/|$)/iu;
|
|
3190
|
+
const LOGIN_TITLE_PATTERN = /\b(?:authenticate|log[ -]?in|sign[ -]?in)\b/iu;
|
|
3191
|
+
function softAuthenticationGate($, finalUrl, title) {
|
|
3192
|
+
const passwordForms = $("form").toArray().filter((form) => $(form).find("input[type=\"password\" i]").length > 0);
|
|
3193
|
+
if (passwordForms.length === 0) return void 0;
|
|
3194
|
+
const pagePath = new URL(finalUrl).pathname;
|
|
3195
|
+
const action = $(passwordForms[0]).attr("action");
|
|
3196
|
+
const actionUrl = action ? httpUrl$1(action, finalUrl) : void 0;
|
|
3197
|
+
const actionPath = actionUrl ? new URL(actionUrl).pathname : void 0;
|
|
3198
|
+
const indicators = [
|
|
3199
|
+
...LOGIN_PATH_PATTERN.test(pagePath) ? ["login-path"] : [],
|
|
3200
|
+
...LOGIN_TITLE_PATTERN.test(title ?? "") ? ["login-title"] : [],
|
|
3201
|
+
...LOGIN_PATH_PATTERN.test(actionPath ?? "") ? ["login-form-action"] : []
|
|
3202
|
+
];
|
|
3203
|
+
if (indicators.length === 0) return void 0;
|
|
3204
|
+
return {
|
|
3205
|
+
kind: "login-form",
|
|
3206
|
+
indicators,
|
|
3207
|
+
...actionPath ? { formActionPath: actionPath } : {}
|
|
3208
|
+
};
|
|
3209
|
+
}
|
|
3210
|
+
function tabbedContent($) {
|
|
3211
|
+
const groups = $("[role=\"tablist\" i]").length;
|
|
3212
|
+
const panels = $("[role=\"tabpanel\" i]").toArray();
|
|
3213
|
+
if (groups === 0 || panels.length === 0) return void 0;
|
|
3214
|
+
const panelSketches = panels.slice(0, 12).map((panel) => {
|
|
3215
|
+
const labelledBy = $(panel).attr("aria-labelledby");
|
|
3216
|
+
const labelledElement = labelledBy ? $("[id]").toArray().find((element) => $(element).attr("id") === labelledBy) : void 0;
|
|
3217
|
+
const label = safeText$1($(panel).attr("aria-label") ?? (labelledElement ? $(labelledElement).text() : void 0));
|
|
3218
|
+
return {
|
|
3219
|
+
...label ? { label: label.slice(0, 80) } : {},
|
|
3220
|
+
sketch: contentSketch($(panel).text())
|
|
3221
|
+
};
|
|
3222
|
+
});
|
|
3223
|
+
return {
|
|
3224
|
+
groups,
|
|
3225
|
+
panels: panels.length,
|
|
3226
|
+
retainedPanels: panelSketches.length,
|
|
3227
|
+
truncated: panels.length > panelSketches.length,
|
|
3228
|
+
panelSketches
|
|
3229
|
+
};
|
|
3230
|
+
}
|
|
3144
3231
|
function safeText$1(value) {
|
|
3145
3232
|
const trimmed = value?.replace(/\s+/g, " ").trim();
|
|
3146
3233
|
return trimmed ? trimmed : void 0;
|
|
@@ -3157,7 +3244,7 @@ function absoluteUrl$2(value, base) {
|
|
|
3157
3244
|
return;
|
|
3158
3245
|
}
|
|
3159
3246
|
}
|
|
3160
|
-
function httpUrl(value, base) {
|
|
3247
|
+
function httpUrl$1(value, base) {
|
|
3161
3248
|
const resolved = absoluteUrl$2(value, base);
|
|
3162
3249
|
if (!resolved) return void 0;
|
|
3163
3250
|
const protocol = new URL(resolved).protocol;
|
|
@@ -3309,14 +3396,14 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
3309
3396
|
let invalidHreflangCount = 0;
|
|
3310
3397
|
for (const element of $("link[rel~=\"alternate\"][hreflang][href]").toArray()) {
|
|
3311
3398
|
const language = $(element).attr("hreflang") ?? "";
|
|
3312
|
-
const href = httpUrl($(element).attr("href"), fetchResult.finalUrl);
|
|
3399
|
+
const href = httpUrl$1($(element).attr("href"), fetchResult.finalUrl);
|
|
3313
3400
|
if (language && href) hreflang.push({
|
|
3314
3401
|
hreflang: language,
|
|
3315
3402
|
href
|
|
3316
3403
|
});
|
|
3317
3404
|
else invalidHreflangCount += 1;
|
|
3318
3405
|
}
|
|
3319
|
-
const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
|
|
3406
|
+
const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl$1($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
|
|
3320
3407
|
const openGraph = Object.fromEntries($("meta[property^=\"og:\"]").toArray().map((element) => [$(element).attr("property") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
3321
3408
|
const twitter = Object.fromEntries($("meta[name^=\"twitter:\"]").toArray().map((element) => [$(element).attr("name") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
3322
3409
|
const structuredData = extractStructuredData($, fetchResult.finalUrl, { retainJsonLd: extractor !== "crawler" });
|
|
@@ -3367,10 +3454,11 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
3367
3454
|
html: sanitizedContentHtml($, fetchResult.html, fetchResult.finalUrl)
|
|
3368
3455
|
}, extractor, dependencies);
|
|
3369
3456
|
const { text, excerpt } = content;
|
|
3457
|
+
const title = safeText$1($("title").first().text());
|
|
3370
3458
|
return {
|
|
3371
3459
|
url: fetchResult.url,
|
|
3372
3460
|
finalUrl: fetchResult.finalUrl,
|
|
3373
|
-
title
|
|
3461
|
+
title,
|
|
3374
3462
|
metaDescription: safeText$1($("meta[name=\"description\"]").attr("content")),
|
|
3375
3463
|
metaRobots,
|
|
3376
3464
|
xRobotsTag: safeText$1(headerValue$4(fetchResult.headers, "x-robots-tag")),
|
|
@@ -3390,7 +3478,10 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
3390
3478
|
externalLinks: [...crawlerExternalLinks],
|
|
3391
3479
|
internalAnchorSamples,
|
|
3392
3480
|
externalAnchorSamples,
|
|
3393
|
-
socialProfileLinks: [...socialProfileLinks]
|
|
3481
|
+
socialProfileLinks: [...socialProfileLinks],
|
|
3482
|
+
contentSketch: contentSketch(text),
|
|
3483
|
+
softAuthenticationGate: softAuthenticationGate($, fetchResult.finalUrl, title),
|
|
3484
|
+
tabbedContent: tabbedContent($)
|
|
3394
3485
|
} } : {},
|
|
3395
3486
|
markdownAlternates,
|
|
3396
3487
|
hreflang,
|
|
@@ -4735,7 +4826,7 @@ function urlInspectionQuotaDate(now) {
|
|
|
4735
4826
|
function urlInspectionQuotaResetAt(now) {
|
|
4736
4827
|
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1));
|
|
4737
4828
|
}
|
|
4738
|
-
function boundedLimit(value) {
|
|
4829
|
+
function boundedLimit$1(value) {
|
|
4739
4830
|
const limit = value ?? 2e3;
|
|
4740
4831
|
if (!Number.isInteger(limit) || limit < 1 || limit > 2e3) throw new SeoError("INVALID_INPUT", "URL Inspection daily limit must be an integer from 1 to 2000.");
|
|
4741
4832
|
return limit;
|
|
@@ -4768,7 +4859,7 @@ function reserveUrlInspectionQuota(input) {
|
|
|
4768
4859
|
const database = input.database ?? getDb();
|
|
4769
4860
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
4770
4861
|
const nowMs = now.getTime();
|
|
4771
|
-
const requestedLimit = boundedLimit(input.limit);
|
|
4862
|
+
const requestedLimit = boundedLimit$1(input.limit);
|
|
4772
4863
|
const quotaDate = urlInspectionQuotaDate(now);
|
|
4773
4864
|
const resetAt = urlInspectionQuotaResetAt(now);
|
|
4774
4865
|
const result = database.transaction(() => {
|
|
@@ -6674,13 +6765,13 @@ function isLowActionabilityQuery(query) {
|
|
|
6674
6765
|
//#endregion
|
|
6675
6766
|
//#region packages/core/src/analyze/page-opportunities-selection.ts
|
|
6676
6767
|
const DEFAULT_MIN_IMPRESSIONS$3 = 10;
|
|
6677
|
-
const DEFAULT_LIMIT$
|
|
6678
|
-
const MAX_LIMIT$
|
|
6768
|
+
const DEFAULT_LIMIT$4 = 25;
|
|
6769
|
+
const MAX_LIMIT$4 = 100;
|
|
6679
6770
|
function integerInRange$2(value, fallback, minimum, maximum) {
|
|
6680
6771
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
6681
6772
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
6682
6773
|
}
|
|
6683
|
-
function isHttpUrl$
|
|
6774
|
+
function isHttpUrl$2(value) {
|
|
6684
6775
|
try {
|
|
6685
6776
|
const parsed = new URL(value);
|
|
6686
6777
|
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
@@ -6702,7 +6793,7 @@ function isBrandRow(input) {
|
|
|
6702
6793
|
function normalizePageOpportunityOptions(input) {
|
|
6703
6794
|
return {
|
|
6704
6795
|
minImpressions: integerInRange$2(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$3, 1, Number.MAX_SAFE_INTEGER),
|
|
6705
|
-
limit: integerInRange$2(input.limit, DEFAULT_LIMIT$
|
|
6796
|
+
limit: integerInRange$2(input.limit, DEFAULT_LIMIT$4, 1, MAX_LIMIT$4)
|
|
6706
6797
|
};
|
|
6707
6798
|
}
|
|
6708
6799
|
function selectTargetRows(input) {
|
|
@@ -6721,7 +6812,7 @@ function selectTargetRows(input) {
|
|
|
6721
6812
|
for (const row of input.rows) {
|
|
6722
6813
|
const query = row.keys[0]?.trim() ?? "";
|
|
6723
6814
|
const page = row.keys[1] ?? "";
|
|
6724
|
-
if (row.keys.length < 2 || !query || !isHttpUrl$
|
|
6815
|
+
if (row.keys.length < 2 || !query || !isHttpUrl$2(page) || !hasValidMetrics(row)) {
|
|
6725
6816
|
selection.invalidRows++;
|
|
6726
6817
|
continue;
|
|
6727
6818
|
}
|
|
@@ -6757,7 +6848,7 @@ function selectTargetRows(input) {
|
|
|
6757
6848
|
function selectBenchmarkRows$1(input) {
|
|
6758
6849
|
return input.rows.filter((row) => {
|
|
6759
6850
|
const query = row.keys[0]?.trim() ?? "";
|
|
6760
|
-
return row.keys.length >= 2 && isHttpUrl$
|
|
6851
|
+
return row.keys.length >= 2 && isHttpUrl$2(row.keys[1] ?? "") && hasValidMetrics(row) && row.position >= 1 && row.position <= 10 && Boolean(query) && !isLowActionabilityQuery(query) && !isBrandRow({
|
|
6761
6852
|
query,
|
|
6762
6853
|
site: input.site,
|
|
6763
6854
|
brandTerms: input.brandTerms,
|
|
@@ -6939,7 +7030,7 @@ const defaultDependencies$15 = {
|
|
|
6939
7030
|
extract: extractPage,
|
|
6940
7031
|
now: () => /* @__PURE__ */ new Date()
|
|
6941
7032
|
};
|
|
6942
|
-
function errorMessage$
|
|
7033
|
+
function errorMessage$2(error) {
|
|
6943
7034
|
return error instanceof Error ? error.message : String(error);
|
|
6944
7035
|
}
|
|
6945
7036
|
function unique$4(values) {
|
|
@@ -6960,7 +7051,7 @@ async function verifyPage(input) {
|
|
|
6960
7051
|
});
|
|
6961
7052
|
warnings.push(...fetched.warnings);
|
|
6962
7053
|
} catch (error) {
|
|
6963
|
-
const reason = `Page fetch failed: ${errorMessage$
|
|
7054
|
+
const reason = `Page fetch failed: ${errorMessage$2(error)}`;
|
|
6964
7055
|
return {
|
|
6965
7056
|
verification: {
|
|
6966
7057
|
status: "failed",
|
|
@@ -6982,7 +7073,7 @@ async function verifyPage(input) {
|
|
|
6982
7073
|
warnings: unique$4(warnings)
|
|
6983
7074
|
};
|
|
6984
7075
|
} catch (error) {
|
|
6985
|
-
const reason = `Page extraction failed: ${errorMessage$
|
|
7076
|
+
const reason = `Page extraction failed: ${errorMessage$2(error)}`;
|
|
6986
7077
|
return {
|
|
6987
7078
|
fetched,
|
|
6988
7079
|
verification: {
|
|
@@ -7623,6 +7714,8 @@ function markdownQuality(markdown, page) {
|
|
|
7623
7714
|
const codeFences = markdown.match(/^```/gmu)?.length ?? 0;
|
|
7624
7715
|
const wordCount = bodyWordCount(markdown);
|
|
7625
7716
|
const sourceWordCount = page.wordCount ?? 0;
|
|
7717
|
+
const evaluatedPanels = (page.tabbedContent?.panelSketches ?? []).map((panel) => contentSketchCoverage(panel.sketch, markdown)).filter((coverage) => coverage !== null);
|
|
7718
|
+
const retainedPanels = evaluatedPanels.filter((coverage) => coverage >= .6).length;
|
|
7626
7719
|
return {
|
|
7627
7720
|
frontmatterTitle: /^---[\s\S]*?^title:\s*.+$/mu.test(markdown),
|
|
7628
7721
|
h1Count: markdown.match(/^#\s+\S.+$/gmu)?.length ?? 0,
|
|
@@ -7639,7 +7732,15 @@ function markdownQuality(markdown, page) {
|
|
|
7639
7732
|
sourceWordCount,
|
|
7640
7733
|
wordRetentionRatio: sourceWordCount > 0 ? Math.round(wordCount / sourceWordCount * 1e3) / 1e3 : null,
|
|
7641
7734
|
introductoryCopyRetained: introProbe ? normalizedMarkdown.includes(introProbe) : null,
|
|
7642
|
-
navigationOnly: bodyWordCount(markdown) < 25
|
|
7735
|
+
navigationOnly: bodyWordCount(markdown) < 25,
|
|
7736
|
+
contentSketchCoverage: contentSketchCoverage(page.contentSketch, markdown),
|
|
7737
|
+
tabbedContent: {
|
|
7738
|
+
detectedPanels: page.tabbedContent?.panels ?? 0,
|
|
7739
|
+
evaluatedPanels: evaluatedPanels.length,
|
|
7740
|
+
retainedPanels,
|
|
7741
|
+
missingPanels: Math.max(0, evaluatedPanels.length - retainedPanels),
|
|
7742
|
+
complete: evaluatedPanels.length === 0 ? null : retainedPanels === evaluatedPanels.length
|
|
7743
|
+
}
|
|
7643
7744
|
};
|
|
7644
7745
|
}
|
|
7645
7746
|
function combinedSignal(timeoutMs, signal) {
|
|
@@ -7693,6 +7794,8 @@ async function fetchRepresentation(input) {
|
|
|
7693
7794
|
status: response.status,
|
|
7694
7795
|
contentType: response.headers.get("content-type") ?? void 0,
|
|
7695
7796
|
bytes: Buffer.byteLength(body),
|
|
7797
|
+
characters: body.length,
|
|
7798
|
+
estimatedTokens: Math.ceil(body.length / 4),
|
|
7696
7799
|
sha256: sha256(body),
|
|
7697
7800
|
canonicalUrl: link.find((entry) => entry.rel.includes("canonical"))?.url,
|
|
7698
7801
|
alternateUrl: link.find((entry) => entry.rel.includes("alternate") && entry.type === "text/markdown")?.url,
|
|
@@ -7751,11 +7854,15 @@ async function inspectMarkdownPage(input) {
|
|
|
7751
7854
|
if (explicit) observation.explicit = explicit.observation;
|
|
7752
7855
|
observation.negotiated = negotiated.observation;
|
|
7753
7856
|
observation.repeated = repeated.observation;
|
|
7754
|
-
|
|
7755
|
-
const
|
|
7756
|
-
|
|
7757
|
-
|
|
7758
|
-
|
|
7857
|
+
const isMarkdown = (result) => result?.observation.status !== void 0 && result.observation.status >= 200 && result.observation.status < 300 && /^\s*text\/markdown\b/iu.test(result.observation.contentType ?? "");
|
|
7858
|
+
const explicitMarkdown = isMarkdown(explicit) ? explicit : void 0;
|
|
7859
|
+
const negotiatedMarkdown = isMarkdown(negotiated) ? negotiated : void 0;
|
|
7860
|
+
const repeatedMarkdown = isMarkdown(repeated) ? repeated : void 0;
|
|
7861
|
+
observation.explicitMatchesNegotiated = explicitMarkdown?.observation.sha256 && negotiatedMarkdown?.observation.sha256 ? explicitMarkdown.observation.sha256 === negotiatedMarkdown.observation.sha256 : null;
|
|
7862
|
+
const primary = explicitMarkdown ?? negotiatedMarkdown;
|
|
7863
|
+
observation.repeatedHashStable = primary?.observation.sha256 && repeatedMarkdown?.observation.sha256 ? primary.observation.sha256 === repeatedMarkdown.observation.sha256 : null;
|
|
7864
|
+
observation.markdownCanonicalMatchesHtml = sameDocument(primary?.observation.canonicalUrl, input.page.finalUrl);
|
|
7865
|
+
if (primary?.body) observation.quality = markdownQuality(primary.body, input.page);
|
|
7759
7866
|
return observation;
|
|
7760
7867
|
}
|
|
7761
7868
|
async function fetchText(input) {
|
|
@@ -7999,17 +8106,16 @@ async function inspectRouteManifest(input) {
|
|
|
7999
8106
|
const declared = Array.isArray(parsed.pages) ? parsed.pages : [];
|
|
8000
8107
|
const declaredHtmlRoutes = declared.map((page) => page.htmlPath).filter((path) => typeof path === "string").sort();
|
|
8001
8108
|
const declaredMarkdownRoutes = declared.map((page) => page.markdownPath).filter((path) => typeof path === "string").sort();
|
|
8002
|
-
const crawledHtmlRoutes = new Set(input.pages.map((page) => new URL(page.finalUrl).pathname.replace(/\/$/u, "") || "/"));
|
|
8003
|
-
const observedMarkdownRoutes = new Set(input.observations.flatMap((page) => page.advertisedUrls.map((value) => new URL(value).pathname)));
|
|
8004
8109
|
return {
|
|
8005
8110
|
url,
|
|
8006
8111
|
status: result.response.status,
|
|
8007
8112
|
valid: Array.isArray(parsed.pages),
|
|
8008
8113
|
declaredHtmlRoutes,
|
|
8009
8114
|
declaredMarkdownRoutes,
|
|
8010
|
-
missingHtmlRoutes:
|
|
8011
|
-
missingMarkdownRoutes:
|
|
8012
|
-
orphanMarkdownRoutes: [
|
|
8115
|
+
missingHtmlRoutes: [],
|
|
8116
|
+
missingMarkdownRoutes: [],
|
|
8117
|
+
orphanMarkdownRoutes: [],
|
|
8118
|
+
comparisonStatus: "partial"
|
|
8013
8119
|
};
|
|
8014
8120
|
} catch (error) {
|
|
8015
8121
|
return {
|
|
@@ -8020,6 +8126,7 @@ async function inspectRouteManifest(input) {
|
|
|
8020
8126
|
missingHtmlRoutes: [],
|
|
8021
8127
|
missingMarkdownRoutes: [],
|
|
8022
8128
|
orphanMarkdownRoutes: [],
|
|
8129
|
+
comparisonStatus: "not-applicable",
|
|
8023
8130
|
error: safeError(error)
|
|
8024
8131
|
};
|
|
8025
8132
|
}
|
|
@@ -8063,9 +8170,7 @@ async function collectAgentDiscovery(input) {
|
|
|
8063
8170
|
}),
|
|
8064
8171
|
inspectRouteManifest({
|
|
8065
8172
|
...input,
|
|
8066
|
-
origin
|
|
8067
|
-
pages,
|
|
8068
|
-
observations
|
|
8173
|
+
origin
|
|
8069
8174
|
}),
|
|
8070
8175
|
inspectAgentEndpoints({
|
|
8071
8176
|
...input,
|
|
@@ -8170,20 +8275,13 @@ async function collectAgentDiscovery(input) {
|
|
|
8170
8275
|
};
|
|
8171
8276
|
}
|
|
8172
8277
|
//#endregion
|
|
8173
|
-
//#region packages/core/src/analyze/crawler/agent-readiness.ts
|
|
8174
|
-
function check$
|
|
8278
|
+
//#region packages/core/src/analyze/crawler/agent-readiness-markdown.ts
|
|
8279
|
+
function check$2(section, input) {
|
|
8175
8280
|
return {
|
|
8176
8281
|
section,
|
|
8177
8282
|
...input
|
|
8178
8283
|
};
|
|
8179
8284
|
}
|
|
8180
|
-
function section$1(id, title, checks) {
|
|
8181
|
-
return {
|
|
8182
|
-
id,
|
|
8183
|
-
title,
|
|
8184
|
-
checks
|
|
8185
|
-
};
|
|
8186
|
-
}
|
|
8187
8285
|
function failedMarkdownUrls(discovery) {
|
|
8188
8286
|
return discovery.markdownAlternates.pages.filter((page) => {
|
|
8189
8287
|
const negotiatedOk = page.negotiated?.status === 200 && /^\s*text\/markdown\b/iu.test(page.negotiated.contentType ?? "") && page.negotiated.varyAccept;
|
|
@@ -8194,38 +8292,12 @@ function failedMarkdownUrls(discovery) {
|
|
|
8194
8292
|
function qualityFailureUrls(discovery) {
|
|
8195
8293
|
return discovery.markdownAlternates.pages.filter((page) => {
|
|
8196
8294
|
const quality = page.quality;
|
|
8197
|
-
if (!quality) return
|
|
8198
|
-
return !quality.frontmatterTitle || quality.h1Count !== 1 || !quality.codeFenceBalanced || quality.rawHtmlTags > 0 || quality.rawSvgTags > 0 || quality.rawScriptTags > 0 || quality.rawStyleTags > 0 || quality.navigationOnly || quality.repeatedLines > 0 || quality.wordRetentionRatio !== null && quality.wordRetentionRatio < .4;
|
|
8295
|
+
if (!quality) return false;
|
|
8296
|
+
return !quality.frontmatterTitle || quality.h1Count !== 1 || !quality.codeFenceBalanced || quality.rawHtmlTags > 0 || quality.rawSvgTags > 0 || quality.rawScriptTags > 0 || quality.rawStyleTags > 0 || quality.navigationOnly || quality.repeatedLines > 0 || quality.contentSketchCoverage !== null && quality.contentSketchCoverage !== void 0 && quality.contentSketchCoverage < .6 || quality.tabbedContent?.complete === false || quality.wordRetentionRatio !== null && quality.wordRetentionRatio < .4;
|
|
8199
8297
|
}).map((page) => page.htmlUrl);
|
|
8200
8298
|
}
|
|
8201
|
-
function
|
|
8202
|
-
return
|
|
8203
|
-
"api",
|
|
8204
|
-
"application",
|
|
8205
|
-
"commerce"
|
|
8206
|
-
].map((profile) => check$1("profile", {
|
|
8207
|
-
id: `profile-${profile}`,
|
|
8208
|
-
status: "notApplicable",
|
|
8209
|
-
title: `${profile[0]?.toUpperCase()}${profile.slice(1)} checks are not applicable`,
|
|
8210
|
-
plainEnglish: discovery.profileApplicability[profile].reason,
|
|
8211
|
-
action: "Choose this profile only when the site exposes that capability publicly.",
|
|
8212
|
-
evidence: {
|
|
8213
|
-
profile,
|
|
8214
|
-
selectedProfile: discovery.profile
|
|
8215
|
-
}
|
|
8216
|
-
}));
|
|
8217
|
-
}
|
|
8218
|
-
function summariseChecks(checks) {
|
|
8219
|
-
const count = (status) => checks.filter((item) => item.status === status).length;
|
|
8220
|
-
return {
|
|
8221
|
-
checks: checks.length,
|
|
8222
|
-
passed: count("pass"),
|
|
8223
|
-
warnings: count("warning"),
|
|
8224
|
-
failed: count("fail"),
|
|
8225
|
-
unknown: count("unknown"),
|
|
8226
|
-
information: count("info"),
|
|
8227
|
-
notApplicable: count("notApplicable")
|
|
8228
|
-
};
|
|
8299
|
+
function confirmedMarkdown(page) {
|
|
8300
|
+
for (const response of [page.explicit, page.negotiated]) if (response?.status !== void 0 && response.status >= 200 && response.status < 300 && /^\s*text\/markdown\b/iu.test(response.contentType ?? "")) return response;
|
|
8229
8301
|
}
|
|
8230
8302
|
function markdownChecks(discovery) {
|
|
8231
8303
|
const markdown = discovery.markdownAlternates;
|
|
@@ -8235,12 +8307,28 @@ function markdownChecks(discovery) {
|
|
|
8235
8307
|
const explicitPages = markdown.pages.filter((page) => page.htmlAlternateUnique).length;
|
|
8236
8308
|
const coverageComplete = markdown.eligibleHtmlPages > 0 && markdown.evaluatedPages === markdown.eligibleHtmlPages;
|
|
8237
8309
|
const negotiationComplete = markdown.eligibleHtmlPages > 0 && negotiatedPages === markdown.eligibleHtmlPages && markdown.pages.every((page) => page.negotiated?.varyAccept) && markdown.pages.every((page) => !page.htmlAlternateUnique || page.explicitMatchesNegotiated === true);
|
|
8238
|
-
const stable = markdown.
|
|
8310
|
+
const stable = markdown.evaluatedPages > 0 && markdown.stableResponses === markdown.evaluatedPages;
|
|
8239
8311
|
const tokenHeaders = markdown.pages.filter((page) => {
|
|
8240
|
-
return (page
|
|
8312
|
+
return confirmedMarkdown(page)?.markdownTokens !== void 0 && (!page.explicit || !page.negotiated || page.explicit.markdownTokens === page.negotiated.markdownTokens);
|
|
8241
8313
|
}).length;
|
|
8314
|
+
const sizedPages = markdown.pages.filter((page) => {
|
|
8315
|
+
return confirmedMarkdown(page)?.characters !== void 0;
|
|
8316
|
+
});
|
|
8317
|
+
const largePages = sizedPages.filter((page) => {
|
|
8318
|
+
return (confirmedMarkdown(page)?.characters ?? 0) > 5e4;
|
|
8319
|
+
});
|
|
8320
|
+
const veryLargePages = sizedPages.filter((page) => {
|
|
8321
|
+
return (confirmedMarkdown(page)?.characters ?? 0) > 1e5;
|
|
8322
|
+
});
|
|
8323
|
+
const maxCharacters = Math.max(0, ...sizedPages.map((page) => confirmedMarkdown(page)?.characters ?? 0));
|
|
8324
|
+
const maxEstimatedTokens = Math.max(0, ...sizedPages.map((page) => confirmedMarkdown(page)?.estimatedTokens ?? 0));
|
|
8325
|
+
const parityPages = markdown.pages.filter((page) => page.quality?.contentSketchCoverage !== null && page.quality?.contentSketchCoverage !== void 0);
|
|
8326
|
+
const parityFailures = parityPages.filter((page) => (page.quality?.contentSketchCoverage ?? 0) < .6);
|
|
8327
|
+
const tabbedPages = markdown.pages.filter((page) => (page.quality?.tabbedContent?.detectedPanels ?? 0) > 0);
|
|
8328
|
+
const tabFailures = tabbedPages.filter((page) => page.quality?.tabbedContent?.complete === false);
|
|
8329
|
+
const tabUnknown = tabbedPages.filter((page) => page.quality?.tabbedContent?.complete === null);
|
|
8242
8330
|
return [
|
|
8243
|
-
check$
|
|
8331
|
+
check$2("representations", {
|
|
8244
8332
|
id: "markdown-coverage",
|
|
8245
8333
|
status: markdown.eligibleHtmlPages === 0 ? "unknown" : coverageComplete ? "pass" : "fail",
|
|
8246
8334
|
title: coverageComplete ? "Every successful HTML page has a Markdown representation" : "Markdown representation coverage is incomplete",
|
|
@@ -8254,17 +8342,62 @@ function markdownChecks(discovery) {
|
|
|
8254
8342
|
},
|
|
8255
8343
|
urls: failedUrls.slice(0, 25)
|
|
8256
8344
|
}),
|
|
8257
|
-
check$
|
|
8345
|
+
check$2("representations", {
|
|
8258
8346
|
id: "markdown-token-estimates",
|
|
8259
|
-
status: markdown.evaluatedPages === 0 ? "unknown" : tokenHeaders === markdown.evaluatedPages ? "pass" : "
|
|
8347
|
+
status: markdown.evaluatedPages === 0 ? "unknown" : tokenHeaders === markdown.evaluatedPages ? "pass" : "info",
|
|
8260
8348
|
title: tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes a token estimate" : "Some Markdown responses are missing a stable token estimate",
|
|
8261
8349
|
plainEnglish: `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned X-Markdown-Tokens. Paired explicit and negotiated responses agreed where both existed. The value is an estimate, not an exact model-specific token count.`,
|
|
8262
|
-
action:
|
|
8350
|
+
action: tokenHeaders === markdown.evaluatedPages ? "Keep the estimate stable when both explicit and negotiated responses exist." : "No change is required. X-Markdown-Tokens is optional; local size estimates are reported separately.",
|
|
8263
8351
|
urls: markdown.pages.filter((page) => {
|
|
8264
|
-
return (page
|
|
8352
|
+
return confirmedMarkdown(page)?.markdownTokens === void 0 || page.explicit !== void 0 && page.negotiated !== void 0 && page.explicit.markdownTokens !== page.negotiated.markdownTokens;
|
|
8265
8353
|
}).map((page) => page.htmlUrl).slice(0, 25)
|
|
8266
8354
|
}),
|
|
8267
|
-
check$
|
|
8355
|
+
check$2("representations", {
|
|
8356
|
+
id: "markdown-size",
|
|
8357
|
+
status: sizedPages.length === 0 ? "unknown" : largePages.length === 0 ? "pass" : "warning",
|
|
8358
|
+
title: sizedPages.length === 0 ? "Markdown size was not measured" : largePages.length === 0 ? "Markdown representations stay within the review threshold" : "Some Markdown representations are large enough to review",
|
|
8359
|
+
plainEnglish: `${sizedPages.length} Markdown responses were measured locally. ${largePages.length} exceeded 50,000 characters and ${veryLargePages.length} exceeded 100,000 characters. These are processing-risk thresholds, not universal model limits.`,
|
|
8360
|
+
action: largePages.length === 0 ? "Keep important content early and retain stable section structure as pages grow." : "Split genuinely separate topics, keep navigation and repeated chrome out of Markdown, and put the most useful content early. Do not remove necessary detail just to hit a threshold.",
|
|
8361
|
+
evidence: {
|
|
8362
|
+
measuredPages: sizedPages.length,
|
|
8363
|
+
reviewThresholdCharacters: 5e4,
|
|
8364
|
+
highRiskThresholdCharacters: 1e5,
|
|
8365
|
+
largePages: largePages.length,
|
|
8366
|
+
veryLargePages: veryLargePages.length,
|
|
8367
|
+
maxCharacters,
|
|
8368
|
+
maxEstimatedTokens,
|
|
8369
|
+
heuristic: true
|
|
8370
|
+
},
|
|
8371
|
+
urls: largePages.map((page) => page.htmlUrl).slice(0, 25)
|
|
8372
|
+
}),
|
|
8373
|
+
check$2("representations", {
|
|
8374
|
+
id: "markdown-content-parity",
|
|
8375
|
+
status: parityPages.length === 0 ? "unknown" : parityFailures.length === 0 ? "pass" : "warning",
|
|
8376
|
+
title: parityPages.length === 0 ? "HTML and Markdown content parity was not measured" : parityFailures.length === 0 ? "Markdown retains sampled content from the HTML document" : "Some Markdown responses may omit important HTML content",
|
|
8377
|
+
plainEnglish: `${parityPages.length} pages were compared using bounded hashes sampled across the main HTML content. ${parityFailures.length} retained less than 60% of that sample. This detects likely omissions without storing full page bodies.`,
|
|
8378
|
+
action: parityFailures.length === 0 ? "Keep the shared conversion path covered as templates and components change." : "Compare the affected HTML and Markdown around the missing sections, then fix the shared converter or component serialization.",
|
|
8379
|
+
evidence: {
|
|
8380
|
+
evaluatedPages: parityPages.length,
|
|
8381
|
+
failedPages: parityFailures.length,
|
|
8382
|
+
minimumCoverage: .6,
|
|
8383
|
+
sampling: "bounded-content-shingles"
|
|
8384
|
+
},
|
|
8385
|
+
urls: parityFailures.map((page) => page.htmlUrl).slice(0, 25)
|
|
8386
|
+
}),
|
|
8387
|
+
check$2("representations", {
|
|
8388
|
+
id: "markdown-tab-serialization",
|
|
8389
|
+
status: tabbedPages.length === 0 ? "notApplicable" : tabFailures.length > 0 ? "warning" : tabUnknown.length > 0 ? "unknown" : "pass",
|
|
8390
|
+
title: tabbedPages.length === 0 ? "No accessible tabbed content was detected" : tabFailures.length > 0 ? "Some tab panels are missing from Markdown" : tabUnknown.length > 0 ? "Some tab panels were too small to compare reliably" : "Markdown retains detected tab panels",
|
|
8391
|
+
plainEnglish: tabbedPages.length === 0 ? "The crawled HTML did not contain ARIA tablists with tab panels, so this check does not apply." : `${tabbedPages.length} pages contained accessible tab panels. ${tabFailures.length} did not retain every evaluated panel in Markdown, and ${tabUnknown.length} had no panel large enough for a reliable comparison.`,
|
|
8392
|
+
action: tabFailures.length === 0 ? tabUnknown.length > 0 ? "Review the affected page directly if the short tab labels or values are important to the document." : "Keep every panel serialized in document order even when only one tab is visible by default." : "Serialize all tab panels into the Markdown document with headings instead of exporting only the initially active panel.",
|
|
8393
|
+
evidence: {
|
|
8394
|
+
tabbedPages: tabbedPages.length,
|
|
8395
|
+
failedPages: tabFailures.length,
|
|
8396
|
+
unknownPages: tabUnknown.length
|
|
8397
|
+
},
|
|
8398
|
+
urls: [...tabFailures, ...tabUnknown].map((page) => page.htmlUrl).slice(0, 25)
|
|
8399
|
+
}),
|
|
8400
|
+
check$2("representations", {
|
|
8268
8401
|
id: "markdown-negotiation",
|
|
8269
8402
|
status: markdown.eligibleHtmlPages === 0 ? "unknown" : negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "pass" : "fail",
|
|
8270
8403
|
title: negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "Markdown content negotiation works across the site" : "Markdown content negotiation needs attention",
|
|
@@ -8278,16 +8411,16 @@ function markdownChecks(discovery) {
|
|
|
8278
8411
|
},
|
|
8279
8412
|
urls: failedUrls.slice(0, 25)
|
|
8280
8413
|
}),
|
|
8281
|
-
check$
|
|
8414
|
+
check$2("representations", {
|
|
8282
8415
|
id: "markdown-determinism",
|
|
8283
|
-
status: markdown.
|
|
8416
|
+
status: markdown.evaluatedPages === 0 ? "unknown" : stable ? "pass" : "fail",
|
|
8284
8417
|
title: stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
|
|
8285
|
-
plainEnglish: `${markdown.stableResponses} of ${markdown.
|
|
8418
|
+
plainEnglish: `${markdown.stableResponses} of ${markdown.evaluatedPages} repeated Markdown responses kept the same SHA-256 digest.`,
|
|
8286
8419
|
action: "Remove timestamps, random output, or other runtime rewriting that changes the same Markdown response between requests.",
|
|
8287
8420
|
evidence: { stableResponses: markdown.stableResponses },
|
|
8288
8421
|
urls: failedUrls.slice(0, 25)
|
|
8289
8422
|
}),
|
|
8290
|
-
check$
|
|
8423
|
+
check$2("representations", {
|
|
8291
8424
|
id: "markdown-quality",
|
|
8292
8425
|
status: markdown.evaluatedPages === 0 ? "unknown" : qualityUrls.length === 0 ? "pass" : "warning",
|
|
8293
8426
|
title: qualityUrls.length === 0 ? "Markdown keeps the useful document structure cleanly" : "Some Markdown alternatives need an extraction review",
|
|
@@ -8297,6 +8430,50 @@ function markdownChecks(discovery) {
|
|
|
8297
8430
|
})
|
|
8298
8431
|
];
|
|
8299
8432
|
}
|
|
8433
|
+
//#endregion
|
|
8434
|
+
//#region packages/core/src/analyze/crawler/agent-readiness.ts
|
|
8435
|
+
function check$1(section, input) {
|
|
8436
|
+
return {
|
|
8437
|
+
section,
|
|
8438
|
+
...input
|
|
8439
|
+
};
|
|
8440
|
+
}
|
|
8441
|
+
function section$2(id, title, checks) {
|
|
8442
|
+
return {
|
|
8443
|
+
id,
|
|
8444
|
+
title,
|
|
8445
|
+
checks
|
|
8446
|
+
};
|
|
8447
|
+
}
|
|
8448
|
+
function profileChecks(discovery) {
|
|
8449
|
+
return [
|
|
8450
|
+
"api",
|
|
8451
|
+
"application",
|
|
8452
|
+
"commerce"
|
|
8453
|
+
].map((profile) => check$1("profile", {
|
|
8454
|
+
id: `profile-${profile}`,
|
|
8455
|
+
status: "notApplicable",
|
|
8456
|
+
title: `${profile[0]?.toUpperCase()}${profile.slice(1)} checks are not applicable`,
|
|
8457
|
+
plainEnglish: discovery.profileApplicability[profile].reason,
|
|
8458
|
+
action: "Choose this profile only when the site exposes that capability publicly.",
|
|
8459
|
+
evidence: {
|
|
8460
|
+
profile,
|
|
8461
|
+
selectedProfile: discovery.profile
|
|
8462
|
+
}
|
|
8463
|
+
}));
|
|
8464
|
+
}
|
|
8465
|
+
function summariseChecks(checks) {
|
|
8466
|
+
const count = (status) => checks.filter((item) => item.status === status).length;
|
|
8467
|
+
return {
|
|
8468
|
+
checks: checks.length,
|
|
8469
|
+
passed: count("pass"),
|
|
8470
|
+
warnings: count("warning"),
|
|
8471
|
+
failed: count("fail"),
|
|
8472
|
+
unknown: count("unknown"),
|
|
8473
|
+
information: count("info"),
|
|
8474
|
+
notApplicable: count("notApplicable")
|
|
8475
|
+
};
|
|
8476
|
+
}
|
|
8300
8477
|
function contentSignalsCheck(discovery, robotsContentSignals) {
|
|
8301
8478
|
const signals = discovery.contentSignals;
|
|
8302
8479
|
const robotsDeclared = robotsContentSignals ?? [];
|
|
@@ -8397,8 +8574,8 @@ function discoveryChecks(discovery, robotsContentSignals) {
|
|
|
8397
8574
|
check$1("discovery", {
|
|
8398
8575
|
id: "route-manifest",
|
|
8399
8576
|
status: !discovery.routeManifest.valid ? "info" : discovery.routeManifest.missingHtmlRoutes.length === 0 && discovery.routeManifest.missingMarkdownRoutes.length === 0 && discovery.routeManifest.orphanMarkdownRoutes.length === 0 ? "pass" : "warning",
|
|
8400
|
-
title: discovery.routeManifest.valid ? "The public route manifest
|
|
8401
|
-
plainEnglish: discovery.routeManifest.valid ? `The manifest declared ${discovery.routeManifest.declaredHtmlRoutes.length} HTML routes and ${discovery.routeManifest.declaredMarkdownRoutes.length} Markdown routes.
|
|
8577
|
+
title: discovery.routeManifest.valid ? "The public route manifest is valid" : "No public route manifest was available",
|
|
8578
|
+
plainEnglish: discovery.routeManifest.valid ? `The manifest declared ${discovery.routeManifest.declaredHtmlRoutes.length} HTML routes and ${discovery.routeManifest.declaredMarkdownRoutes.length} Markdown routes. This bounded crawl did not treat unvisited routes as missing.` : "A public route manifest is optional. Without one, this crawl cannot prove that no orphan Markdown files were deployed.",
|
|
8402
8579
|
action: "Generate one deterministic route inventory from the build when exact HTML and Markdown parity matters.",
|
|
8403
8580
|
evidence: discovery.routeManifest
|
|
8404
8581
|
})
|
|
@@ -8529,25 +8706,42 @@ function accessChecks(report, discovery) {
|
|
|
8529
8706
|
const blocked = bots.filter((bot) => bot.allowed === false);
|
|
8530
8707
|
const robotsKnown = report.ai?.robotsTxt?.availability === "available" || report.ai?.robotsTxt?.availability === "absent";
|
|
8531
8708
|
const protocol = discovery.protocolVariants;
|
|
8532
|
-
|
|
8533
|
-
|
|
8534
|
-
|
|
8535
|
-
|
|
8536
|
-
|
|
8537
|
-
|
|
8538
|
-
|
|
8539
|
-
|
|
8540
|
-
|
|
8541
|
-
|
|
8542
|
-
|
|
8543
|
-
|
|
8544
|
-
|
|
8545
|
-
|
|
8546
|
-
|
|
8547
|
-
|
|
8548
|
-
|
|
8549
|
-
|
|
8550
|
-
|
|
8709
|
+
const authenticationGates = report.issues.filter((issue) => issue.ruleId === "soft_authentication_gate");
|
|
8710
|
+
const clientRenderedPages = report.issues.filter((issue) => issue.ruleId === "client_rendered_content");
|
|
8711
|
+
return [
|
|
8712
|
+
check$1("access", {
|
|
8713
|
+
id: "crawler-access",
|
|
8714
|
+
status: !robotsKnown ? "unknown" : blocked.length > 0 ? "warning" : "pass",
|
|
8715
|
+
title: blocked.length ? "robots.txt blocks selected crawler tokens" : "No selected crawler token is blocked at the start URL",
|
|
8716
|
+
plainEnglish: robotsKnown ? `${blocked.length} of ${bots.length} selected crawler tokens were blocked by the start URL robots.txt policy.` : "The robots.txt response was not stable enough to make a crawler-access claim.",
|
|
8717
|
+
action: "Keep blocks intentional. A crawler token being allowed here does not prove that the service will crawl, index, or cite the site.",
|
|
8718
|
+
evidence: {
|
|
8719
|
+
availability: report.ai?.robotsTxt?.availability,
|
|
8720
|
+
blocked,
|
|
8721
|
+
botAccess: bots
|
|
8722
|
+
}
|
|
8723
|
+
}),
|
|
8724
|
+
check$1("access", {
|
|
8725
|
+
id: "public-content-access",
|
|
8726
|
+
status: authenticationGates.length > 0 || clientRenderedPages.length > 0 ? "warning" : "pass",
|
|
8727
|
+
title: authenticationGates.length > 0 || clientRenderedPages.length > 0 ? "Some public content may be difficult for constrained clients to reach" : "No login gate or empty initial document was detected",
|
|
8728
|
+
plainEnglish: `${authenticationGates.length} crawled pages looked like login gates and ${clientRenderedPages.length} relied on JavaScript to replace an empty or near-empty initial document. These are general site-health observations reused here because they affect agents that do not maintain sessions or run JavaScript.`,
|
|
8729
|
+
action: authenticationGates.length > 0 || clientRenderedPages.length > 0 ? "Make intended public content available without a session and include its important document content in the initial HTML or a complete server-generated representation." : "Keep important public content available without a session or client-side rendering requirement.",
|
|
8730
|
+
evidence: {
|
|
8731
|
+
authenticationGates: authenticationGates.length,
|
|
8732
|
+
clientRenderedPages: clientRenderedPages.length
|
|
8733
|
+
},
|
|
8734
|
+
urls: [...authenticationGates, ...clientRenderedPages].map((issue) => issue.url).slice(0, 25)
|
|
8735
|
+
}),
|
|
8736
|
+
check$1("access", {
|
|
8737
|
+
id: "protocol-canonicalization",
|
|
8738
|
+
status: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "pass" : protocol.http.permanentRedirectToHttps === null ? "unknown" : "warning",
|
|
8739
|
+
title: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "HTTP redirects permanently to HTTPS and HTTPS sends HSTS" : "Protocol canonicalization needs attention or could not be proven",
|
|
8740
|
+
plainEnglish: `The HTTP variant ${protocol.http.permanentRedirectToHttps === true ? "redirected permanently to HTTPS" : "did not produce a confirmed permanent HTTPS redirect"}. The HTTPS start page ${protocol.hstsOnStartPage ? "sent HSTS" : "did not provide confirmed HSTS evidence"}.`,
|
|
8741
|
+
action: "Use one permanent HTTP-to-HTTPS redirect and send HSTS only after HTTPS is stable across the site.",
|
|
8742
|
+
evidence: protocol
|
|
8743
|
+
})
|
|
8744
|
+
];
|
|
8551
8745
|
}
|
|
8552
8746
|
function identityChecks(report) {
|
|
8553
8747
|
const types = new Set(report.pages.flatMap((page) => page.schemaTypes ?? []));
|
|
@@ -8615,19 +8809,19 @@ function agentReadiness(report) {
|
|
|
8615
8809
|
headline: "No agent-discovery evidence is stored in this crawl. Run a fresh focused check.",
|
|
8616
8810
|
profileApplicability: profiles,
|
|
8617
8811
|
summary: summariseChecks(checks),
|
|
8618
|
-
sections: [section$
|
|
8812
|
+
sections: [section$2("collection", "Evidence collection", [missing]), section$2("profile", "Profile scope", scopeChecks)],
|
|
8619
8813
|
checks,
|
|
8620
8814
|
topActions: [missing],
|
|
8621
8815
|
caveats: ["Not evaluated does not mean absent or broken.", "This report does not measure rankings, AI mentions, citations, or selection."]
|
|
8622
8816
|
};
|
|
8623
8817
|
}
|
|
8624
8818
|
const sections = [
|
|
8625
|
-
section$
|
|
8626
|
-
section$
|
|
8627
|
-
section$
|
|
8628
|
-
section$
|
|
8629
|
-
section$
|
|
8630
|
-
section$
|
|
8819
|
+
section$2("profile", "Profile scope", profileChecks(discovery)),
|
|
8820
|
+
section$2("access", "Crawler and protocol access", accessChecks(report, discovery)),
|
|
8821
|
+
section$2("representations", "HTML and Markdown representations", markdownChecks(discovery)),
|
|
8822
|
+
section$2("discovery", "Agent discovery files", discoveryChecks(discovery, report.ai?.robotsTxt?.contentSignals)),
|
|
8823
|
+
section$2("endpoints", "Agent endpoints and auth discovery", endpointChecks(discovery)),
|
|
8824
|
+
section$2("identity", "Identity evidence", identityChecks(report))
|
|
8631
8825
|
];
|
|
8632
8826
|
const checks = sections.flatMap((item) => item.checks);
|
|
8633
8827
|
const actionable = checks.filter((item) => [
|
|
@@ -8814,7 +9008,7 @@ function status(score, maxScore) {
|
|
|
8814
9008
|
if (ratio >= .5) return "warning";
|
|
8815
9009
|
return "fail";
|
|
8816
9010
|
}
|
|
8817
|
-
function section(id, title, checks) {
|
|
9011
|
+
function section$1(id, title, checks) {
|
|
8818
9012
|
return {
|
|
8819
9013
|
id,
|
|
8820
9014
|
title,
|
|
@@ -8872,7 +9066,7 @@ function aiReadiness(report) {
|
|
|
8872
9066
|
const blockedSnippets = restrictedSnippets.filter(({ control }) => control.status === "blocked");
|
|
8873
9067
|
const limitedSnippets = restrictedSnippets.filter(({ control }) => control.status === "limited");
|
|
8874
9068
|
const sections = [
|
|
8875
|
-
section("agent-access", "Agent access", [
|
|
9069
|
+
section$1("agent-access", "Agent access", [
|
|
8876
9070
|
check({
|
|
8877
9071
|
id: "robots-ai-bots",
|
|
8878
9072
|
section: "agent-access",
|
|
@@ -8915,7 +9109,7 @@ function aiReadiness(report) {
|
|
|
8915
9109
|
evidence: { resources: report.ai?.agentResources ?? [] }
|
|
8916
9110
|
})
|
|
8917
9111
|
]),
|
|
8918
|
-
section("machine-readable", "Machine-readable signals", [check({
|
|
9112
|
+
section$1("machine-readable", "Machine-readable signals", [check({
|
|
8919
9113
|
id: "llms-txt",
|
|
8920
9114
|
section: "machine-readable",
|
|
8921
9115
|
maxScore: 0,
|
|
@@ -8940,7 +9134,7 @@ function aiReadiness(report) {
|
|
|
8940
9134
|
action: invalidJsonLdPages.length ? "Fix invalid JSON-LD before relying on its structured-data claims." : "Use structured data only where it accurately represents the visible page and a supported use case.",
|
|
8941
9135
|
urls: invalidJsonLdPages.slice(0, 10).map((page) => page.finalUrl)
|
|
8942
9136
|
})]),
|
|
8943
|
-
section("content-clarity", "Content clarity", [
|
|
9137
|
+
section$1("content-clarity", "Content clarity", [
|
|
8944
9138
|
check({
|
|
8945
9139
|
id: "answerable-content",
|
|
8946
9140
|
section: "content-clarity",
|
|
@@ -8990,7 +9184,7 @@ function aiReadiness(report) {
|
|
|
8990
9184
|
action: "Set the html lang attribute so crawlers and assistive tech understand the page language."
|
|
8991
9185
|
})
|
|
8992
9186
|
]),
|
|
8993
|
-
section("entity-signals", "Entity signals", [check({
|
|
9187
|
+
section$1("entity-signals", "Entity signals", [check({
|
|
8994
9188
|
id: "entity-schema",
|
|
8995
9189
|
section: "entity-signals",
|
|
8996
9190
|
maxScore: 0,
|
|
@@ -9009,7 +9203,7 @@ function aiReadiness(report) {
|
|
|
9009
9203
|
plainEnglish: "Author and date signals help agents judge freshness and accountability.",
|
|
9010
9204
|
action: "Add visible authors, reviewed dates, and updated dates to content where trust and freshness matter."
|
|
9011
9205
|
})]),
|
|
9012
|
-
section("technical-ux", "Technical UX", [
|
|
9206
|
+
section$1("technical-ux", "Technical UX", [
|
|
9013
9207
|
check({
|
|
9014
9208
|
id: "snippet-controls",
|
|
9015
9209
|
section: "technical-ux",
|
|
@@ -9047,7 +9241,7 @@ function aiReadiness(report) {
|
|
|
9047
9241
|
action: "Fix 4xx/5xx pages, accidental noindex, and blocked internal URLs before expanding AI discovery work."
|
|
9048
9242
|
})
|
|
9049
9243
|
]),
|
|
9050
|
-
section("agent-content", "Agent content observations", [
|
|
9244
|
+
section$1("agent-content", "Agent content observations", [
|
|
9051
9245
|
check({
|
|
9052
9246
|
id: "markdown-alternates",
|
|
9053
9247
|
section: "agent-content",
|
|
@@ -9087,7 +9281,7 @@ function aiReadiness(report) {
|
|
|
9087
9281
|
evidence: agentDiscovery?.profileApplicability
|
|
9088
9282
|
})
|
|
9089
9283
|
]),
|
|
9090
|
-
section("crawl-completeness", "Crawl completeness", [check({
|
|
9284
|
+
section$1("crawl-completeness", "Crawl completeness", [check({
|
|
9091
9285
|
id: "crawl-depth",
|
|
9092
9286
|
section: "crawl-completeness",
|
|
9093
9287
|
maxScore: 8,
|
|
@@ -10080,6 +10274,48 @@ const RULE_DEFINITIONS = [
|
|
|
10080
10274
|
suggestedCommands: ["seo crawl --sitemap-url <url> --health --json"]
|
|
10081
10275
|
}
|
|
10082
10276
|
},
|
|
10277
|
+
{
|
|
10278
|
+
id: "soft_authentication_gate",
|
|
10279
|
+
title: "Page appears to be an authentication gate",
|
|
10280
|
+
category: "response",
|
|
10281
|
+
defaultSeverity: "medium",
|
|
10282
|
+
whyItMatters: "The URL returned a successful HTML response, but the visible document looks like a login gate rather than the intended public page. Users and crawlers may receive a 200 response without the content they expected.",
|
|
10283
|
+
howToFix: "Confirm whether authentication is intentional. Public URLs should return their public content without a login form. Protected URLs should use a clear 401 or 403 response where appropriate and should not appear in public sitemaps or internal crawl paths.",
|
|
10284
|
+
impactIfIgnored: "A login page can be mistaken for the intended content, producing misleading crawl results and weak search or agent access.",
|
|
10285
|
+
howToVerify: "Fetch the affected URL without a session and confirm it returns the intended public page, or a deliberate authentication status for protected content.",
|
|
10286
|
+
agentHints: {
|
|
10287
|
+
evidenceFields: ["page.softAuthenticationGate", "page.finalUrl"],
|
|
10288
|
+
suggestedCommands: ["seo crawl <url> --max-pages 1 --json"]
|
|
10289
|
+
}
|
|
10290
|
+
},
|
|
10291
|
+
{
|
|
10292
|
+
id: "client_rendered_content",
|
|
10293
|
+
title: "Important content requires JavaScript rendering",
|
|
10294
|
+
category: "response",
|
|
10295
|
+
defaultSeverity: "medium",
|
|
10296
|
+
whyItMatters: "The raw HTML contained very little content and the useful document appeared only after JavaScript rendering. Some crawlers, agents, previews, and constrained clients do not run the page JavaScript.",
|
|
10297
|
+
howToFix: "Render the important page content into the initial HTML, or publish a complete server-generated representation such as negotiated Markdown. Keep navigation, headings, canonicals, and indexability signals available without client execution.",
|
|
10298
|
+
impactIfIgnored: "Clients that do not execute JavaScript may see an empty shell or miss important content and links.",
|
|
10299
|
+
howToVerify: "Re-run the crawl and confirm the raw and rendered content observations contain the same important document content.",
|
|
10300
|
+
agentHints: {
|
|
10301
|
+
evidenceFields: ["page.fetchDiagnostics.rendering.documentDifference", "page.usedJs"],
|
|
10302
|
+
suggestedCommands: ["seo crawl <url> --json"]
|
|
10303
|
+
}
|
|
10304
|
+
},
|
|
10305
|
+
{
|
|
10306
|
+
id: "soft_404",
|
|
10307
|
+
title: "Unknown URLs return a successful response",
|
|
10308
|
+
category: "response",
|
|
10309
|
+
defaultSeverity: "high",
|
|
10310
|
+
whyItMatters: "Known-nonexistent URLs returned a successful response or redirected to another successful page. Search engines and users cannot reliably distinguish missing content from a real page.",
|
|
10311
|
+
howToFix: "Return 404 or 410 for missing URLs. If a replacement exists, redirect only the specific retired URL to its closest relevant replacement rather than sending every unknown path to a generic page.",
|
|
10312
|
+
impactIfIgnored: "Missing URLs can remain in crawl queues, waste crawl attention, and create confusing search or monitoring evidence.",
|
|
10313
|
+
howToVerify: "Request several known-nonexistent paths and confirm each returns 404 or 410 without redirecting to a generic successful page.",
|
|
10314
|
+
agentHints: {
|
|
10315
|
+
evidenceFields: ["report.siteChecks.soft404"],
|
|
10316
|
+
suggestedCommands: ["seo crawl <url> --json"]
|
|
10317
|
+
}
|
|
10318
|
+
},
|
|
10083
10319
|
{
|
|
10084
10320
|
id: "client_error",
|
|
10085
10321
|
title: "Client error",
|
|
@@ -10569,6 +10805,8 @@ const RULE_RECOMMENDATIONS = {
|
|
|
10569
10805
|
x_robots_noindex: "review",
|
|
10570
10806
|
robots_blocked: "review",
|
|
10571
10807
|
crawler_access_blocked: "review",
|
|
10808
|
+
soft_authentication_gate: "review",
|
|
10809
|
+
client_rendered_content: "review",
|
|
10572
10810
|
orphan_page: "review",
|
|
10573
10811
|
redirected_url: "review",
|
|
10574
10812
|
slow_response: "review",
|
|
@@ -10662,6 +10900,28 @@ const WEAK_VALUABLE_INLINKS = 1;
|
|
|
10662
10900
|
const QUERY_COVERAGE_MIN_IMPRESSIONS = 50;
|
|
10663
10901
|
const QUERY_COVERAGE_MIN = .6;
|
|
10664
10902
|
const OVERSIZED_IMAGE_CANDIDATE_LIMIT = 2e3;
|
|
10903
|
+
const EMPTY_INITIAL_HTML_CHARACTERS = 200;
|
|
10904
|
+
const SUBSTANTIVE_RENDERED_CHARACTERS = 500;
|
|
10905
|
+
function clientRenderedContentEvidence(page) {
|
|
10906
|
+
const rendering = page.fetchDiagnostics?.rendering;
|
|
10907
|
+
const difference = rendering?.documentDifference;
|
|
10908
|
+
if (rendering?.status !== "rendered" || !difference?.changed.includes("content")) return;
|
|
10909
|
+
const rawCharacters = difference.raw.content.characters;
|
|
10910
|
+
const renderedCharacters = difference.rendered.content.characters;
|
|
10911
|
+
if (rawCharacters >= EMPTY_INITIAL_HTML_CHARACTERS || renderedCharacters < SUBSTANTIVE_RENDERED_CHARACTERS || renderedCharacters < rawCharacters * 3) return;
|
|
10912
|
+
return {
|
|
10913
|
+
rawCharacters,
|
|
10914
|
+
rawWords: difference.raw.content.wordCount,
|
|
10915
|
+
renderedCharacters,
|
|
10916
|
+
renderedWords: difference.rendered.content.wordCount,
|
|
10917
|
+
changed: difference.changed,
|
|
10918
|
+
thresholds: {
|
|
10919
|
+
emptyInitialHtmlCharacters: EMPTY_INITIAL_HTML_CHARACTERS,
|
|
10920
|
+
substantiveRenderedCharacters: SUBSTANTIVE_RENDERED_CHARACTERS,
|
|
10921
|
+
minimumGrowthRatio: 3
|
|
10922
|
+
}
|
|
10923
|
+
};
|
|
10924
|
+
}
|
|
10665
10925
|
function issue(ruleId, page, detail, evidence) {
|
|
10666
10926
|
const rule = explainRule(ruleId);
|
|
10667
10927
|
if (!rule) throw new Error(`Missing rule guidance for ${ruleId}`);
|
|
@@ -10876,6 +11136,9 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
10876
11136
|
}));
|
|
10877
11137
|
if (page.auditScope === "status") continue;
|
|
10878
11138
|
if (redirected) continue;
|
|
11139
|
+
if (page.softAuthenticationGate) issues.push(issue("soft_authentication_gate", page, "A password form and login-specific page signals were observed.", page.softAuthenticationGate));
|
|
11140
|
+
const renderedContent = clientRenderedContentEvidence(page);
|
|
11141
|
+
if (renderedContent) issues.push(issue("client_rendered_content", page, "The raw response was an empty or near-empty shell.", renderedContent));
|
|
10879
11142
|
if (isHtmlPage(page) && (page.sizeBytes ?? 0) > LARGE_HTML_BYTES) issues.push(issue("large_html", page, `${Math.round((page.sizeBytes ?? 0) / 1024)} KB`, {
|
|
10880
11143
|
sizeBytes: page.sizeBytes,
|
|
10881
11144
|
thresholdBytes: LARGE_HTML_BYTES
|
|
@@ -11481,7 +11744,7 @@ function createCrawlReport(input) {
|
|
|
11481
11744
|
const requests = sortRequests(sanitizeTenantValue(input.requests ?? []));
|
|
11482
11745
|
const requestEvidenceStatus = input.requestEvidenceStatus ?? (input.requests ? "available" : "unavailable");
|
|
11483
11746
|
if (requestEvidenceStatus === "unavailable" && requests.length > 0) throw new Error("Unavailable request evidence cannot include request observations.");
|
|
11484
|
-
const safeIssues = sanitizeIssues(input.issues ?? [...auditCrawlRequests(requests), ...auditCrawlPages(safePagesWithLinks, { startUrl: config.url })]);
|
|
11747
|
+
const safeIssues = sanitizeIssues([...input.issues ?? [...auditCrawlRequests(requests), ...auditCrawlPages(safePagesWithLinks, { startUrl: config.url })], ...input.additionalIssues ?? []]);
|
|
11485
11748
|
const pages = safePagesWithLinks;
|
|
11486
11749
|
return {
|
|
11487
11750
|
id: input.id ?? crawlRunId(),
|
|
@@ -11510,6 +11773,7 @@ function createCrawlReport(input) {
|
|
|
11510
11773
|
...input.ai ? { ai: sanitizeTenantValue(input.ai) } : {},
|
|
11511
11774
|
...input.sitemapDiscovery ? { sitemapDiscovery: sanitizeTenantValue(input.sitemapDiscovery) } : {},
|
|
11512
11775
|
...input.externalLinkVerification ? { externalLinkVerification: sanitizeTenantValue(input.externalLinkVerification) } : {},
|
|
11776
|
+
...input.siteChecks ? { siteChecks: sanitizeTenantValue(input.siteChecks) } : {},
|
|
11513
11777
|
...input.agentDiscovery ? { agentDiscovery: sanitizeTenantValue(input.agentDiscovery) } : {},
|
|
11514
11778
|
warnings: sanitizeMessages(input.warnings).sort(compareText$13),
|
|
11515
11779
|
caveats: sanitizeMessages(input.caveats).sort(compareText$13)
|
|
@@ -12079,6 +12343,75 @@ function observationFromPage(requestedUrl, page) {
|
|
|
12079
12343
|
};
|
|
12080
12344
|
}
|
|
12081
12345
|
//#endregion
|
|
12346
|
+
//#region packages/core/src/analyze/crawler/site-checks.ts
|
|
12347
|
+
const PROBE_COUNT = 2;
|
|
12348
|
+
function probeUrls(startUrl) {
|
|
12349
|
+
const origin = new URL(startUrl).origin;
|
|
12350
|
+
const seed = createHash("sha256").update(origin).digest("hex").slice(0, 12);
|
|
12351
|
+
return Array.from({ length: PROBE_COUNT }, (_, index) => new URL(`/.well-known/seo-audit/not-found-${seed}-${index + 1}`, origin).toString());
|
|
12352
|
+
}
|
|
12353
|
+
function probeObservation(url, result) {
|
|
12354
|
+
const request = result.request;
|
|
12355
|
+
if (!request) return {
|
|
12356
|
+
url,
|
|
12357
|
+
error: result.warning ?? "No response evidence."
|
|
12358
|
+
};
|
|
12359
|
+
if (request.outcome === "response") return {
|
|
12360
|
+
url,
|
|
12361
|
+
status: request.status,
|
|
12362
|
+
finalUrl: request.finalUrl,
|
|
12363
|
+
redirected: request.finalUrl !== url,
|
|
12364
|
+
accessBlocked: Boolean(request.accessBlock)
|
|
12365
|
+
};
|
|
12366
|
+
return {
|
|
12367
|
+
url,
|
|
12368
|
+
error: request.outcome === "failure" ? request.error : `Probe skipped: ${request.reason}`
|
|
12369
|
+
};
|
|
12370
|
+
}
|
|
12371
|
+
async function collectCrawlSiteChecks(input) {
|
|
12372
|
+
const probes = [];
|
|
12373
|
+
for (const url of probeUrls(input.startUrl)) {
|
|
12374
|
+
if (input.signal?.aborted) break;
|
|
12375
|
+
const result = await input.fetchStatusPage(url, {
|
|
12376
|
+
js: "off",
|
|
12377
|
+
refresh: true,
|
|
12378
|
+
timeoutMs: input.timeoutMs,
|
|
12379
|
+
signal: input.signal,
|
|
12380
|
+
writeCache: false,
|
|
12381
|
+
respectRobots: input.respectRobots
|
|
12382
|
+
});
|
|
12383
|
+
probes.push(probeObservation(url, result));
|
|
12384
|
+
}
|
|
12385
|
+
const conclusive = probes.filter((probe) => probe.status !== void 0 && !probe.accessBlocked);
|
|
12386
|
+
const successful = conclusive.filter((probe) => (probe.status ?? 0) >= 200 && (probe.status ?? 0) < 400);
|
|
12387
|
+
const correctMissing = conclusive.filter((probe) => probe.status === 404 || probe.status === 410);
|
|
12388
|
+
const status = conclusive.length !== PROBE_COUNT ? "unknown" : successful.length === PROBE_COUNT ? "warning" : correctMissing.length === PROBE_COUNT ? "pass" : "unknown";
|
|
12389
|
+
const checks = { soft404: {
|
|
12390
|
+
status,
|
|
12391
|
+
probes,
|
|
12392
|
+
probeLimit: PROBE_COUNT,
|
|
12393
|
+
complete: probes.length === PROBE_COUNT
|
|
12394
|
+
} };
|
|
12395
|
+
if (status !== "warning") return {
|
|
12396
|
+
checks,
|
|
12397
|
+
issues: []
|
|
12398
|
+
};
|
|
12399
|
+
const rule = explainRule("soft_404");
|
|
12400
|
+
if (!rule) throw new Error("Missing rule guidance for soft_404");
|
|
12401
|
+
return {
|
|
12402
|
+
checks,
|
|
12403
|
+
issues: [{
|
|
12404
|
+
ruleId: "soft_404",
|
|
12405
|
+
title: rule.title,
|
|
12406
|
+
category: rule.category,
|
|
12407
|
+
severity: rule.defaultSeverity,
|
|
12408
|
+
url: new URL("/", input.startUrl).toString(),
|
|
12409
|
+
detail: `${successful.length} known-nonexistent paths returned a successful response or redirect.`,
|
|
12410
|
+
evidence: checks.soft404
|
|
12411
|
+
}]
|
|
12412
|
+
};
|
|
12413
|
+
}
|
|
12414
|
+
//#endregion
|
|
12082
12415
|
//#region packages/core/src/analyze/crawler/site-crawl-ai-signals.ts
|
|
12083
12416
|
const MAX_AUXILIARY_RESPONSE_BYTES = 2 * 1024 * 1024;
|
|
12084
12417
|
const AI_BOT_USER_AGENTS = [
|
|
@@ -12798,6 +13131,9 @@ async function crawlOne(url, opts = {}) {
|
|
|
12798
13131
|
mainContentHash: hashText(extracted.contentText),
|
|
12799
13132
|
textRatio: fetched.html.length ? Math.min(1, extracted.contentText.length / fetched.html.length) : void 0,
|
|
12800
13133
|
contentSample: truncate$1(extracted.contentText, 300),
|
|
13134
|
+
contentSketch: crawlerEvidence.contentSketch,
|
|
13135
|
+
softAuthenticationGate: crawlerEvidence.softAuthenticationGate,
|
|
13136
|
+
tabbedContent: crawlerEvidence.tabbedContent,
|
|
12801
13137
|
lang: extracted.lang,
|
|
12802
13138
|
hasViewport: extracted.hasViewport,
|
|
12803
13139
|
isHttps: new URL(extracted.finalUrl).protocol === "https:",
|
|
@@ -14082,6 +14418,13 @@ async function crawlSite(input, dependencies) {
|
|
|
14082
14418
|
signal,
|
|
14083
14419
|
concurrency: Math.min(config.concurrency, 4)
|
|
14084
14420
|
}) : void 0;
|
|
14421
|
+
const siteCheckResult = !healthStrategy && !cancelled && !stoppedForMemory ? await collectCrawlSiteChecks({
|
|
14422
|
+
startUrl: config.url,
|
|
14423
|
+
timeoutMs: config.timeoutMs,
|
|
14424
|
+
fetchStatusPage: deps.fetchStatusPage,
|
|
14425
|
+
respectRobots: config.respectRobots,
|
|
14426
|
+
signal
|
|
14427
|
+
}) : void 0;
|
|
14085
14428
|
for (const page of pages) {
|
|
14086
14429
|
if (!page.geo || !llmsTxt) continue;
|
|
14087
14430
|
page.geo = {
|
|
@@ -14107,6 +14450,10 @@ async function crawlSite(input, dependencies) {
|
|
|
14107
14450
|
} } : {},
|
|
14108
14451
|
...sitemapDiscovery ? { sitemapDiscovery } : {},
|
|
14109
14452
|
...externalLinkVerification ? { externalLinkVerification } : {},
|
|
14453
|
+
...siteCheckResult ? {
|
|
14454
|
+
siteChecks: siteCheckResult.checks,
|
|
14455
|
+
additionalIssues: siteCheckResult.issues
|
|
14456
|
+
} : {},
|
|
14110
14457
|
...agentDiscovery ? { agentDiscovery } : {},
|
|
14111
14458
|
status: reportStatus,
|
|
14112
14459
|
warnings,
|
|
@@ -15603,19 +15950,19 @@ function renderCrawlHtml(report, fixes = topFixes(report)) {
|
|
|
15603
15950
|
const reviews = reviewObservations(report);
|
|
15604
15951
|
const issueRows = report.issueGroups.map((group) => {
|
|
15605
15952
|
const rule = explainRule(group.ruleId);
|
|
15606
|
-
return `<tr><td>${escapeHtml(group.severity)}</td><td>${escapeHtml(group.title)}</td><td>${group.count}</td><td>${escapeHtml(rule?.howToFix ?? "")}</td><td>${escapeHtml(group.sampleUrls[0] ?? "")}</td></tr>`;
|
|
15953
|
+
return `<tr><td>${escapeHtml$1(group.severity)}</td><td>${escapeHtml$1(group.title)}</td><td>${group.count}</td><td>${escapeHtml$1(rule?.howToFix ?? "")}</td><td>${escapeHtml$1(group.sampleUrls[0] ?? "")}</td></tr>`;
|
|
15607
15954
|
}).join("\n");
|
|
15608
|
-
const fixItems = fixes.slice(0, 10).map((fix) => `<li><strong>${escapeHtml(fix.title)}</strong><br>${escapeHtml(fix.howToFix)}<br><span>Verify: ${escapeHtml(fix.howToVerify)}</span><br><code>${escapeHtml(fix.verification.command)}</code></li>`).join("\n");
|
|
15609
|
-
const reviewItems = reviews.slice(0, 10).map((observation) => `<li><strong>${escapeHtml(observation.title)}</strong><br>${escapeHtml(observation.howToFix)}<br><span>Check before scheduling work. ${escapeHtml(observation.howToVerify)}</span></li>`).join("\n");
|
|
15610
|
-
const caveatItems = report.caveats.map((caveat) => `<li>${escapeHtml(caveat)}</li>`).join("\n");
|
|
15611
|
-
const warningItems = report.warnings.map((warning) => `<li>${escapeHtml(warning)}</li>`).join("\n");
|
|
15612
|
-
const accessItems = report.access.samples.map((sample) => `<li><strong>${escapeHtml(sample.url)}</strong><br>${escapeHtml(sample.evidence.guidance.summary)}<br>${escapeHtml(sample.evidence.guidance.recommendedAction)}<br><span>${escapeHtml(sample.evidence.guidance.securityNote)}</span></li>`).join("\n");
|
|
15955
|
+
const fixItems = fixes.slice(0, 10).map((fix) => `<li><strong>${escapeHtml$1(fix.title)}</strong><br>${escapeHtml$1(fix.howToFix)}<br><span>Verify: ${escapeHtml$1(fix.howToVerify)}</span><br><code>${escapeHtml$1(fix.verification.command)}</code></li>`).join("\n");
|
|
15956
|
+
const reviewItems = reviews.slice(0, 10).map((observation) => `<li><strong>${escapeHtml$1(observation.title)}</strong><br>${escapeHtml$1(observation.howToFix)}<br><span>Check before scheduling work. ${escapeHtml$1(observation.howToVerify)}</span></li>`).join("\n");
|
|
15957
|
+
const caveatItems = report.caveats.map((caveat) => `<li>${escapeHtml$1(caveat)}</li>`).join("\n");
|
|
15958
|
+
const warningItems = report.warnings.map((warning) => `<li>${escapeHtml$1(warning)}</li>`).join("\n");
|
|
15959
|
+
const accessItems = report.access.samples.map((sample) => `<li><strong>${escapeHtml$1(sample.url)}</strong><br>${escapeHtml$1(sample.evidence.guidance.summary)}<br>${escapeHtml$1(sample.evidence.guidance.recommendedAction)}<br><span>${escapeHtml$1(sample.evidence.guidance.securityNote)}</span></li>`).join("\n");
|
|
15613
15960
|
return `<!doctype html>
|
|
15614
15961
|
<html lang="en">
|
|
15615
15962
|
<head>
|
|
15616
15963
|
<meta charset="utf-8">
|
|
15617
15964
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
15618
|
-
<title>Crawl report for ${escapeHtml(report.config.url)}</title>
|
|
15965
|
+
<title>Crawl report for ${escapeHtml$1(report.config.url)}</title>
|
|
15619
15966
|
<style>
|
|
15620
15967
|
body { color: #17202a; font: 15px/1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; margin: 32px; max-width: 1120px; }
|
|
15621
15968
|
h1, h2 { line-height: 1.2; }
|
|
@@ -15631,10 +15978,10 @@ function renderCrawlHtml(report, fixes = topFixes(report)) {
|
|
|
15631
15978
|
</head>
|
|
15632
15979
|
<body>
|
|
15633
15980
|
<h1>Crawl report</h1>
|
|
15634
|
-
<p>${escapeHtml(report.config.url)}</p>
|
|
15635
|
-
<p>Strategy: ${escapeHtml(report.config.strategy === "health" ? "sitemap health pass" : "full audit")}<br>Crawler: <code>${escapeHtml(report.access.crawler.userAgent)}</code></p>
|
|
15981
|
+
<p>${escapeHtml$1(report.config.url)}</p>
|
|
15982
|
+
<p>Strategy: ${escapeHtml$1(report.config.strategy === "health" ? "sitemap health pass" : "full audit")}<br>Crawler: <code>${escapeHtml$1(report.access.crawler.userAgent)}</code></p>
|
|
15636
15983
|
<section class="summary">
|
|
15637
|
-
<div class="metric"><span>Status</span><strong>${escapeHtml(report.status)}</strong></div>
|
|
15984
|
+
<div class="metric"><span>Status</span><strong>${escapeHtml$1(report.status)}</strong></div>
|
|
15638
15985
|
<div class="metric"><span>Documents</span><strong>${report.summary.totalPages}</strong></div>
|
|
15639
15986
|
<div class="metric"><span>Requests</span><strong>${report.requestEvidenceStatus === "unavailable" ? "n/a" : report.requests.length}${report.requestEvidenceStatus === "partial" ? " (partial)" : ""}</strong></div>
|
|
15640
15987
|
<div class="metric"><span>Discovered</span><strong>${report.summary.discoveredUrls}</strong></div>
|
|
@@ -15688,11 +16035,11 @@ function renderCrawlMarkdownTickets(report, fixes = topFixes(report)) {
|
|
|
15688
16035
|
if (reviews.length) lines.push("## Review observations", "", "These are not implementation tickets. Check the evidence before deciding whether to work on them.", "", ...reviews.slice(0, 10).flatMap((observation) => [`- ${observation.title} (${observation.severity}, ${observation.count} URL${observation.count === 1 ? "" : "s"}): ${observation.howToFix}`, ` - First URL: ${observation.sampleUrls[0] ?? "No sample URL recorded"}`]), "");
|
|
15689
16036
|
return `${lines.join("\n")}\n`;
|
|
15690
16037
|
}
|
|
15691
|
-
function escapeHtml(value) {
|
|
16038
|
+
function escapeHtml$1(value) {
|
|
15692
16039
|
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll("\"", """).replaceAll("'", "'");
|
|
15693
16040
|
}
|
|
15694
16041
|
function escapeXml(value) {
|
|
15695
|
-
return escapeHtml(value);
|
|
16042
|
+
return escapeHtml$1(value);
|
|
15696
16043
|
}
|
|
15697
16044
|
//#endregion
|
|
15698
16045
|
//#region packages/core/src/analyze/crawler/history-provenance.ts
|
|
@@ -16399,6 +16746,8 @@ const agentRepresentationResponseSchema = z.object({
|
|
|
16399
16746
|
status: z.number().int().optional(),
|
|
16400
16747
|
contentType: z.string().optional(),
|
|
16401
16748
|
bytes: z.number().int().nonnegative().optional(),
|
|
16749
|
+
characters: z.number().int().nonnegative().optional(),
|
|
16750
|
+
estimatedTokens: z.number().int().nonnegative().optional(),
|
|
16402
16751
|
sha256: z.string().optional(),
|
|
16403
16752
|
canonicalUrl: z.string().url().optional(),
|
|
16404
16753
|
alternateUrl: z.string().url().optional(),
|
|
@@ -16456,7 +16805,15 @@ const agentDiscoverySchema = z.object({
|
|
|
16456
16805
|
sourceWordCount: z.number().int().nonnegative(),
|
|
16457
16806
|
wordRetentionRatio: z.number().nonnegative().nullable(),
|
|
16458
16807
|
introductoryCopyRetained: z.boolean().nullable(),
|
|
16459
|
-
navigationOnly: z.boolean()
|
|
16808
|
+
navigationOnly: z.boolean(),
|
|
16809
|
+
contentSketchCoverage: z.number().min(0).max(1).nullable().optional(),
|
|
16810
|
+
tabbedContent: z.object({
|
|
16811
|
+
detectedPanels: z.number().int().nonnegative(),
|
|
16812
|
+
evaluatedPanels: z.number().int().nonnegative(),
|
|
16813
|
+
retainedPanels: z.number().int().nonnegative(),
|
|
16814
|
+
missingPanels: z.number().int().nonnegative(),
|
|
16815
|
+
complete: z.boolean().nullable()
|
|
16816
|
+
}).optional()
|
|
16460
16817
|
}).optional()
|
|
16461
16818
|
}))
|
|
16462
16819
|
}),
|
|
@@ -16475,6 +16832,11 @@ const agentDiscoverySchema = z.object({
|
|
|
16475
16832
|
missingHtmlRoutes: z.array(z.string()),
|
|
16476
16833
|
missingMarkdownRoutes: z.array(z.string()),
|
|
16477
16834
|
orphanMarkdownRoutes: z.array(z.string()),
|
|
16835
|
+
comparisonStatus: z.enum([
|
|
16836
|
+
"complete",
|
|
16837
|
+
"partial",
|
|
16838
|
+
"not-applicable"
|
|
16839
|
+
]).optional(),
|
|
16478
16840
|
error: z.string().optional()
|
|
16479
16841
|
}),
|
|
16480
16842
|
agentSkills: z.object({
|
|
@@ -16580,6 +16942,25 @@ const agentDiscoverySchema = z.object({
|
|
|
16580
16942
|
warnings: z.array(z.string())
|
|
16581
16943
|
});
|
|
16582
16944
|
//#endregion
|
|
16945
|
+
//#region packages/core/src/analyze/crawler/site-checks-schema.ts
|
|
16946
|
+
const crawlSiteChecksSchema = z.object({ soft404: z.object({
|
|
16947
|
+
status: z.enum([
|
|
16948
|
+
"pass",
|
|
16949
|
+
"warning",
|
|
16950
|
+
"unknown"
|
|
16951
|
+
]),
|
|
16952
|
+
probes: z.array(z.object({
|
|
16953
|
+
url: z.string().url(),
|
|
16954
|
+
status: z.number().int().optional(),
|
|
16955
|
+
finalUrl: z.string().url().optional(),
|
|
16956
|
+
redirected: z.boolean().optional(),
|
|
16957
|
+
accessBlocked: z.boolean().optional(),
|
|
16958
|
+
error: z.string().optional()
|
|
16959
|
+
})),
|
|
16960
|
+
probeLimit: z.number().int().positive(),
|
|
16961
|
+
complete: z.boolean()
|
|
16962
|
+
}) });
|
|
16963
|
+
//#endregion
|
|
16583
16964
|
//#region packages/core/src/analyze/crawler/schemas.ts
|
|
16584
16965
|
const crawlerRuleSeveritySchema = z.enum([
|
|
16585
16966
|
"low",
|
|
@@ -17009,6 +17390,34 @@ const crawlPageSnapshotSchema = z.object({
|
|
|
17009
17390
|
mainContentHash: z.string().optional(),
|
|
17010
17391
|
textRatio: z.number().min(0).max(1).optional(),
|
|
17011
17392
|
contentSample: z.string().optional(),
|
|
17393
|
+
contentSketch: z.object({
|
|
17394
|
+
version: z.literal(1),
|
|
17395
|
+
wordCount: z.number().int().nonnegative(),
|
|
17396
|
+
shingleSize: z.number().int().positive(),
|
|
17397
|
+
sampledShingles: z.number().int().nonnegative(),
|
|
17398
|
+
hashes: z.array(z.string())
|
|
17399
|
+
}).optional(),
|
|
17400
|
+
softAuthenticationGate: z.object({
|
|
17401
|
+
kind: z.literal("login-form"),
|
|
17402
|
+
indicators: z.array(z.string()),
|
|
17403
|
+
formActionPath: z.string().optional()
|
|
17404
|
+
}).optional(),
|
|
17405
|
+
tabbedContent: z.object({
|
|
17406
|
+
groups: z.number().int().positive(),
|
|
17407
|
+
panels: z.number().int().positive(),
|
|
17408
|
+
retainedPanels: z.number().int().nonnegative(),
|
|
17409
|
+
truncated: z.boolean(),
|
|
17410
|
+
panelSketches: z.array(z.object({
|
|
17411
|
+
label: z.string().optional(),
|
|
17412
|
+
sketch: z.object({
|
|
17413
|
+
version: z.literal(1),
|
|
17414
|
+
wordCount: z.number().int().nonnegative(),
|
|
17415
|
+
shingleSize: z.number().int().positive(),
|
|
17416
|
+
sampledShingles: z.number().int().nonnegative(),
|
|
17417
|
+
hashes: z.array(z.string())
|
|
17418
|
+
})
|
|
17419
|
+
}))
|
|
17420
|
+
}).optional(),
|
|
17012
17421
|
lang: z.string().optional(),
|
|
17013
17422
|
hasViewport: z.boolean().optional(),
|
|
17014
17423
|
isHttps: z.boolean().optional(),
|
|
@@ -17500,6 +17909,7 @@ const crawlReportBaseSchema = z.object({
|
|
|
17500
17909
|
ai: crawlAiSignalsSchema.optional(),
|
|
17501
17910
|
sitemapDiscovery: crawlSitemapDiscoverySchema.optional(),
|
|
17502
17911
|
externalLinkVerification: crawlExternalLinkVerificationSchema.optional(),
|
|
17912
|
+
siteChecks: crawlSiteChecksSchema.optional(),
|
|
17503
17913
|
agentDiscovery: agentDiscoverySchema.optional(),
|
|
17504
17914
|
warnings: z.array(z.string()),
|
|
17505
17915
|
caveats: z.array(z.string())
|
|
@@ -17549,7 +17959,7 @@ function compareText$10(left, right) {
|
|
|
17549
17959
|
}
|
|
17550
17960
|
return leftPoints.length - rightPoints.length;
|
|
17551
17961
|
}
|
|
17552
|
-
function boundedInteger$
|
|
17962
|
+
function boundedInteger$6(value, fallback, minimum, maximum) {
|
|
17553
17963
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
17554
17964
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
17555
17965
|
}
|
|
@@ -17626,8 +18036,8 @@ function initialSelection(sourceRows) {
|
|
|
17626
18036
|
};
|
|
17627
18037
|
}
|
|
17628
18038
|
function analyzeCtrUnderperformersFromRows(input) {
|
|
17629
|
-
const minImpressions = boundedInteger$
|
|
17630
|
-
const limit = boundedInteger$
|
|
18039
|
+
const minImpressions = boundedInteger$6(input.minImpressions, 200, 1, CTR_MAX_MIN_IMPRESSIONS);
|
|
18040
|
+
const limit = boundedInteger$6(input.limit, 25, 1, 100);
|
|
17631
18041
|
const selection = initialSelection(input.rows.length);
|
|
17632
18042
|
const benchmarkRows = aggregateCtrRows(input.rows, selection).filter((row) => {
|
|
17633
18043
|
const query = row.keys[0] ?? "";
|
|
@@ -18126,7 +18536,7 @@ function analyzeSegmentRows(input) {
|
|
|
18126
18536
|
}
|
|
18127
18537
|
//#endregion
|
|
18128
18538
|
//#region packages/core/src/analyze/segment-impact-input.ts
|
|
18129
|
-
const DAY_MS$
|
|
18539
|
+
const DAY_MS$2 = 864e5;
|
|
18130
18540
|
const MAX_WINDOW_DAYS = 240;
|
|
18131
18541
|
function subtractDays$1(value, days) {
|
|
18132
18542
|
const date = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -18134,7 +18544,7 @@ function subtractDays$1(value, days) {
|
|
|
18134
18544
|
return date.toISOString().slice(0, 10);
|
|
18135
18545
|
}
|
|
18136
18546
|
function segmentRangeDays(range) {
|
|
18137
|
-
return Math.floor((Date.parse(`${range.endDate}T00:00:00.000Z`) - Date.parse(`${range.startDate}T00:00:00.000Z`)) / DAY_MS$
|
|
18547
|
+
return Math.floor((Date.parse(`${range.endDate}T00:00:00.000Z`) - Date.parse(`${range.startDate}T00:00:00.000Z`)) / DAY_MS$2) + 1;
|
|
18138
18548
|
}
|
|
18139
18549
|
function segmentComparisonRange(after, compareDays) {
|
|
18140
18550
|
const endDate = subtractDays$1(after.startDate, 1);
|
|
@@ -18761,13 +19171,13 @@ const MINIMUM_PAGE_SHARE = .1;
|
|
|
18761
19171
|
const MINIMUM_PAGE_IMPRESSIONS = 10;
|
|
18762
19172
|
const MAXIMUM_DOMINANT_SHARE = .8;
|
|
18763
19173
|
const MAX_SUPPRESSIONS = 100;
|
|
18764
|
-
function boundedInteger$
|
|
19174
|
+
function boundedInteger$5(value, fallback, minimum, maximum) {
|
|
18765
19175
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
18766
19176
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
18767
19177
|
}
|
|
18768
19178
|
function analyzeCannibalRows(input) {
|
|
18769
|
-
const minImpressions = boundedInteger$
|
|
18770
|
-
const limit = boundedInteger$
|
|
19179
|
+
const minImpressions = boundedInteger$5(input.minImpressions, 50, 0, 1e9);
|
|
19180
|
+
const limit = boundedInteger$5(input.limit, 25, 1, 100);
|
|
18771
19181
|
const selection = {
|
|
18772
19182
|
sourceRows: input.rows.length,
|
|
18773
19183
|
invalidRows: 0,
|
|
@@ -19549,7 +19959,7 @@ function groupQuickWins(items) {
|
|
|
19549
19959
|
}
|
|
19550
19960
|
//#endregion
|
|
19551
19961
|
//#region packages/core/src/analyze/site-diagnostics/quick-wins-analysis-primitives.ts
|
|
19552
|
-
function boundedInteger$
|
|
19962
|
+
function boundedInteger$4(value, fallback, minimum, maximum) {
|
|
19553
19963
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
19554
19964
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
19555
19965
|
}
|
|
@@ -19670,12 +20080,12 @@ function roundedSum(values) {
|
|
|
19670
20080
|
//#region packages/core/src/analyze/site-diagnostics/quick-wins-analysis.ts
|
|
19671
20081
|
const DEFAULT_MIN_IMPRESSIONS$2 = 200;
|
|
19672
20082
|
const MAX_MIN_IMPRESSIONS$2 = 1e9;
|
|
19673
|
-
const DEFAULT_LIMIT$
|
|
19674
|
-
const MAX_LIMIT$
|
|
20083
|
+
const DEFAULT_LIMIT$3 = 25;
|
|
20084
|
+
const MAX_LIMIT$3 = 100;
|
|
19675
20085
|
const FALLBACK_SOURCE = "builtin_position_ctr_curve_v1";
|
|
19676
20086
|
function analyzeQuickWinsFromRows(input) {
|
|
19677
|
-
const minImpressions = boundedInteger$
|
|
19678
|
-
const limit = boundedInteger$
|
|
20087
|
+
const minImpressions = boundedInteger$4(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$2, 0, MAX_MIN_IMPRESSIONS$2);
|
|
20088
|
+
const limit = boundedInteger$4(input.limit, DEFAULT_LIMIT$3, 1, MAX_LIMIT$3);
|
|
19679
20089
|
const selected = selectBenchmarkRows(input);
|
|
19680
20090
|
const benchmarkContext = createCtrBenchmarkContext(selected.rows, {
|
|
19681
20091
|
samplePopulation: "all_qualified_url_samples",
|
|
@@ -20016,7 +20426,7 @@ function compareText$5(left, right) {
|
|
|
20016
20426
|
}
|
|
20017
20427
|
return leftPoints.length - rightPoints.length;
|
|
20018
20428
|
}
|
|
20019
|
-
function isHttpUrl(value) {
|
|
20429
|
+
function isHttpUrl$1(value) {
|
|
20020
20430
|
try {
|
|
20021
20431
|
const parsed = new URL(value);
|
|
20022
20432
|
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
@@ -20025,7 +20435,7 @@ function isHttpUrl(value) {
|
|
|
20025
20435
|
}
|
|
20026
20436
|
}
|
|
20027
20437
|
function isValidStrikingDistanceRow(row, query, url) {
|
|
20028
|
-
return Boolean(query) && isHttpUrl(url) && Number.isFinite(row.clicks) && Number.isFinite(row.impressions) && Number.isFinite(row.ctr) && Number.isFinite(row.position) && row.clicks >= 0 && row.impressions > 0 && row.clicks <= row.impressions && row.ctr >= 0 && row.ctr <= 1 && row.position > 0;
|
|
20438
|
+
return Boolean(query) && isHttpUrl$1(url) && Number.isFinite(row.clicks) && Number.isFinite(row.impressions) && Number.isFinite(row.ctr) && Number.isFinite(row.position) && row.clicks >= 0 && row.impressions > 0 && row.clicks <= row.impressions && row.ctr >= 0 && row.ctr <= 1 && row.position > 0;
|
|
20029
20439
|
}
|
|
20030
20440
|
function isStrikingDistanceBrandRow(input) {
|
|
20031
20441
|
return shouldExcludeBrandQuery({
|
|
@@ -20115,8 +20525,8 @@ function groupStrikingDistanceItems(items) {
|
|
|
20115
20525
|
//#region packages/core/src/analyze/striking-distance-analysis.ts
|
|
20116
20526
|
const DEFAULT_MIN_IMPRESSIONS$1 = 100;
|
|
20117
20527
|
const MAX_MIN_IMPRESSIONS$1 = 1e9;
|
|
20118
|
-
const DEFAULT_LIMIT$
|
|
20119
|
-
const MAX_LIMIT$
|
|
20528
|
+
const DEFAULT_LIMIT$2 = 25;
|
|
20529
|
+
const MAX_LIMIT$2 = 100;
|
|
20120
20530
|
function integerInRange$1(value, fallback, minimum, maximum) {
|
|
20121
20531
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
20122
20532
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
@@ -20158,7 +20568,7 @@ function selectRows(input) {
|
|
|
20158
20568
|
}
|
|
20159
20569
|
function analyzeStrikingDistanceRows(input) {
|
|
20160
20570
|
const minImpressions = integerInRange$1(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$1, 0, MAX_MIN_IMPRESSIONS$1);
|
|
20161
|
-
const limit = integerInRange$1(input.limit, DEFAULT_LIMIT$
|
|
20571
|
+
const limit = integerInRange$1(input.limit, DEFAULT_LIMIT$2, 1, MAX_LIMIT$2);
|
|
20162
20572
|
const selected = selectRows({
|
|
20163
20573
|
...input,
|
|
20164
20574
|
minImpressions
|
|
@@ -20232,7 +20642,7 @@ const defaultDependencies$7 = {
|
|
|
20232
20642
|
verifyContent: verifyQueryContent,
|
|
20233
20643
|
now: () => /* @__PURE__ */ new Date()
|
|
20234
20644
|
};
|
|
20235
|
-
function boundedInteger$
|
|
20645
|
+
function boundedInteger$3(value, fallback, minimum, maximum) {
|
|
20236
20646
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
20237
20647
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
20238
20648
|
}
|
|
@@ -20296,7 +20706,7 @@ async function strikingDistance(input, dependencies = defaultDependencies$7) {
|
|
|
20296
20706
|
brandTerms: input.brandTerms,
|
|
20297
20707
|
includeBrand: input.includeBrand
|
|
20298
20708
|
});
|
|
20299
|
-
const verifyLimit = boundedInteger$
|
|
20709
|
+
const verifyLimit = boundedInteger$3(input.verifyLimit, DEFAULT_VERIFY_LIMIT, 0, analysis.limit);
|
|
20300
20710
|
const verificationRequested = input.verifyContent === true || input.verifyContent !== false && input.verifyLimit !== void 0;
|
|
20301
20711
|
const attempted = verificationRequested ? Math.min(verifyLimit, analysis.items.length) : 0;
|
|
20302
20712
|
const items = await Promise.all(analysis.items.map(async (item, index) => {
|
|
@@ -20485,7 +20895,7 @@ function standardDeviation(values) {
|
|
|
20485
20895
|
function toFixedNumber(value, digits = 3) {
|
|
20486
20896
|
return Number(value.toFixed(digits));
|
|
20487
20897
|
}
|
|
20488
|
-
function percentChange(baseline, comparison) {
|
|
20898
|
+
function percentChange$1(baseline, comparison) {
|
|
20489
20899
|
if (!baseline) return comparison ? null : 0;
|
|
20490
20900
|
return (comparison - baseline) / baseline * 100;
|
|
20491
20901
|
}
|
|
@@ -20671,7 +21081,7 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
20671
21081
|
const outsideFlatBaseline = zScore === null && comparisonMean !== baselineMean;
|
|
20672
21082
|
const significant = outsideFlatBaseline || zScore !== null && Math.abs(zScore) >= 2;
|
|
20673
21083
|
const direction = significant ? comparisonMean < baselineMean ? "drop" : "spike" : "normal";
|
|
20674
|
-
const relativeChange = percentChange(baselineMean, comparisonMean);
|
|
21084
|
+
const relativeChange = percentChange$1(baselineMean, comparisonMean);
|
|
20675
21085
|
return {
|
|
20676
21086
|
site: input.site,
|
|
20677
21087
|
metric,
|
|
@@ -21730,7 +22140,7 @@ function dateShift(date, days) {
|
|
|
21730
22140
|
}
|
|
21731
22141
|
//#endregion
|
|
21732
22142
|
//#region packages/core/src/analyze/experiments/measurement/window.ts
|
|
21733
|
-
const DAY_MS = 864e5;
|
|
22143
|
+
const DAY_MS$1 = 864e5;
|
|
21734
22144
|
function exactDate(value, label) {
|
|
21735
22145
|
const pattern = /^\d{4}-\d{2}-\d{2}$/;
|
|
21736
22146
|
const parsed = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -21745,7 +22155,7 @@ function boundedDays(value, fallback, label) {
|
|
|
21745
22155
|
function calendarDays(startDate, endDate) {
|
|
21746
22156
|
const start = (/* @__PURE__ */ new Date(`${startDate}T00:00:00.000Z`)).getTime();
|
|
21747
22157
|
const end = (/* @__PURE__ */ new Date(`${endDate}T00:00:00.000Z`)).getTime();
|
|
21748
|
-
return Math.floor((end - start) / DAY_MS) + 1;
|
|
22158
|
+
return Math.floor((end - start) / DAY_MS$1) + 1;
|
|
21749
22159
|
}
|
|
21750
22160
|
function subtractCalendarMonths(value, months) {
|
|
21751
22161
|
const source = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -22293,7 +22703,7 @@ function internalLinksVerdict(input) {
|
|
|
22293
22703
|
function uniqueInternalLinksWarnings(warnings) {
|
|
22294
22704
|
return [...new Map(warnings.map((warning) => [`${warning.stage}\u0000${warning.url}\u0000${warning.code}\u0000${warning.message}`, warning])).values()];
|
|
22295
22705
|
}
|
|
22296
|
-
function errorMessage(error) {
|
|
22706
|
+
function errorMessage$1(error) {
|
|
22297
22707
|
return error instanceof Error ? error.message : String(error);
|
|
22298
22708
|
}
|
|
22299
22709
|
function acceptedAlias(site, value) {
|
|
@@ -22343,7 +22753,7 @@ async function verifyInternalLinkTarget(input) {
|
|
|
22343
22753
|
stage: "target-fetch",
|
|
22344
22754
|
url: input.targetUrl,
|
|
22345
22755
|
code: "fetch-failed",
|
|
22346
|
-
message: errorMessage(error)
|
|
22756
|
+
message: errorMessage$1(error)
|
|
22347
22757
|
}]
|
|
22348
22758
|
};
|
|
22349
22759
|
}
|
|
@@ -22364,7 +22774,7 @@ async function verifyInternalLinkTarget(input) {
|
|
|
22364
22774
|
stage: "target-extract",
|
|
22365
22775
|
url: input.targetUrl,
|
|
22366
22776
|
code: "extract-failed",
|
|
22367
|
-
message: errorMessage(error)
|
|
22777
|
+
message: errorMessage$1(error)
|
|
22368
22778
|
}]
|
|
22369
22779
|
};
|
|
22370
22780
|
}
|
|
@@ -22427,7 +22837,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
22427
22837
|
stage: "source-fetch",
|
|
22428
22838
|
url: input.candidate.sourceUrl,
|
|
22429
22839
|
code: "fetch-failed",
|
|
22430
|
-
message: errorMessage(error)
|
|
22840
|
+
message: errorMessage$1(error)
|
|
22431
22841
|
}]
|
|
22432
22842
|
};
|
|
22433
22843
|
}
|
|
@@ -22445,7 +22855,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
22445
22855
|
stage: "source-fetch",
|
|
22446
22856
|
url: input.candidate.sourceUrl,
|
|
22447
22857
|
code: "fetch-failed",
|
|
22448
|
-
message: errorMessage(error)
|
|
22858
|
+
message: errorMessage$1(error)
|
|
22449
22859
|
}]
|
|
22450
22860
|
};
|
|
22451
22861
|
}
|
|
@@ -22459,7 +22869,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
22459
22869
|
stage: "source-extract",
|
|
22460
22870
|
url: input.candidate.sourceUrl,
|
|
22461
22871
|
code: "extract-failed",
|
|
22462
|
-
message: errorMessage(error)
|
|
22872
|
+
message: errorMessage$1(error)
|
|
22463
22873
|
}]
|
|
22464
22874
|
};
|
|
22465
22875
|
}
|
|
@@ -23270,7 +23680,7 @@ function compareText$3(left, right) {
|
|
|
23270
23680
|
}
|
|
23271
23681
|
return leftPoints.length - rightPoints.length;
|
|
23272
23682
|
}
|
|
23273
|
-
function boundedInteger$
|
|
23683
|
+
function boundedInteger$2(value, fallback, maximum) {
|
|
23274
23684
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
23275
23685
|
return Math.min(maximum, Math.max(1, Math.trunc(value)));
|
|
23276
23686
|
}
|
|
@@ -23463,10 +23873,10 @@ function caveats(input) {
|
|
|
23463
23873
|
return items;
|
|
23464
23874
|
}
|
|
23465
23875
|
function analyzeIndexCoverageSignals(input) {
|
|
23466
|
-
const itemLimit = boundedInteger$
|
|
23467
|
-
const clusterLimit = boundedInteger$
|
|
23468
|
-
const clusterSamples = boundedInteger$
|
|
23469
|
-
const minUrls = boundedInteger$
|
|
23876
|
+
const itemLimit = boundedInteger$2(input.limits?.itemsPerSection, DEFAULT_ITEMS_PER_SECTION, MAX_ITEMS_PER_SECTION);
|
|
23877
|
+
const clusterLimit = boundedInteger$2(input.limits?.templateClusters, DEFAULT_TEMPLATE_LIMIT, MAX_TEMPLATE_LIMIT);
|
|
23878
|
+
const clusterSamples = boundedInteger$2(input.limits?.templateSamples, DEFAULT_TEMPLATE_SAMPLES, MAX_TEMPLATE_SAMPLES);
|
|
23879
|
+
const minUrls = boundedInteger$2(input.templateThresholds?.minUrls, DEFAULT_TEMPLATE_MIN_URLS, Number.MAX_SAFE_INTEGER);
|
|
23470
23880
|
const requestedMinShare = input.templateThresholds?.minShare;
|
|
23471
23881
|
const minShare = Number.isFinite(requestedMinShare) ? Math.min(1, Math.max(0, requestedMinShare ?? DEFAULT_TEMPLATE_MIN_SHARE)) : DEFAULT_TEMPLATE_MIN_SHARE;
|
|
23472
23882
|
const crawl = crawlInput(input.crawl);
|
|
@@ -27241,16 +27651,16 @@ function benchmarkDetails(benchmark) {
|
|
|
27241
27651
|
positiveUrlSamples: benchmark.positiveUrlSamples
|
|
27242
27652
|
};
|
|
27243
27653
|
}
|
|
27244
|
-
function boundedInteger(input) {
|
|
27654
|
+
function boundedInteger$1(input) {
|
|
27245
27655
|
const value = Number.isFinite(input.value) ? input.value ?? 0 : input.fallback;
|
|
27246
27656
|
return Math.min(input.maximum ?? Number.POSITIVE_INFINITY, Math.max(1, Math.floor(value)));
|
|
27247
27657
|
}
|
|
27248
27658
|
function analyzeQueryClustersFromRows(input) {
|
|
27249
|
-
const minImpressions = boundedInteger({
|
|
27659
|
+
const minImpressions = boundedInteger$1({
|
|
27250
27660
|
value: input.minImpressions,
|
|
27251
27661
|
fallback: 25
|
|
27252
27662
|
});
|
|
27253
|
-
const limit = boundedInteger({
|
|
27663
|
+
const limit = boundedInteger$1({
|
|
27254
27664
|
value: input.limit,
|
|
27255
27665
|
fallback: 25,
|
|
27256
27666
|
maximum: 100
|
|
@@ -27892,8 +28302,8 @@ function reportPresentation(report) {
|
|
|
27892
28302
|
//#region packages/core/src/analyze/second-page-analysis.ts
|
|
27893
28303
|
const DEFAULT_MIN_IMPRESSIONS = 50;
|
|
27894
28304
|
const MAX_MIN_IMPRESSIONS = 1e9;
|
|
27895
|
-
const DEFAULT_LIMIT = 10;
|
|
27896
|
-
const MAX_LIMIT = 100;
|
|
28305
|
+
const DEFAULT_LIMIT$1 = 10;
|
|
28306
|
+
const MAX_LIMIT$1 = 100;
|
|
27897
28307
|
function integerInRange(value, fallback, minimum, maximum) {
|
|
27898
28308
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
27899
28309
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
@@ -27980,7 +28390,7 @@ function compareItems(left, right) {
|
|
|
27980
28390
|
}
|
|
27981
28391
|
function analyzeSecondPageRows(input) {
|
|
27982
28392
|
const minImpressions = integerInRange(input.minImpressions, DEFAULT_MIN_IMPRESSIONS, 0, MAX_MIN_IMPRESSIONS);
|
|
27983
|
-
const limit = integerInRange(input.limit, DEFAULT_LIMIT, 1, MAX_LIMIT);
|
|
28393
|
+
const limit = integerInRange(input.limit, DEFAULT_LIMIT$1, 1, MAX_LIMIT$1);
|
|
27984
28394
|
const selection = {
|
|
27985
28395
|
sourceRows: input.rows.length,
|
|
27986
28396
|
invalidRows: 0,
|
|
@@ -29515,15 +29925,911 @@ async function updatePostmortemWorkflow(input, dependencies = {}) {
|
|
|
29515
29925
|
});
|
|
29516
29926
|
}
|
|
29517
29927
|
//#endregion
|
|
29518
|
-
//#region packages/core/src/
|
|
29519
|
-
|
|
29520
|
-
|
|
29521
|
-
|
|
29522
|
-
|
|
29523
|
-
|
|
29928
|
+
//#region packages/core/src/bing/analysis.ts
|
|
29929
|
+
const DAY_MS = 864e5;
|
|
29930
|
+
const DEFAULT_COMPARISON_DAYS = 28;
|
|
29931
|
+
const WEEKLY_PERIODS = 4;
|
|
29932
|
+
const OPPORTUNITY_MIN_IMPRESSIONS = 20;
|
|
29933
|
+
const OPPORTUNITY_MIN_POSITION = 4;
|
|
29934
|
+
const OPPORTUNITY_MAX_POSITION = 20;
|
|
29935
|
+
const RESULT_LIMIT = 10;
|
|
29936
|
+
function dateOffset(date, days) {
|
|
29937
|
+
return new Date(Date.parse(`${date}T00:00:00Z`) + days * DAY_MS).toISOString().slice(0, 10);
|
|
29938
|
+
}
|
|
29939
|
+
function percentChange(previous, current) {
|
|
29940
|
+
if (previous === 0) return null;
|
|
29941
|
+
return round((current - previous) / previous * 100);
|
|
29942
|
+
}
|
|
29943
|
+
function round(value, digits = 3) {
|
|
29944
|
+
const factor = 10 ** digits;
|
|
29945
|
+
return Math.round(value * factor) / factor;
|
|
29946
|
+
}
|
|
29947
|
+
function trafficPeriod(byDate, startDate, endDate, expectedDays) {
|
|
29948
|
+
let clicks = 0;
|
|
29949
|
+
let impressions = 0;
|
|
29950
|
+
let observedDays = 0;
|
|
29951
|
+
for (const [date, row] of byDate) {
|
|
29952
|
+
if (date < startDate || date > endDate) continue;
|
|
29953
|
+
observedDays += 1;
|
|
29954
|
+
clicks += row.clicks;
|
|
29955
|
+
impressions += row.impressions;
|
|
29956
|
+
}
|
|
29957
|
+
return {
|
|
29958
|
+
startDate,
|
|
29959
|
+
endDate,
|
|
29960
|
+
expectedDays,
|
|
29961
|
+
observedDays,
|
|
29962
|
+
missingDays: Math.max(0, expectedDays - observedDays),
|
|
29963
|
+
clicks,
|
|
29964
|
+
impressions,
|
|
29965
|
+
ctr: impressions ? round(clicks / impressions * 100) : null
|
|
29966
|
+
};
|
|
29524
29967
|
}
|
|
29525
|
-
function
|
|
29526
|
-
|
|
29968
|
+
function analyzeBingTraffic(rows, comparisonDays = DEFAULT_COMPARISON_DAYS) {
|
|
29969
|
+
if (!rows.length) return void 0;
|
|
29970
|
+
const byDate = /* @__PURE__ */ new Map();
|
|
29971
|
+
for (const row of rows) {
|
|
29972
|
+
const existing = byDate.get(row.date) ?? {
|
|
29973
|
+
clicks: 0,
|
|
29974
|
+
impressions: 0
|
|
29975
|
+
};
|
|
29976
|
+
existing.clicks += row.clicks;
|
|
29977
|
+
existing.impressions += row.impressions;
|
|
29978
|
+
byDate.set(row.date, existing);
|
|
29979
|
+
}
|
|
29980
|
+
const latestDate = [...byDate.keys()].sort().at(-1);
|
|
29981
|
+
if (!latestDate) return void 0;
|
|
29982
|
+
const currentStart = dateOffset(latestDate, -(comparisonDays - 1));
|
|
29983
|
+
const previousEnd = dateOffset(currentStart, -1);
|
|
29984
|
+
const previousStart = dateOffset(previousEnd, -(comparisonDays - 1));
|
|
29985
|
+
const current = trafficPeriod(byDate, currentStart, latestDate, comparisonDays);
|
|
29986
|
+
const previous = trafficPeriod(byDate, previousStart, previousEnd, comparisonDays);
|
|
29987
|
+
return {
|
|
29988
|
+
comparisonDays,
|
|
29989
|
+
current,
|
|
29990
|
+
previous,
|
|
29991
|
+
changes: {
|
|
29992
|
+
clicks: current.clicks - previous.clicks,
|
|
29993
|
+
clicksPercent: percentChange(previous.clicks, current.clicks),
|
|
29994
|
+
impressions: current.impressions - previous.impressions,
|
|
29995
|
+
impressionsPercent: percentChange(previous.impressions, current.impressions),
|
|
29996
|
+
ctrPercentagePoints: current.ctr === null || previous.ctr === null ? null : round(current.ctr - previous.ctr)
|
|
29997
|
+
}
|
|
29998
|
+
};
|
|
29999
|
+
}
|
|
30000
|
+
const CRAWL_FIELDS = [
|
|
30001
|
+
"crawledPages",
|
|
30002
|
+
"inIndex",
|
|
30003
|
+
"inLinks",
|
|
30004
|
+
"crawlErrors",
|
|
30005
|
+
"code4xx",
|
|
30006
|
+
"code5xx",
|
|
30007
|
+
"blockedByRobotsTxt",
|
|
30008
|
+
"connectionTimeout",
|
|
30009
|
+
"dnsFailures",
|
|
30010
|
+
"containsMalware"
|
|
30011
|
+
];
|
|
30012
|
+
function analyzeBingCrawl(rows, comparisonDays = DEFAULT_COMPARISON_DAYS) {
|
|
30013
|
+
const ordered = [...rows].sort((a, b) => a.date.localeCompare(b.date, "en"));
|
|
30014
|
+
const current = ordered.at(-1);
|
|
30015
|
+
if (!current) return void 0;
|
|
30016
|
+
const targetDate = dateOffset(current.date, -comparisonDays);
|
|
30017
|
+
const previous = ordered.filter((row) => row.date <= targetDate).at(-1);
|
|
30018
|
+
const changes = {};
|
|
30019
|
+
if (!previous) return {
|
|
30020
|
+
comparisonDays,
|
|
30021
|
+
targetDate,
|
|
30022
|
+
current,
|
|
30023
|
+
previous: void 0,
|
|
30024
|
+
changes
|
|
30025
|
+
};
|
|
30026
|
+
for (const field of CRAWL_FIELDS) {
|
|
30027
|
+
const currentValue = current[field];
|
|
30028
|
+
const previousValue = previous[field];
|
|
30029
|
+
if (typeof currentValue !== "number" || typeof previousValue !== "number") continue;
|
|
30030
|
+
changes[field] = {
|
|
30031
|
+
previous: previousValue,
|
|
30032
|
+
current: currentValue,
|
|
30033
|
+
absolute: currentValue - previousValue,
|
|
30034
|
+
percent: percentChange(previousValue, currentValue)
|
|
30035
|
+
};
|
|
30036
|
+
}
|
|
30037
|
+
return {
|
|
30038
|
+
comparisonDays,
|
|
30039
|
+
targetDate,
|
|
30040
|
+
current,
|
|
30041
|
+
previous,
|
|
30042
|
+
changes
|
|
30043
|
+
};
|
|
30044
|
+
}
|
|
30045
|
+
function aggregateDimensions(rows, dates) {
|
|
30046
|
+
const values = /* @__PURE__ */ new Map();
|
|
30047
|
+
for (const row of rows) {
|
|
30048
|
+
if (!dates.has(row.date)) continue;
|
|
30049
|
+
const item = values.get(row.value) ?? {
|
|
30050
|
+
clicks: 0,
|
|
30051
|
+
impressions: 0,
|
|
30052
|
+
clickPositionTotal: 0,
|
|
30053
|
+
clickPositionWeight: 0,
|
|
30054
|
+
impressionPositionTotal: 0,
|
|
30055
|
+
impressionPositionWeight: 0,
|
|
30056
|
+
observedDates: /* @__PURE__ */ new Set()
|
|
30057
|
+
};
|
|
30058
|
+
item.clicks += row.clicks;
|
|
30059
|
+
item.impressions += row.impressions;
|
|
30060
|
+
item.observedDates.add(row.date);
|
|
30061
|
+
if (row.avgClickPosition !== void 0 && row.clicks > 0) {
|
|
30062
|
+
const weight = row.clicks;
|
|
30063
|
+
item.clickPositionTotal += row.avgClickPosition * weight;
|
|
30064
|
+
item.clickPositionWeight += weight;
|
|
30065
|
+
}
|
|
30066
|
+
if (row.avgImpressionPosition !== void 0 && row.impressions > 0) {
|
|
30067
|
+
const weight = row.impressions;
|
|
30068
|
+
item.impressionPositionTotal += row.avgImpressionPosition * weight;
|
|
30069
|
+
item.impressionPositionWeight += weight;
|
|
30070
|
+
}
|
|
30071
|
+
values.set(row.value, item);
|
|
30072
|
+
}
|
|
30073
|
+
return new Map([...values].map(([value, item]) => [value, {
|
|
30074
|
+
value,
|
|
30075
|
+
observedWeeks: item.observedDates.size,
|
|
30076
|
+
clicks: item.clicks,
|
|
30077
|
+
impressions: item.impressions,
|
|
30078
|
+
avgClickPosition: item.clickPositionWeight ? round(item.clickPositionTotal / item.clickPositionWeight) : null,
|
|
30079
|
+
avgImpressionPosition: item.impressionPositionWeight ? round(item.impressionPositionTotal / item.impressionPositionWeight) : null
|
|
30080
|
+
}]));
|
|
30081
|
+
}
|
|
30082
|
+
function compareValue(a, b) {
|
|
30083
|
+
return a.value.localeCompare(b.value, "en");
|
|
30084
|
+
}
|
|
30085
|
+
function analyzeBingDimensions(rows, kind) {
|
|
30086
|
+
const dates = [...new Set(rows.map((row) => row.date))].sort().reverse();
|
|
30087
|
+
const currentDates = dates.slice(0, WEEKLY_PERIODS);
|
|
30088
|
+
const previousDates = dates.slice(WEEKLY_PERIODS, WEEKLY_PERIODS * 2);
|
|
30089
|
+
const current = aggregateDimensions(rows, new Set(currentDates));
|
|
30090
|
+
const previous = aggregateDimensions(rows, new Set(previousDates));
|
|
30091
|
+
const matchedValues = [...current.keys()].filter((value) => previous.has(value));
|
|
30092
|
+
const comparableValues = currentDates.length === WEEKLY_PERIODS && previousDates.length === WEEKLY_PERIODS ? matchedValues.filter((value) => {
|
|
30093
|
+
const currentValue = current.get(value);
|
|
30094
|
+
const previousValue = previous.get(value);
|
|
30095
|
+
return currentValue?.observedWeeks === WEEKLY_PERIODS && previousValue?.observedWeeks === WEEKLY_PERIODS;
|
|
30096
|
+
}) : [];
|
|
30097
|
+
const movements = comparableValues.map((value) => {
|
|
30098
|
+
const currentValue = current.get(value);
|
|
30099
|
+
const previousValue = previous.get(value);
|
|
30100
|
+
return {
|
|
30101
|
+
value,
|
|
30102
|
+
current: currentValue,
|
|
30103
|
+
previous: previousValue,
|
|
30104
|
+
changes: {
|
|
30105
|
+
clicks: currentValue.clicks - previousValue.clicks,
|
|
30106
|
+
clicksPercent: percentChange(previousValue.clicks, currentValue.clicks),
|
|
30107
|
+
impressions: currentValue.impressions - previousValue.impressions,
|
|
30108
|
+
impressionsPercent: percentChange(previousValue.impressions, currentValue.impressions)
|
|
30109
|
+
}
|
|
30110
|
+
};
|
|
30111
|
+
}).sort((a, b) => Math.abs(b.changes.clicks) - Math.abs(a.changes.clicks) || Math.abs(b.changes.impressions) - Math.abs(a.changes.impressions) || compareValue(a, b)).slice(0, RESULT_LIMIT);
|
|
30112
|
+
const opportunities = [...current.values()].filter((row) => row.impressions >= OPPORTUNITY_MIN_IMPRESSIONS && row.avgImpressionPosition !== null && row.avgImpressionPosition >= OPPORTUNITY_MIN_POSITION && row.avgImpressionPosition <= OPPORTUNITY_MAX_POSITION).sort((a, b) => b.impressions - a.impressions || b.clicks - a.clicks || compareValue(a, b)).slice(0, RESULT_LIMIT);
|
|
30113
|
+
return {
|
|
30114
|
+
kind,
|
|
30115
|
+
sourceSemantics: "Bing weekly top-list evidence. Missing dimensions are unknown, not zero. Movements require observation in every week of both periods.",
|
|
30116
|
+
periods: {
|
|
30117
|
+
weeksPerPeriod: WEEKLY_PERIODS,
|
|
30118
|
+
currentDates,
|
|
30119
|
+
previousDates
|
|
30120
|
+
},
|
|
30121
|
+
coverage: {
|
|
30122
|
+
sourceRows: rows.length,
|
|
30123
|
+
currentDimensions: current.size,
|
|
30124
|
+
previousDimensions: previous.size,
|
|
30125
|
+
matchedDimensions: matchedValues.length,
|
|
30126
|
+
comparableDimensions: comparableValues.length,
|
|
30127
|
+
incompleteMatchedDimensions: matchedValues.length - comparableValues.length,
|
|
30128
|
+
currentOnlyDimensions: [...current.keys()].filter((value) => !previous.has(value)).length,
|
|
30129
|
+
previousOnlyDimensions: [...previous.keys()].filter((value) => !current.has(value)).length
|
|
30130
|
+
},
|
|
30131
|
+
thresholds: {
|
|
30132
|
+
heuristic: true,
|
|
30133
|
+
opportunityMinImpressions: OPPORTUNITY_MIN_IMPRESSIONS,
|
|
30134
|
+
opportunityPositionRange: [OPPORTUNITY_MIN_POSITION, OPPORTUNITY_MAX_POSITION],
|
|
30135
|
+
resultLimit: RESULT_LIMIT
|
|
30136
|
+
},
|
|
30137
|
+
movements,
|
|
30138
|
+
opportunities
|
|
30139
|
+
};
|
|
30140
|
+
}
|
|
30141
|
+
const bingAnalysisDefaults = {
|
|
30142
|
+
comparisonDays: DEFAULT_COMPARISON_DAYS,
|
|
30143
|
+
weeklyPeriods: WEEKLY_PERIODS,
|
|
30144
|
+
resultLimit: RESULT_LIMIT
|
|
30145
|
+
};
|
|
30146
|
+
//#endregion
|
|
30147
|
+
//#region packages/core/src/bing/client.ts
|
|
30148
|
+
const DEFAULT_BASE_URL = "https://ssl.bing.com/webmaster/api.svc/json";
|
|
30149
|
+
const MAX_RESPONSE_BYTES$1 = 2e6;
|
|
30150
|
+
const MAX_STAT_ROWS = 400;
|
|
30151
|
+
const MAX_DIMENSION_ROWS = 8e3;
|
|
30152
|
+
const MAX_LINK_ROWS = 200;
|
|
30153
|
+
function bingDate(value) {
|
|
30154
|
+
if (typeof value !== "string") return void 0;
|
|
30155
|
+
const legacy = /^\/Date\((-?\d+)(?:[+-]\d{4})?\)\/$/.exec(value);
|
|
30156
|
+
const date = legacy ? new Date(Number(legacy[1])) : new Date(value);
|
|
30157
|
+
return Number.isNaN(date.getTime()) ? void 0 : date.toISOString().slice(0, 10);
|
|
30158
|
+
}
|
|
30159
|
+
function nonnegativeNumber(value) {
|
|
30160
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
|
|
30161
|
+
}
|
|
30162
|
+
function positionNumber(value) {
|
|
30163
|
+
if (value === void 0 || value === null || value === -1) return { valid: true };
|
|
30164
|
+
const number = nonnegativeNumber(value);
|
|
30165
|
+
return number === void 0 ? { valid: false } : {
|
|
30166
|
+
valid: true,
|
|
30167
|
+
value: number
|
|
30168
|
+
};
|
|
30169
|
+
}
|
|
30170
|
+
function isHttpUrl(value) {
|
|
30171
|
+
if (typeof value !== "string" || value.length > 2e3) return false;
|
|
30172
|
+
try {
|
|
30173
|
+
const url = new URL(value);
|
|
30174
|
+
return url.protocol === "http:" || url.protocol === "https:";
|
|
30175
|
+
} catch {
|
|
30176
|
+
return false;
|
|
30177
|
+
}
|
|
30178
|
+
}
|
|
30179
|
+
function boundedRows(rows, invalidRows, limit = MAX_STAT_ROWS) {
|
|
30180
|
+
const capped = rows.length > limit;
|
|
30181
|
+
return {
|
|
30182
|
+
rows: capped ? rows.slice(-limit) : rows,
|
|
30183
|
+
invalidRows,
|
|
30184
|
+
capped,
|
|
30185
|
+
returnedRows: rows.length
|
|
30186
|
+
};
|
|
30187
|
+
}
|
|
30188
|
+
function dimensionValue(value, kind) {
|
|
30189
|
+
if (typeof value !== "string") return void 0;
|
|
30190
|
+
const normalized = value.trim();
|
|
30191
|
+
if (!normalized || normalized.length > 2e3) return void 0;
|
|
30192
|
+
if (kind === "page" && !isHttpUrl(normalized)) return void 0;
|
|
30193
|
+
return normalized;
|
|
30194
|
+
}
|
|
30195
|
+
function errorMessage(body) {
|
|
30196
|
+
try {
|
|
30197
|
+
const parsed = JSON.parse(body);
|
|
30198
|
+
return typeof parsed.Message === "string" ? parsed.Message : void 0;
|
|
30199
|
+
} catch {
|
|
30200
|
+
return;
|
|
30201
|
+
}
|
|
30202
|
+
}
|
|
30203
|
+
var BingWebmasterClient = class {
|
|
30204
|
+
credentials;
|
|
30205
|
+
authentication;
|
|
30206
|
+
baseUrl;
|
|
30207
|
+
fetchImpl;
|
|
30208
|
+
timeoutMs;
|
|
30209
|
+
constructor(credentials, options = {}) {
|
|
30210
|
+
this.credentials = credentials;
|
|
30211
|
+
if (!(credentials.apiKey?.trim() || credentials.accessToken?.trim())) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster credentials are empty.");
|
|
30212
|
+
this.authentication = credentials.apiKey ? "api-key" : "oauth";
|
|
30213
|
+
this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, "");
|
|
30214
|
+
this.fetchImpl = options.fetchImpl ?? publicHttpFetch;
|
|
30215
|
+
this.timeoutMs = options.timeoutMs ?? 3e4;
|
|
30216
|
+
}
|
|
30217
|
+
async request(method, params = {}) {
|
|
30218
|
+
const url = new URL(`${this.baseUrl}/${method}`);
|
|
30219
|
+
for (const [key, value] of Object.entries(params)) url.searchParams.set(key, value);
|
|
30220
|
+
if (this.credentials.apiKey) url.searchParams.set("apikey", this.credentials.apiKey);
|
|
30221
|
+
let response;
|
|
30222
|
+
try {
|
|
30223
|
+
response = await this.fetchImpl(url.toString(), {
|
|
30224
|
+
headers: {
|
|
30225
|
+
accept: "application/json",
|
|
30226
|
+
...this.credentials.accessToken ? { authorization: `Bearer ${this.credentials.accessToken}` } : {}
|
|
30227
|
+
},
|
|
30228
|
+
signal: AbortSignal.timeout(this.timeoutMs)
|
|
30229
|
+
});
|
|
30230
|
+
} catch (error) {
|
|
30231
|
+
if (error instanceof SeoError) throw error;
|
|
30232
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster did not respond: ${error instanceof Error ? error.message : String(error)}`);
|
|
30233
|
+
}
|
|
30234
|
+
const body = await readBoundedResponseText(response, MAX_RESPONSE_BYTES$1, "Bing Webmaster response");
|
|
30235
|
+
if (!response.ok) {
|
|
30236
|
+
const message = errorMessage(body);
|
|
30237
|
+
if (response.status === 401 || /invalidapikey|unauthorized/i.test(message ?? "")) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster rejected the saved credential. Run `seo providers bing connect` again.");
|
|
30238
|
+
if (response.status === 403) throw new SeoError("ACCESS_DENIED", "Bing Webmaster denied access to this site. Check that it is verified for the connected account.");
|
|
30239
|
+
if (response.status === 429) throw new SeoError("RATE_LIMITED", "Bing Webmaster rate limited this request. Try again later.");
|
|
30240
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster request failed with HTTP ${response.status}${message ? ` (${message})` : ""}.`);
|
|
30241
|
+
}
|
|
30242
|
+
try {
|
|
30243
|
+
return JSON.parse(body).d;
|
|
30244
|
+
} catch {
|
|
30245
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid JSON.");
|
|
30246
|
+
}
|
|
30247
|
+
}
|
|
30248
|
+
async listSites() {
|
|
30249
|
+
const data = await this.request("GetUserSites");
|
|
30250
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned an invalid site list.");
|
|
30251
|
+
const sites = [];
|
|
30252
|
+
let invalidRows = 0;
|
|
30253
|
+
for (const item of data) {
|
|
30254
|
+
if (!item || typeof item !== "object") {
|
|
30255
|
+
invalidRows += 1;
|
|
30256
|
+
continue;
|
|
30257
|
+
}
|
|
30258
|
+
const row = item;
|
|
30259
|
+
if (typeof row.Url !== "string" || typeof row.IsVerified !== "boolean") {
|
|
30260
|
+
invalidRows += 1;
|
|
30261
|
+
continue;
|
|
30262
|
+
}
|
|
30263
|
+
sites.push({
|
|
30264
|
+
url: row.Url,
|
|
30265
|
+
isVerified: row.IsVerified
|
|
30266
|
+
});
|
|
30267
|
+
}
|
|
30268
|
+
sites.sort((a, b) => a.url.localeCompare(b.url, "en"));
|
|
30269
|
+
return {
|
|
30270
|
+
sites,
|
|
30271
|
+
invalidRows
|
|
30272
|
+
};
|
|
30273
|
+
}
|
|
30274
|
+
async getTraffic(siteUrl) {
|
|
30275
|
+
const data = await this.request("GetRankAndTrafficStats", { siteUrl });
|
|
30276
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid traffic statistics.");
|
|
30277
|
+
const rows = [];
|
|
30278
|
+
let invalidRows = 0;
|
|
30279
|
+
for (const item of data) {
|
|
30280
|
+
if (!item || typeof item !== "object") {
|
|
30281
|
+
invalidRows += 1;
|
|
30282
|
+
continue;
|
|
30283
|
+
}
|
|
30284
|
+
const row = item;
|
|
30285
|
+
const date = bingDate(row.Date);
|
|
30286
|
+
const clicks = nonnegativeNumber(row.Clicks);
|
|
30287
|
+
const impressions = nonnegativeNumber(row.Impressions);
|
|
30288
|
+
if (!date || clicks === void 0 || impressions === void 0) {
|
|
30289
|
+
invalidRows += 1;
|
|
30290
|
+
continue;
|
|
30291
|
+
}
|
|
30292
|
+
rows.push({
|
|
30293
|
+
date,
|
|
30294
|
+
clicks,
|
|
30295
|
+
impressions
|
|
30296
|
+
});
|
|
30297
|
+
}
|
|
30298
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en") || a.clicks - b.clicks || a.impressions - b.impressions);
|
|
30299
|
+
return boundedRows(rows, invalidRows);
|
|
30300
|
+
}
|
|
30301
|
+
async getCrawlStats(siteUrl) {
|
|
30302
|
+
const data = await this.request("GetCrawlStats", { siteUrl });
|
|
30303
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid crawl statistics.");
|
|
30304
|
+
const fieldMap = {
|
|
30305
|
+
AllOtherCodes: "allOtherCodes",
|
|
30306
|
+
BlockedByRobotsTxt: "blockedByRobotsTxt",
|
|
30307
|
+
Code2xx: "code2xx",
|
|
30308
|
+
Code301: "code301",
|
|
30309
|
+
Code302: "code302",
|
|
30310
|
+
Code4xx: "code4xx",
|
|
30311
|
+
Code5xx: "code5xx",
|
|
30312
|
+
ConnectionTimeout: "connectionTimeout",
|
|
30313
|
+
ContainsMalware: "containsMalware",
|
|
30314
|
+
CrawledPages: "crawledPages",
|
|
30315
|
+
CrawlErrors: "crawlErrors",
|
|
30316
|
+
DnsFailures: "dnsFailures",
|
|
30317
|
+
InIndex: "inIndex",
|
|
30318
|
+
InLinks: "inLinks"
|
|
30319
|
+
};
|
|
30320
|
+
const rows = [];
|
|
30321
|
+
let invalidRows = 0;
|
|
30322
|
+
for (const item of data) {
|
|
30323
|
+
if (!item || typeof item !== "object") {
|
|
30324
|
+
invalidRows += 1;
|
|
30325
|
+
continue;
|
|
30326
|
+
}
|
|
30327
|
+
const source = item;
|
|
30328
|
+
const date = bingDate(source.Date);
|
|
30329
|
+
if (!date) {
|
|
30330
|
+
invalidRows += 1;
|
|
30331
|
+
continue;
|
|
30332
|
+
}
|
|
30333
|
+
const row = { date };
|
|
30334
|
+
let invalid = false;
|
|
30335
|
+
for (const [providerField, field] of Object.entries(fieldMap)) {
|
|
30336
|
+
const value = source[providerField];
|
|
30337
|
+
if (value === void 0 || value === null) continue;
|
|
30338
|
+
const number = nonnegativeNumber(value);
|
|
30339
|
+
if (number === void 0) {
|
|
30340
|
+
invalid = true;
|
|
30341
|
+
break;
|
|
30342
|
+
}
|
|
30343
|
+
row[field] = number;
|
|
30344
|
+
}
|
|
30345
|
+
if (invalid) {
|
|
30346
|
+
invalidRows += 1;
|
|
30347
|
+
continue;
|
|
30348
|
+
}
|
|
30349
|
+
rows.push(row);
|
|
30350
|
+
}
|
|
30351
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en"));
|
|
30352
|
+
return boundedRows(rows, invalidRows);
|
|
30353
|
+
}
|
|
30354
|
+
async getQueryStats(siteUrl) {
|
|
30355
|
+
return this.getDimensionStats("GetQueryStats", siteUrl, "query");
|
|
30356
|
+
}
|
|
30357
|
+
async getPageStats(siteUrl) {
|
|
30358
|
+
return this.getDimensionStats("GetPageStats", siteUrl, "page");
|
|
30359
|
+
}
|
|
30360
|
+
async getDimensionStats(method, siteUrl, kind) {
|
|
30361
|
+
const data = await this.request(method, { siteUrl });
|
|
30362
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster returned invalid ${kind} statistics.`);
|
|
30363
|
+
const rows = [];
|
|
30364
|
+
let invalidRows = 0;
|
|
30365
|
+
for (const item of data) {
|
|
30366
|
+
if (!item || typeof item !== "object") {
|
|
30367
|
+
invalidRows += 1;
|
|
30368
|
+
continue;
|
|
30369
|
+
}
|
|
30370
|
+
const source = item;
|
|
30371
|
+
const date = bingDate(source.Date);
|
|
30372
|
+
const value = dimensionValue(source.Query, kind);
|
|
30373
|
+
const clicks = nonnegativeNumber(source.Clicks);
|
|
30374
|
+
const impressions = nonnegativeNumber(source.Impressions);
|
|
30375
|
+
const clickPosition = positionNumber(source.AvgClickPosition);
|
|
30376
|
+
const impressionPosition = positionNumber(source.AvgImpressionPosition);
|
|
30377
|
+
if (!date || !value || clicks === void 0 || impressions === void 0 || !clickPosition.valid || !impressionPosition.valid) {
|
|
30378
|
+
invalidRows += 1;
|
|
30379
|
+
continue;
|
|
30380
|
+
}
|
|
30381
|
+
rows.push({
|
|
30382
|
+
date,
|
|
30383
|
+
value,
|
|
30384
|
+
clicks,
|
|
30385
|
+
impressions,
|
|
30386
|
+
avgClickPosition: clickPosition.value,
|
|
30387
|
+
avgImpressionPosition: impressionPosition.value
|
|
30388
|
+
});
|
|
30389
|
+
}
|
|
30390
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en") || a.value.localeCompare(b.value, "en"));
|
|
30391
|
+
return boundedRows(rows, invalidRows, MAX_DIMENSION_ROWS);
|
|
30392
|
+
}
|
|
30393
|
+
async getLinkCounts(siteUrl, page) {
|
|
30394
|
+
const data = await this.request("GetLinkCounts", {
|
|
30395
|
+
siteUrl,
|
|
30396
|
+
page: String(page)
|
|
30397
|
+
});
|
|
30398
|
+
if (!data || typeof data !== "object") throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link counts.");
|
|
30399
|
+
const response = data;
|
|
30400
|
+
if (!Array.isArray(response.Links)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link counts.");
|
|
30401
|
+
const rows = [];
|
|
30402
|
+
let invalidRows = 0;
|
|
30403
|
+
for (const item of response.Links) {
|
|
30404
|
+
if (!item || typeof item !== "object") {
|
|
30405
|
+
invalidRows += 1;
|
|
30406
|
+
continue;
|
|
30407
|
+
}
|
|
30408
|
+
const row = item;
|
|
30409
|
+
const count = nonnegativeNumber(row.Count);
|
|
30410
|
+
if (!isHttpUrl(row.Url) || count === void 0) {
|
|
30411
|
+
invalidRows += 1;
|
|
30412
|
+
continue;
|
|
30413
|
+
}
|
|
30414
|
+
rows.push({
|
|
30415
|
+
url: row.Url,
|
|
30416
|
+
count
|
|
30417
|
+
});
|
|
30418
|
+
}
|
|
30419
|
+
rows.sort((a, b) => b.count - a.count || a.url.localeCompare(b.url, "en"));
|
|
30420
|
+
return linkPage(rows, response.TotalPages, invalidRows);
|
|
30421
|
+
}
|
|
30422
|
+
async getUrlLinks(siteUrl, link, page) {
|
|
30423
|
+
const data = await this.request("GetUrlLinks", {
|
|
30424
|
+
siteUrl,
|
|
30425
|
+
link,
|
|
30426
|
+
page: String(page)
|
|
30427
|
+
});
|
|
30428
|
+
if (!data || typeof data !== "object") throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid referring links.");
|
|
30429
|
+
const response = data;
|
|
30430
|
+
if (!Array.isArray(response.Details)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid referring links.");
|
|
30431
|
+
const rows = [];
|
|
30432
|
+
let invalidRows = 0;
|
|
30433
|
+
for (const item of response.Details) {
|
|
30434
|
+
if (!item || typeof item !== "object") {
|
|
30435
|
+
invalidRows += 1;
|
|
30436
|
+
continue;
|
|
30437
|
+
}
|
|
30438
|
+
const row = item;
|
|
30439
|
+
if (!isHttpUrl(row.Url)) {
|
|
30440
|
+
invalidRows += 1;
|
|
30441
|
+
continue;
|
|
30442
|
+
}
|
|
30443
|
+
rows.push({
|
|
30444
|
+
url: row.Url,
|
|
30445
|
+
anchorText: typeof row.AnchorText === "string" && row.AnchorText.trim() ? row.AnchorText.trim() : void 0
|
|
30446
|
+
});
|
|
30447
|
+
}
|
|
30448
|
+
rows.sort((a, b) => a.url.localeCompare(b.url, "en") || (a.anchorText ?? "").localeCompare(b.anchorText ?? "", "en"));
|
|
30449
|
+
return linkPage(rows, response.TotalPages, invalidRows);
|
|
30450
|
+
}
|
|
30451
|
+
};
|
|
30452
|
+
function linkPage(rows, totalPagesValue, invalidRows) {
|
|
30453
|
+
const totalPages = nonnegativeNumber(totalPagesValue);
|
|
30454
|
+
if (totalPages === void 0 || !Number.isInteger(totalPages)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link pagination.");
|
|
30455
|
+
const capped = rows.length > MAX_LINK_ROWS;
|
|
30456
|
+
return {
|
|
30457
|
+
rows: capped ? rows.slice(0, MAX_LINK_ROWS) : rows,
|
|
30458
|
+
totalPages,
|
|
30459
|
+
invalidRows,
|
|
30460
|
+
capped,
|
|
30461
|
+
returnedRows: rows.length
|
|
30462
|
+
};
|
|
30463
|
+
}
|
|
30464
|
+
//#endregion
|
|
30465
|
+
//#region packages/core/src/storage/provider-secrets.ts
|
|
30466
|
+
const KEYRING_SERVICE = "seo";
|
|
30467
|
+
const PRIVATE_FILE_MODE = 384;
|
|
30468
|
+
function account(name) {
|
|
30469
|
+
return `provider:${name}`;
|
|
30470
|
+
}
|
|
30471
|
+
function readFile$1() {
|
|
30472
|
+
return readJsonFile(getSeoCliPaths().providerSecretsFile) ?? {
|
|
30473
|
+
version: 1,
|
|
30474
|
+
secrets: {}
|
|
30475
|
+
};
|
|
30476
|
+
}
|
|
30477
|
+
function writeFile$1(file) {
|
|
30478
|
+
const path = getSeoCliPaths().providerSecretsFile;
|
|
30479
|
+
if (Object.keys(file.secrets).length === 0) {
|
|
30480
|
+
safeRemove(path);
|
|
30481
|
+
return;
|
|
30482
|
+
}
|
|
30483
|
+
writeJsonAtomic(path, file, PRIVATE_FILE_MODE);
|
|
30484
|
+
}
|
|
30485
|
+
function deleteFileSecret(name) {
|
|
30486
|
+
const file = readFile$1();
|
|
30487
|
+
if (!(name in file.secrets)) return;
|
|
30488
|
+
delete file.secrets[name];
|
|
30489
|
+
writeFile$1(file);
|
|
30490
|
+
}
|
|
30491
|
+
async function readProviderSecret(input) {
|
|
30492
|
+
const environment = input.env ?? process.env;
|
|
30493
|
+
const environmentValue = input.envVar ? environment[input.envVar]?.trim() : void 0;
|
|
30494
|
+
if (environmentValue) return {
|
|
30495
|
+
value: environmentValue,
|
|
30496
|
+
source: "environment"
|
|
30497
|
+
};
|
|
30498
|
+
if (readConfig().security.useKeychain) try {
|
|
30499
|
+
const value = await getKeyringPassword(KEYRING_SERVICE, account(input.name));
|
|
30500
|
+
if (value) return {
|
|
30501
|
+
value,
|
|
30502
|
+
source: "keychain"
|
|
30503
|
+
};
|
|
30504
|
+
} catch {}
|
|
30505
|
+
const value = readFile$1().secrets[input.name];
|
|
30506
|
+
return value ? {
|
|
30507
|
+
value,
|
|
30508
|
+
source: "file"
|
|
30509
|
+
} : void 0;
|
|
30510
|
+
}
|
|
30511
|
+
async function writeProviderSecret(name, value) {
|
|
30512
|
+
const normalized = value.trim();
|
|
30513
|
+
if (!normalized) throw new Error("Provider secret cannot be empty.");
|
|
30514
|
+
if (readConfig().security.useKeychain) try {
|
|
30515
|
+
await setKeyringPassword(KEYRING_SERVICE, account(name), normalized);
|
|
30516
|
+
deleteFileSecret(name);
|
|
30517
|
+
return "keychain";
|
|
30518
|
+
} catch {}
|
|
30519
|
+
const file = readFile$1();
|
|
30520
|
+
file.secrets[name] = normalized;
|
|
30521
|
+
writeFile$1(file);
|
|
30522
|
+
return "file";
|
|
30523
|
+
}
|
|
30524
|
+
async function deleteProviderSecret(name) {
|
|
30525
|
+
await deleteKeyringPassword(KEYRING_SERVICE, account(name)).catch(() => false);
|
|
30526
|
+
deleteFileSecret(name);
|
|
30527
|
+
}
|
|
30528
|
+
//#endregion
|
|
30529
|
+
//#region packages/core/src/bing/credentials.ts
|
|
30530
|
+
const BING_API_KEY_SECRET = "bing-api-key";
|
|
30531
|
+
const BING_API_KEY_ENV = "SEO_BING_API_KEY";
|
|
30532
|
+
async function createBingWebmasterClient(input = {}) {
|
|
30533
|
+
const secret = await readProviderSecret({
|
|
30534
|
+
name: BING_API_KEY_SECRET,
|
|
30535
|
+
envVar: BING_API_KEY_ENV,
|
|
30536
|
+
env: input.env
|
|
30537
|
+
});
|
|
30538
|
+
if (!secret) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster is not connected. Run `seo providers bing connect`, or set SEO_BING_API_KEY for this process.");
|
|
30539
|
+
return {
|
|
30540
|
+
client: new BingWebmasterClient({ apiKey: secret.value }),
|
|
30541
|
+
credentialSource: secret.source
|
|
30542
|
+
};
|
|
30543
|
+
}
|
|
30544
|
+
//#endregion
|
|
30545
|
+
//#region packages/core/src/bing/overview.ts
|
|
30546
|
+
const DAILY_OUTPUT_ROWS = 14;
|
|
30547
|
+
function message(error) {
|
|
30548
|
+
return error instanceof Error ? error.message : String(error);
|
|
30549
|
+
}
|
|
30550
|
+
function range(rows) {
|
|
30551
|
+
return rows.length ? {
|
|
30552
|
+
startDate: rows[0]?.date ?? "",
|
|
30553
|
+
endDate: rows.at(-1)?.date ?? ""
|
|
30554
|
+
} : void 0;
|
|
30555
|
+
}
|
|
30556
|
+
function dailyOutput(data) {
|
|
30557
|
+
const rows = data.rows.slice(-14);
|
|
30558
|
+
return {
|
|
30559
|
+
...data,
|
|
30560
|
+
rows,
|
|
30561
|
+
retainedRows: data.rows.length,
|
|
30562
|
+
outputSelection: {
|
|
30563
|
+
strategy: "most-recent",
|
|
30564
|
+
availableRows: data.rows.length,
|
|
30565
|
+
returnedRows: rows.length,
|
|
30566
|
+
omittedRows: Math.max(0, data.rows.length - rows.length)
|
|
30567
|
+
}
|
|
30568
|
+
};
|
|
30569
|
+
}
|
|
30570
|
+
async function section(work, options) {
|
|
30571
|
+
try {
|
|
30572
|
+
const data = await work();
|
|
30573
|
+
const reasons = [
|
|
30574
|
+
...data.invalidRows ? [`${data.invalidRows} invalid provider rows were excluded.`] : [],
|
|
30575
|
+
...data.capped ? [`The provider response exceeded the ${options.rowLimit}-row limit.`] : [],
|
|
30576
|
+
...options.sourceWarning ? [options.sourceWarning] : []
|
|
30577
|
+
];
|
|
30578
|
+
return reasons.length ? {
|
|
30579
|
+
status: "partial",
|
|
30580
|
+
data,
|
|
30581
|
+
warning: reasons.join(" ")
|
|
30582
|
+
} : {
|
|
30583
|
+
status: "complete",
|
|
30584
|
+
data
|
|
30585
|
+
};
|
|
30586
|
+
} catch (error) {
|
|
30587
|
+
if (error instanceof SeoError && (error.code === "AUTH_REQUIRED" || error.code === "ACCESS_DENIED")) throw error;
|
|
30588
|
+
return {
|
|
30589
|
+
status: "unavailable",
|
|
30590
|
+
warning: message(error)
|
|
30591
|
+
};
|
|
30592
|
+
}
|
|
30593
|
+
}
|
|
30594
|
+
function changeFinding(input) {
|
|
30595
|
+
return {
|
|
30596
|
+
code: input.code,
|
|
30597
|
+
severity: "warning",
|
|
30598
|
+
title: input.title,
|
|
30599
|
+
evidence: {
|
|
30600
|
+
current: input.current,
|
|
30601
|
+
previous: input.previous,
|
|
30602
|
+
absoluteChange: input.absolute,
|
|
30603
|
+
percentChange: input.percent,
|
|
30604
|
+
currentDate: input.dates.current,
|
|
30605
|
+
previousDate: input.dates.previous
|
|
30606
|
+
},
|
|
30607
|
+
interpretation: input.interpretation,
|
|
30608
|
+
verification: input.verification
|
|
30609
|
+
};
|
|
30610
|
+
}
|
|
30611
|
+
function buildFindings(input) {
|
|
30612
|
+
const findings = [];
|
|
30613
|
+
const traffic = input.traffic;
|
|
30614
|
+
if (traffic && traffic.current.observedDays === traffic.current.expectedDays && traffic.previous.observedDays === traffic.previous.expectedDays && (traffic.changes.clicksPercent !== null && Math.abs(traffic.changes.clicksPercent) >= 10 || traffic.changes.impressionsPercent !== null && Math.abs(traffic.changes.impressionsPercent) >= 10)) {
|
|
30615
|
+
const metric = traffic.changes.clicksPercent !== null && Math.abs(traffic.changes.clicksPercent) >= 10 ? "clicks" : "impressions";
|
|
30616
|
+
const declined = traffic.changes[metric] < 0;
|
|
30617
|
+
findings.push({
|
|
30618
|
+
code: `bing_${metric}_${declined ? "declined" : "increased"}`,
|
|
30619
|
+
severity: declined ? "warning" : "info",
|
|
30620
|
+
title: `Bing ${metric} ${declined ? "declined" : "increased"} in the latest complete period`,
|
|
30621
|
+
evidence: {
|
|
30622
|
+
currentClicks: traffic.current.clicks,
|
|
30623
|
+
previousClicks: traffic.previous.clicks,
|
|
30624
|
+
clicksChange: traffic.changes.clicks,
|
|
30625
|
+
clicksPercentChange: traffic.changes.clicksPercent,
|
|
30626
|
+
currentImpressions: traffic.current.impressions,
|
|
30627
|
+
previousImpressions: traffic.previous.impressions,
|
|
30628
|
+
impressionsChange: traffic.changes.impressions,
|
|
30629
|
+
impressionsPercentChange: traffic.changes.impressionsPercent,
|
|
30630
|
+
ctrPercentagePointChange: traffic.changes.ctrPercentagePoints,
|
|
30631
|
+
currentRange: {
|
|
30632
|
+
startDate: traffic.current.startDate,
|
|
30633
|
+
endDate: traffic.current.endDate
|
|
30634
|
+
},
|
|
30635
|
+
previousRange: {
|
|
30636
|
+
startDate: traffic.previous.startDate,
|
|
30637
|
+
endDate: traffic.previous.endDate
|
|
30638
|
+
}
|
|
30639
|
+
},
|
|
30640
|
+
interpretation: "This is a Bing-observed traffic change. It does not establish the cause or describe Google traffic.",
|
|
30641
|
+
verification: "Review the query and page evidence for the same dates, then compare current pages and crawl health before changing content."
|
|
30642
|
+
});
|
|
30643
|
+
}
|
|
30644
|
+
const crawl = input.crawl;
|
|
30645
|
+
if (crawl?.previous) {
|
|
30646
|
+
const dates = {
|
|
30647
|
+
current: crawl.current.date,
|
|
30648
|
+
previous: crawl.previous.date
|
|
30649
|
+
};
|
|
30650
|
+
const errors = crawl.changes.crawlErrors;
|
|
30651
|
+
if (errors && errors.absolute >= Math.max(10, Math.ceil(errors.previous * .5))) findings.push(changeFinding({
|
|
30652
|
+
code: "bing_crawl_errors_increased",
|
|
30653
|
+
title: "Bing crawl errors increased",
|
|
30654
|
+
...errors,
|
|
30655
|
+
dates,
|
|
30656
|
+
interpretation: "Bing reported more crawl errors in its latest daily snapshot. The count may include URLs outside the current sitemap.",
|
|
30657
|
+
verification: "Run the fast sitemap health check, inspect Bing crawl issue samples, then audit any affected URLs directly."
|
|
30658
|
+
}));
|
|
30659
|
+
const code4xx = crawl.changes.code4xx;
|
|
30660
|
+
if (code4xx && code4xx.absolute >= Math.max(10, Math.ceil(code4xx.previous * .5))) findings.push(changeFinding({
|
|
30661
|
+
code: "bing_4xx_increased",
|
|
30662
|
+
title: "Bing-observed 4xx responses increased",
|
|
30663
|
+
...code4xx,
|
|
30664
|
+
dates,
|
|
30665
|
+
interpretation: "Bing encountered more 4xx responses. Intentional removals and obsolete discovered URLs can contribute to this count.",
|
|
30666
|
+
verification: "Inspect Bing crawl issue URLs and recover or redirect only URLs that should still exist or retain search value."
|
|
30667
|
+
}));
|
|
30668
|
+
const code5xx = crawl.changes.code5xx;
|
|
30669
|
+
if (code5xx && code5xx.current > 0 && code5xx.absolute > 0) findings.push(changeFinding({
|
|
30670
|
+
code: "bing_5xx_increased",
|
|
30671
|
+
title: "Bing-observed server errors increased",
|
|
30672
|
+
...code5xx,
|
|
30673
|
+
dates,
|
|
30674
|
+
interpretation: "Bing encountered more server errors in the latest snapshot. Even a small increase deserves a live check.",
|
|
30675
|
+
verification: "Check current server logs and run targeted URL audits before treating the provider count as an active outage."
|
|
30676
|
+
}));
|
|
30677
|
+
}
|
|
30678
|
+
for (const [kind, analysis] of [["query", input.queries], ["page", input.pages]]) {
|
|
30679
|
+
if (!analysis?.opportunities.length) continue;
|
|
30680
|
+
findings.push({
|
|
30681
|
+
code: `bing_${kind}_opportunities`,
|
|
30682
|
+
severity: "review",
|
|
30683
|
+
title: `${analysis.opportunities.length} Bing ${kind} opportunities need review`,
|
|
30684
|
+
evidence: {
|
|
30685
|
+
retainedOpportunities: analysis.opportunities.length,
|
|
30686
|
+
currentDimensions: analysis.coverage.currentDimensions,
|
|
30687
|
+
matchedDimensions: analysis.coverage.matchedDimensions,
|
|
30688
|
+
comparableDimensions: analysis.coverage.comparableDimensions,
|
|
30689
|
+
thresholds: analysis.thresholds
|
|
30690
|
+
},
|
|
30691
|
+
interpretation: "These are high-impression top-list entries currently observed around positions 4 to 20. The threshold is a prioritization heuristic, not a search-engine rule.",
|
|
30692
|
+
verification: kind === "query" ? "Check intent and the ranking page before changing copy or creating content." : "Audit the selected page and its matching queries before changing internal links, titles, or content."
|
|
30693
|
+
});
|
|
30694
|
+
}
|
|
30695
|
+
return findings;
|
|
30696
|
+
}
|
|
30697
|
+
async function bingWebmasterOverview(input) {
|
|
30698
|
+
if (!input.site.trim()) throw new SeoError("INVALID_INPUT", "Pass a Bing Webmaster site URL.");
|
|
30699
|
+
const resolved = input.client ? {
|
|
30700
|
+
client: input.client,
|
|
30701
|
+
credentialSource: input.credentialSource ?? "environment"
|
|
30702
|
+
} : await createBingWebmasterClient();
|
|
30703
|
+
const observedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
30704
|
+
const topListWarning = "Bing returns weekly top-list rows, so dimensions missing from a period remain unknown.";
|
|
30705
|
+
const [traffic, crawl, queries, pages] = await Promise.all([
|
|
30706
|
+
section(() => resolved.client.getTraffic(input.site), { rowLimit: 400 }),
|
|
30707
|
+
section(() => resolved.client.getCrawlStats(input.site), { rowLimit: 400 }),
|
|
30708
|
+
section(() => resolved.client.getQueryStats(input.site), {
|
|
30709
|
+
rowLimit: 8e3,
|
|
30710
|
+
sourceWarning: topListWarning
|
|
30711
|
+
}),
|
|
30712
|
+
section(() => resolved.client.getPageStats(input.site), {
|
|
30713
|
+
rowLimit: 8e3,
|
|
30714
|
+
sourceWarning: topListWarning
|
|
30715
|
+
})
|
|
30716
|
+
]);
|
|
30717
|
+
const statuses = [
|
|
30718
|
+
traffic.status,
|
|
30719
|
+
crawl.status,
|
|
30720
|
+
queries.status,
|
|
30721
|
+
pages.status
|
|
30722
|
+
];
|
|
30723
|
+
const dataStatus = statuses.every((status) => status === "complete") ? "complete" : statuses.every((status) => status === "unavailable") ? "unavailable" : "partial";
|
|
30724
|
+
const trafficAnalysis = traffic.status === "unavailable" ? void 0 : analyzeBingTraffic(traffic.data.rows);
|
|
30725
|
+
const crawlAnalysis = crawl.status === "unavailable" ? void 0 : analyzeBingCrawl(crawl.data.rows);
|
|
30726
|
+
const queryAnalysis = queries.status === "unavailable" ? void 0 : analyzeBingDimensions(queries.data.rows, "query");
|
|
30727
|
+
const pageAnalysis = pages.status === "unavailable" ? void 0 : analyzeBingDimensions(pages.data.rows, "page");
|
|
30728
|
+
const findings = buildFindings({
|
|
30729
|
+
traffic: trafficAnalysis,
|
|
30730
|
+
crawl: crawlAnalysis,
|
|
30731
|
+
queries: queryAnalysis,
|
|
30732
|
+
pages: pageAnalysis
|
|
30733
|
+
});
|
|
30734
|
+
return {
|
|
30735
|
+
schemaVersion: 2,
|
|
30736
|
+
site: input.site,
|
|
30737
|
+
generatedAt: observedAt,
|
|
30738
|
+
dataStatus,
|
|
30739
|
+
summary: {
|
|
30740
|
+
findings: findings.length,
|
|
30741
|
+
warnings: findings.filter((finding) => finding.severity === "warning").length,
|
|
30742
|
+
reviewItems: findings.filter((finding) => finding.severity === "review").length
|
|
30743
|
+
},
|
|
30744
|
+
findings,
|
|
30745
|
+
outputBudget: {
|
|
30746
|
+
maxFindings: 6,
|
|
30747
|
+
maxDailyRowsPerSection: DAILY_OUTPUT_ROWS,
|
|
30748
|
+
maxDailyRowsTotal: DAILY_OUTPUT_ROWS * 2,
|
|
30749
|
+
maxDimensionItemsPerList: 10,
|
|
30750
|
+
maxDimensionItemsTotal: 40
|
|
30751
|
+
},
|
|
30752
|
+
provenance: {
|
|
30753
|
+
provider: "bing-webmaster",
|
|
30754
|
+
authentication: resolved.client.authentication,
|
|
30755
|
+
credentialSource: resolved.credentialSource,
|
|
30756
|
+
observedAt,
|
|
30757
|
+
cached: false,
|
|
30758
|
+
rowLimits: {
|
|
30759
|
+
traffic: 400,
|
|
30760
|
+
crawl: 400,
|
|
30761
|
+
queries: 8e3,
|
|
30762
|
+
pages: 8e3
|
|
30763
|
+
},
|
|
30764
|
+
methods: [
|
|
30765
|
+
"GetRankAndTrafficStats",
|
|
30766
|
+
"GetCrawlStats",
|
|
30767
|
+
"GetQueryStats",
|
|
30768
|
+
"GetPageStats"
|
|
30769
|
+
]
|
|
30770
|
+
},
|
|
30771
|
+
traffic: traffic.status === "unavailable" ? traffic : {
|
|
30772
|
+
...traffic,
|
|
30773
|
+
data: {
|
|
30774
|
+
...dailyOutput(traffic.data),
|
|
30775
|
+
range: range(traffic.data.rows),
|
|
30776
|
+
clicks: traffic.data.rows.reduce((total, row) => total + row.clicks, 0),
|
|
30777
|
+
impressions: traffic.data.rows.reduce((total, row) => total + row.impressions, 0),
|
|
30778
|
+
analysis: trafficAnalysis
|
|
30779
|
+
}
|
|
30780
|
+
},
|
|
30781
|
+
crawl: crawl.status === "unavailable" ? crawl : {
|
|
30782
|
+
...crawl,
|
|
30783
|
+
data: {
|
|
30784
|
+
...dailyOutput(crawl.data),
|
|
30785
|
+
range: range(crawl.data.rows),
|
|
30786
|
+
latest: crawl.data.rows.at(-1),
|
|
30787
|
+
analysis: crawlAnalysis
|
|
30788
|
+
}
|
|
30789
|
+
},
|
|
30790
|
+
queries: queries.status === "unavailable" ? queries : {
|
|
30791
|
+
...queries,
|
|
30792
|
+
data: {
|
|
30793
|
+
returnedRows: queries.data.returnedRows,
|
|
30794
|
+
retainedRows: queries.data.rows.length,
|
|
30795
|
+
invalidRows: queries.data.invalidRows,
|
|
30796
|
+
capped: queries.data.capped,
|
|
30797
|
+
range: range(queries.data.rows),
|
|
30798
|
+
analysis: queryAnalysis
|
|
30799
|
+
}
|
|
30800
|
+
},
|
|
30801
|
+
pages: pages.status === "unavailable" ? pages : {
|
|
30802
|
+
...pages,
|
|
30803
|
+
data: {
|
|
30804
|
+
returnedRows: pages.data.returnedRows,
|
|
30805
|
+
retainedRows: pages.data.rows.length,
|
|
30806
|
+
invalidRows: pages.data.invalidRows,
|
|
30807
|
+
capped: pages.data.capped,
|
|
30808
|
+
range: range(pages.data.rows),
|
|
30809
|
+
analysis: pageAnalysis
|
|
30810
|
+
}
|
|
30811
|
+
},
|
|
30812
|
+
caveats: [
|
|
30813
|
+
"Bing reports its own observed search and crawl evidence. It is not a complete view of every search engine.",
|
|
30814
|
+
"The inIndex field is provider-reported crawl statistics, not independent proof that a URL is indexed.",
|
|
30815
|
+
"Query and page movements include only dimensions observed in every weekly top list in both periods. Missing entries are not treated as zero.",
|
|
30816
|
+
"Opportunity thresholds prioritize review. They are heuristics, not search-engine requirements."
|
|
30817
|
+
]
|
|
30818
|
+
};
|
|
30819
|
+
}
|
|
30820
|
+
//#endregion
|
|
30821
|
+
//#region packages/core/src/clients.ts
|
|
30822
|
+
function googleAnalyticsPropertyId(client) {
|
|
30823
|
+
return client?.analytics.google?.propertyId;
|
|
30824
|
+
}
|
|
30825
|
+
function bingWebmasterSiteUrl(client) {
|
|
30826
|
+
return client?.searchEngines?.bing?.siteUrl;
|
|
30827
|
+
}
|
|
30828
|
+
function slug(value) {
|
|
30829
|
+
return value.toLowerCase().replace(/^https?:\/\//, "").replace(/^sc-domain:/, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60);
|
|
30830
|
+
}
|
|
30831
|
+
function uniqueStrings(values = []) {
|
|
30832
|
+
return [...new Set(values.map((value) => value.trim()).filter(Boolean))];
|
|
29527
30833
|
}
|
|
29528
30834
|
function deriveId(input) {
|
|
29529
30835
|
const id = slug(input.id ?? input.name ?? input.siteUrl);
|
|
@@ -29556,6 +30862,7 @@ function saveClient(input) {
|
|
|
29556
30862
|
siteUrl: input.siteUrl
|
|
29557
30863
|
})),
|
|
29558
30864
|
analytics: input.analytics ?? existing?.analytics ?? {},
|
|
30865
|
+
searchEngines: input.searchEngines ?? existing?.searchEngines,
|
|
29559
30866
|
reportDay: input.reportDay ?? existing?.reportDay,
|
|
29560
30867
|
technicalWeekday: input.technicalWeekday ?? existing?.technicalWeekday,
|
|
29561
30868
|
isDefault: input.isDefault ?? existing?.isDefault,
|
|
@@ -29571,6 +30878,19 @@ function saveClient(input) {
|
|
|
29571
30878
|
writeConfig(config);
|
|
29572
30879
|
return client;
|
|
29573
30880
|
}
|
|
30881
|
+
function setClientBingSite(idOrName, siteUrl) {
|
|
30882
|
+
const client = getClient(idOrName);
|
|
30883
|
+
if (!client) throw new Error(`Client not found: ${idOrName}`);
|
|
30884
|
+
return saveClient({
|
|
30885
|
+
id: client.id,
|
|
30886
|
+
name: client.name,
|
|
30887
|
+
siteUrl: client.siteUrl,
|
|
30888
|
+
searchEngines: {
|
|
30889
|
+
...client.searchEngines,
|
|
30890
|
+
bing: { siteUrl }
|
|
30891
|
+
}
|
|
30892
|
+
});
|
|
30893
|
+
}
|
|
29574
30894
|
function deleteClient(idOrName) {
|
|
29575
30895
|
const config = readConfig();
|
|
29576
30896
|
const client = getClient(idOrName);
|
|
@@ -29678,6 +30998,151 @@ async function runDoctor(dependencies = {}) {
|
|
|
29678
30998
|
};
|
|
29679
30999
|
}
|
|
29680
31000
|
//#endregion
|
|
31001
|
+
//#region packages/core/src/export/report-html.ts
|
|
31002
|
+
function escapeHtml(value) {
|
|
31003
|
+
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll("\"", """).replaceAll("'", "'");
|
|
31004
|
+
}
|
|
31005
|
+
function safeHttpUrl(value) {
|
|
31006
|
+
try {
|
|
31007
|
+
const url = new URL(value);
|
|
31008
|
+
return url.protocol === "http:" || url.protocol === "https:" ? url.toString() : void 0;
|
|
31009
|
+
} catch {
|
|
31010
|
+
return;
|
|
31011
|
+
}
|
|
31012
|
+
}
|
|
31013
|
+
function siteMarkup(site) {
|
|
31014
|
+
const safeUrl = safeHttpUrl(site);
|
|
31015
|
+
const label = escapeHtml(site);
|
|
31016
|
+
return safeUrl ? `<a href="${escapeHtml(safeUrl)}" rel="noreferrer">${label}</a>` : label;
|
|
31017
|
+
}
|
|
31018
|
+
function list(items) {
|
|
31019
|
+
if (items.length === 0) return "<p class=\"empty\">No items to report.</p>";
|
|
31020
|
+
return `<ul>${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul>`;
|
|
31021
|
+
}
|
|
31022
|
+
function reportSections(report) {
|
|
31023
|
+
return report.sections.map((section) => `<section class="panel">
|
|
31024
|
+
<h2>${escapeHtml(section.title)}</h2>
|
|
31025
|
+
${list(section.bullets)}
|
|
31026
|
+
</section>`).join("");
|
|
31027
|
+
}
|
|
31028
|
+
function additionalSections(sections) {
|
|
31029
|
+
return sections.map((section) => `<section class="panel">
|
|
31030
|
+
<h2>${escapeHtml(section.title)}</h2>
|
|
31031
|
+
${section.summary ? `<p>${escapeHtml(section.summary)}</p>` : ""}
|
|
31032
|
+
${list(section.items)}
|
|
31033
|
+
</section>`).join("");
|
|
31034
|
+
}
|
|
31035
|
+
function priorities(report) {
|
|
31036
|
+
if (report.priorities.length === 0) return "<p class=\"empty\">No evidence-backed priorities were found.</p>";
|
|
31037
|
+
return `<ol class="priorities">${report.priorities.map((priority) => `<li>
|
|
31038
|
+
<div class="priority-heading">
|
|
31039
|
+
<strong>${escapeHtml(priority.title)}</strong>
|
|
31040
|
+
<span class="confidence">${escapeHtml(priority.confidence)} confidence</span>
|
|
31041
|
+
</div>
|
|
31042
|
+
<p>${escapeHtml(priority.action)}</p>
|
|
31043
|
+
</li>`).join("")}</ol>`;
|
|
31044
|
+
}
|
|
31045
|
+
function analystEvidence(report) {
|
|
31046
|
+
const skipped = report.diagnosis.skippedSections ?? [];
|
|
31047
|
+
const attempts = report.changeMeasurementAttempts;
|
|
31048
|
+
return `<section class="panel analyst-only">
|
|
31049
|
+
<h2>Evidence coverage</h2>
|
|
31050
|
+
<dl class="facts">
|
|
31051
|
+
<div><dt>Data status</dt><dd>${escapeHtml(report.dataStatus)}</dd></div>
|
|
31052
|
+
<div><dt>Generated</dt><dd>${escapeHtml(report.generatedAt)}</dd></div>
|
|
31053
|
+
<div><dt>Saved changes checked</dt><dd>${attempts.length}</dd></div>
|
|
31054
|
+
<div><dt>Unavailable sections</dt><dd>${skipped.length}</dd></div>
|
|
31055
|
+
</dl>
|
|
31056
|
+
${skipped.length ? `<h3>Unavailable sections</h3>${list(skipped.map((section) => "reason" in section && typeof section.reason === "string" ? `${section.section}: ${section.reason}` : section.section))}` : ""}
|
|
31057
|
+
</section>`;
|
|
31058
|
+
}
|
|
31059
|
+
const styles = `
|
|
31060
|
+
:root { color-scheme: light; font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; color: #17211b; background: #f2f4ef; }
|
|
31061
|
+
* { box-sizing: border-box; }
|
|
31062
|
+
body { margin: 0; line-height: 1.55; }
|
|
31063
|
+
a { color: inherit; text-decoration-color: #ff5b35; text-underline-offset: 0.18em; }
|
|
31064
|
+
main { width: min(980px, calc(100% - 32px)); margin: 32px auto 64px; }
|
|
31065
|
+
header { padding: clamp(28px, 6vw, 64px); color: #f9fbf7; background: #17211b; border-radius: 20px; }
|
|
31066
|
+
.eyebrow { margin: 0 0 12px; color: #ff7958; font-size: 0.78rem; font-weight: 800; letter-spacing: 0.12em; text-transform: uppercase; }
|
|
31067
|
+
h1 { max-width: 760px; margin: 0; font-size: clamp(2rem, 5vw, 4rem); line-height: 1.02; letter-spacing: -0.04em; }
|
|
31068
|
+
header .headline { max-width: 760px; margin: 24px 0 0; font-size: clamp(1.05rem, 2vw, 1.3rem); color: #d9e1d9; }
|
|
31069
|
+
.meta { display: flex; flex-wrap: wrap; gap: 10px 22px; margin-top: 28px; color: #b9c5ba; font-size: 0.9rem; }
|
|
31070
|
+
.status { display: inline-flex; width: fit-content; margin-top: 22px; padding: 6px 10px; border: 1px solid #4f6153; border-radius: 999px; font-size: 0.8rem; font-weight: 700; text-transform: capitalize; }
|
|
31071
|
+
.panel { margin-top: 20px; padding: clamp(22px, 4vw, 38px); background: #fff; border: 1px solid #dfe4dc; border-radius: 16px; box-shadow: 0 8px 30px rgba(23, 33, 27, 0.05); }
|
|
31072
|
+
h2 { margin: 0 0 16px; font-size: clamp(1.25rem, 3vw, 1.65rem); letter-spacing: -0.02em; }
|
|
31073
|
+
h3 { margin: 26px 0 10px; font-size: 1rem; }
|
|
31074
|
+
p { margin: 0 0 14px; }
|
|
31075
|
+
ul, ol { margin: 0; padding-left: 1.25rem; }
|
|
31076
|
+
li + li { margin-top: 10px; }
|
|
31077
|
+
.priorities { list-style: none; padding: 0; counter-reset: priority; }
|
|
31078
|
+
.priorities > li { counter-increment: priority; padding: 18px 0 18px 52px; position: relative; border-top: 1px solid #e6eae4; }
|
|
31079
|
+
.priorities > li:first-child { border-top: 0; padding-top: 4px; }
|
|
31080
|
+
.priorities > li::before { content: counter(priority); position: absolute; left: 0; top: 16px; display: grid; width: 34px; height: 34px; place-items: center; border-radius: 50%; color: #17211b; background: #ff7958; font-weight: 800; }
|
|
31081
|
+
.priorities > li:first-child::before { top: 2px; }
|
|
31082
|
+
.priority-heading { display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 8px 20px; }
|
|
31083
|
+
.confidence { color: #647067; font-size: 0.82rem; text-transform: capitalize; }
|
|
31084
|
+
.priorities p { margin: 8px 0 0; color: #3e4941; }
|
|
31085
|
+
.facts { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 12px; margin: 0; }
|
|
31086
|
+
.facts div { padding: 14px; background: #f5f7f3; border-radius: 10px; }
|
|
31087
|
+
dt { color: #647067; font-size: 0.78rem; font-weight: 700; text-transform: uppercase; }
|
|
31088
|
+
dd { margin: 4px 0 0; font-weight: 700; }
|
|
31089
|
+
details summary { cursor: pointer; font-weight: 750; }
|
|
31090
|
+
details ul { margin-top: 14px; }
|
|
31091
|
+
.empty { color: #647067; font-style: italic; }
|
|
31092
|
+
footer { margin-top: 26px; color: #647067; font-size: 0.8rem; text-align: center; }
|
|
31093
|
+
@media print { :root { background: #fff; } main { width: 100%; margin: 0; } header, .panel { break-inside: avoid; box-shadow: none; } header { border-radius: 0; } }
|
|
31094
|
+
`;
|
|
31095
|
+
function renderReportHtml(input) {
|
|
31096
|
+
const report = input.report;
|
|
31097
|
+
const view = input.view ?? "client";
|
|
31098
|
+
const title = input.title ?? "SEO report";
|
|
31099
|
+
const evidenceNotesOpen = report.dataStatus !== "complete" ? " open" : "";
|
|
31100
|
+
const analyst = view === "analyst" ? analystEvidence(report) : "";
|
|
31101
|
+
return `<!doctype html>
|
|
31102
|
+
<html lang="en">
|
|
31103
|
+
<head>
|
|
31104
|
+
<meta charset="utf-8">
|
|
31105
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
31106
|
+
<meta name="robots" content="noindex,nofollow">
|
|
31107
|
+
<title>${escapeHtml(title)}: ${escapeHtml(report.site)}</title>
|
|
31108
|
+
<style>${styles}</style>
|
|
31109
|
+
</head>
|
|
31110
|
+
<body>
|
|
31111
|
+
<main>
|
|
31112
|
+
<header>
|
|
31113
|
+
<p class="eyebrow">${escapeHtml(title)}</p>
|
|
31114
|
+
<h1>${siteMarkup(report.site)}</h1>
|
|
31115
|
+
<p class="headline">${escapeHtml(report.headline)}</p>
|
|
31116
|
+
<div class="meta">
|
|
31117
|
+
<span>${escapeHtml(report.period.startDate)} to ${escapeHtml(report.period.endDate)}</span>
|
|
31118
|
+
<span>Generated ${escapeHtml(report.generatedAt.slice(0, 10))}</span>
|
|
31119
|
+
<span>${escapeHtml(view)} view</span>
|
|
31120
|
+
</div>
|
|
31121
|
+
<span class="status">${escapeHtml(report.dataStatus)} data</span>
|
|
31122
|
+
</header>
|
|
31123
|
+
|
|
31124
|
+
<section class="panel">
|
|
31125
|
+
<h2>Priorities</h2>
|
|
31126
|
+
${priorities(report)}
|
|
31127
|
+
</section>
|
|
31128
|
+
|
|
31129
|
+
${reportSections(report)}
|
|
31130
|
+
${additionalSections(input.additionalSections ?? [])}
|
|
31131
|
+
${analyst}
|
|
31132
|
+
|
|
31133
|
+
<section class="panel">
|
|
31134
|
+
<details${evidenceNotesOpen}>
|
|
31135
|
+
<summary>Evidence notes and limitations</summary>
|
|
31136
|
+
${list(report.caveats)}
|
|
31137
|
+
</details>
|
|
31138
|
+
</section>
|
|
31139
|
+
|
|
31140
|
+
<footer>Generated locally from observed report evidence.</footer>
|
|
31141
|
+
</main>
|
|
31142
|
+
</body>
|
|
31143
|
+
</html>`;
|
|
31144
|
+
}
|
|
31145
|
+
//#endregion
|
|
29681
31146
|
//#region packages/core/src/export/update-postmortem.ts
|
|
29682
31147
|
const HEADERS = {
|
|
29683
31148
|
summary: [
|
|
@@ -29988,60 +31453,779 @@ function updatePostmortemCsvFiles(report) {
|
|
|
29988
31453
|
headers: [...HEADERS.segment],
|
|
29989
31454
|
rows: postmortemSegmentRows(report, dimension)
|
|
29990
31455
|
})),
|
|
29991
|
-
...[
|
|
29992
|
-
"page",
|
|
29993
|
-
"query",
|
|
29994
|
-
"device",
|
|
29995
|
-
"country"
|
|
29996
|
-
].map((dimension) => ({
|
|
29997
|
-
filename: `postmortem-segment-${dimension}-unmatched.csv`,
|
|
29998
|
-
headers: [...HEADERS.unmatchedSegment],
|
|
29999
|
-
rows: postmortemUnmatchedSegmentRows(report, dimension)
|
|
30000
|
-
}))
|
|
30001
|
-
];
|
|
31456
|
+
...[
|
|
31457
|
+
"page",
|
|
31458
|
+
"query",
|
|
31459
|
+
"device",
|
|
31460
|
+
"country"
|
|
31461
|
+
].map((dimension) => ({
|
|
31462
|
+
filename: `postmortem-segment-${dimension}-unmatched.csv`,
|
|
31463
|
+
headers: [...HEADERS.unmatchedSegment],
|
|
31464
|
+
rows: postmortemUnmatchedSegmentRows(report, dimension)
|
|
31465
|
+
}))
|
|
31466
|
+
];
|
|
31467
|
+
}
|
|
31468
|
+
//#endregion
|
|
31469
|
+
//#region packages/core/src/ga4/matching.ts
|
|
31470
|
+
function hostnameFromUrl(value) {
|
|
31471
|
+
try {
|
|
31472
|
+
const url = new URL(value);
|
|
31473
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
31474
|
+
return url.hostname.toLowerCase().replace(/\.$/, "") || void 0;
|
|
31475
|
+
} catch {
|
|
31476
|
+
return;
|
|
31477
|
+
}
|
|
31478
|
+
}
|
|
31479
|
+
function domainFromProperty(value) {
|
|
31480
|
+
if (!value.startsWith("sc-domain:")) return void 0;
|
|
31481
|
+
const domain = value.slice(10).trim().toLowerCase();
|
|
31482
|
+
if (!domain || /[/:]/.test(domain)) return void 0;
|
|
31483
|
+
return domain.replace(/\.$/, "") || void 0;
|
|
31484
|
+
}
|
|
31485
|
+
function matchForSite(site, streamHostname) {
|
|
31486
|
+
const domain = domainFromProperty(site);
|
|
31487
|
+
if (domain) return streamHostname === domain || streamHostname.endsWith(`.${domain}`) ? "domain" : void 0;
|
|
31488
|
+
return hostnameFromUrl(site) === streamHostname ? "hostname" : void 0;
|
|
31489
|
+
}
|
|
31490
|
+
function matchGa4WebStreams(site, candidates) {
|
|
31491
|
+
return candidates.flatMap((candidate) => {
|
|
31492
|
+
const defaultUri = candidate.stream.webStreamData?.defaultUri;
|
|
31493
|
+
if (!defaultUri) return [];
|
|
31494
|
+
const hostname = hostnameFromUrl(defaultUri);
|
|
31495
|
+
if (!hostname) return [];
|
|
31496
|
+
const match = matchForSite(site, hostname);
|
|
31497
|
+
if (!match) return [];
|
|
31498
|
+
return [{
|
|
31499
|
+
...candidate,
|
|
31500
|
+
defaultUri,
|
|
31501
|
+
hostname,
|
|
31502
|
+
match
|
|
31503
|
+
}];
|
|
31504
|
+
});
|
|
31505
|
+
}
|
|
31506
|
+
function ga4MatchReason(match, site) {
|
|
31507
|
+
const stream = match.stream.displayName ?? match.defaultUri;
|
|
31508
|
+
const location = match.match === "domain" ? "is inside" : "matches";
|
|
31509
|
+
return `Matched Google Analytics web stream ${stream} (${match.defaultUri}): its hostname ${location} ${site}.`;
|
|
31510
|
+
}
|
|
31511
|
+
//#endregion
|
|
31512
|
+
//#region packages/core/src/indexnow/types.ts
|
|
31513
|
+
const INDEXNOW_ENDPOINT = "https://api.indexnow.org/indexnow";
|
|
31514
|
+
const INDEXNOW_MAX_URLS = 1e3;
|
|
31515
|
+
const INDEXNOW_KEY_ENV = "SEO_INDEXNOW_KEY";
|
|
31516
|
+
const INDEXNOW_KEYS_SECRET = "indexnow-keys";
|
|
31517
|
+
//#endregion
|
|
31518
|
+
//#region packages/core/src/indexnow/keys.ts
|
|
31519
|
+
const KEY_PATTERN = /^[A-Za-z0-9-]{8,128}$/;
|
|
31520
|
+
function isPrivateHost(hostname) {
|
|
31521
|
+
const host = hostname.toLowerCase().replace(/^\[|\]$/g, "");
|
|
31522
|
+
if (host === "localhost" || host.endsWith(".localhost") || host.includes(":")) return true;
|
|
31523
|
+
const octets = host.split(".").map(Number);
|
|
31524
|
+
if (octets.length !== 4 || octets.some((value) => !Number.isInteger(value))) return false;
|
|
31525
|
+
const [first, second] = octets;
|
|
31526
|
+
return first === 0 || first === 10 || first === 127 || first === 100 && second !== void 0 && second >= 64 && second <= 127 || first === 169 && second === 254 || first === 172 && second !== void 0 && second >= 16 && second <= 31 || first === 192 && second === 0 || first === 192 && second === 168 || first === 198 && (second === 18 || second === 19) || first !== void 0 && first >= 224;
|
|
31527
|
+
}
|
|
31528
|
+
function parseIndexNowSite(value) {
|
|
31529
|
+
let url;
|
|
31530
|
+
try {
|
|
31531
|
+
url = new URL(value);
|
|
31532
|
+
} catch {
|
|
31533
|
+
throw new SeoError("INVALID_INPUT", "IndexNow site must be a valid URL.");
|
|
31534
|
+
}
|
|
31535
|
+
if (!["http:", "https:"].includes(url.protocol) || !url.hostname) throw new SeoError("INVALID_INPUT", "IndexNow site must use HTTP or HTTPS.");
|
|
31536
|
+
if (url.username || url.password) throw new SeoError("INVALID_INPUT", "IndexNow site URLs cannot contain credentials.");
|
|
31537
|
+
if (isPrivateHost(url.hostname)) throw new SeoError("INVALID_INPUT", "IndexNow requires a public site host.");
|
|
31538
|
+
return url;
|
|
31539
|
+
}
|
|
31540
|
+
function validateIndexNowKey(value) {
|
|
31541
|
+
const key = value.trim();
|
|
31542
|
+
if (!KEY_PATTERN.test(key)) throw new SeoError("INVALID_INPUT", "IndexNow keys must contain 8 to 128 letters, numbers, or hyphens.");
|
|
31543
|
+
return key;
|
|
31544
|
+
}
|
|
31545
|
+
function createIndexNowKey() {
|
|
31546
|
+
return randomBytes(16).toString("hex");
|
|
31547
|
+
}
|
|
31548
|
+
function createIndexNowKeyRecord(input) {
|
|
31549
|
+
const site = parseIndexNowSite(input.site);
|
|
31550
|
+
const key = validateIndexNowKey(input.key ?? createIndexNowKey());
|
|
31551
|
+
const keyLocation = input.keyLocation ? parseIndexNowSite(input.keyLocation) : new URL(`/${key}.txt`, site.origin);
|
|
31552
|
+
if (keyLocation.hostname.toLowerCase() !== site.hostname.toLowerCase()) throw new SeoError("INVALID_INPUT", "IndexNow key location must use the same host as the site.");
|
|
31553
|
+
return {
|
|
31554
|
+
host: site.hostname.toLowerCase(),
|
|
31555
|
+
key,
|
|
31556
|
+
keyLocation: keyLocation.toString(),
|
|
31557
|
+
createdAt: (input.now ?? /* @__PURE__ */ new Date()).toISOString()
|
|
31558
|
+
};
|
|
31559
|
+
}
|
|
31560
|
+
function validateIndexNowKeyRecord(record) {
|
|
31561
|
+
const createdAt = new Date(record.createdAt);
|
|
31562
|
+
if (Number.isNaN(createdAt.getTime())) throw new SeoError("INVALID_INPUT", "IndexNow key record has an invalid creation date.");
|
|
31563
|
+
const normalized = createIndexNowKeyRecord({
|
|
31564
|
+
site: record.keyLocation,
|
|
31565
|
+
key: record.key,
|
|
31566
|
+
keyLocation: record.keyLocation,
|
|
31567
|
+
now: createdAt
|
|
31568
|
+
});
|
|
31569
|
+
if (normalized.host !== record.host.toLowerCase()) throw new SeoError("INVALID_INPUT", "IndexNow key record host does not match its key location.");
|
|
31570
|
+
return normalized;
|
|
31571
|
+
}
|
|
31572
|
+
function emptyKeys() {
|
|
31573
|
+
return {
|
|
31574
|
+
version: 1,
|
|
31575
|
+
sites: {}
|
|
31576
|
+
};
|
|
31577
|
+
}
|
|
31578
|
+
function parseSavedKeys(value) {
|
|
31579
|
+
try {
|
|
31580
|
+
const parsed = JSON.parse(value);
|
|
31581
|
+
if (parsed.version !== 1 || !parsed.sites || typeof parsed.sites !== "object" || Array.isArray(parsed.sites)) throw new Error("invalid shape");
|
|
31582
|
+
const sites = {};
|
|
31583
|
+
for (const [host, value] of Object.entries(parsed.sites)) {
|
|
31584
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("invalid key record");
|
|
31585
|
+
const record = validateIndexNowKeyRecord(value);
|
|
31586
|
+
if (record.host !== host) throw new Error("invalid host mapping");
|
|
31587
|
+
sites[host] = record;
|
|
31588
|
+
}
|
|
31589
|
+
return {
|
|
31590
|
+
version: 1,
|
|
31591
|
+
sites
|
|
31592
|
+
};
|
|
31593
|
+
} catch {
|
|
31594
|
+
throw new SeoError("AUTH_REQUIRED", "Saved IndexNow keys are invalid. Run `seo indexnow setup` again.");
|
|
31595
|
+
}
|
|
31596
|
+
}
|
|
31597
|
+
async function readSavedKeys() {
|
|
31598
|
+
const saved = await readProviderSecret({ name: INDEXNOW_KEYS_SECRET });
|
|
31599
|
+
if (!saved) return { keys: emptyKeys() };
|
|
31600
|
+
return {
|
|
31601
|
+
keys: parseSavedKeys(saved.value),
|
|
31602
|
+
source: saved.source
|
|
31603
|
+
};
|
|
31604
|
+
}
|
|
31605
|
+
async function saveIndexNowKey(record) {
|
|
31606
|
+
const normalized = validateIndexNowKeyRecord(record);
|
|
31607
|
+
const { keys } = await readSavedKeys();
|
|
31608
|
+
keys.sites[normalized.host] = normalized;
|
|
31609
|
+
return writeProviderSecret(INDEXNOW_KEYS_SECRET, JSON.stringify(keys));
|
|
31610
|
+
}
|
|
31611
|
+
async function resolveIndexNowKey(input) {
|
|
31612
|
+
const site = parseIndexNowSite(input.site);
|
|
31613
|
+
const environmentKey = (input.env ?? process.env)[INDEXNOW_KEY_ENV]?.trim();
|
|
31614
|
+
if (environmentKey) return {
|
|
31615
|
+
record: createIndexNowKeyRecord({
|
|
31616
|
+
site: site.origin,
|
|
31617
|
+
key: environmentKey
|
|
31618
|
+
}),
|
|
31619
|
+
source: "environment"
|
|
31620
|
+
};
|
|
31621
|
+
const { keys, source } = await readSavedKeys();
|
|
31622
|
+
const record = keys.sites[site.hostname.toLowerCase()];
|
|
31623
|
+
if (!record || !source) throw new SeoError("AUTH_REQUIRED", `IndexNow is not set up for ${site.hostname}. Run \`seo indexnow setup --site ${site.origin} --output <public-directory>\`, or set SEO_INDEXNOW_KEY for this process.`);
|
|
31624
|
+
return {
|
|
31625
|
+
record: validateIndexNowKeyRecord(record),
|
|
31626
|
+
source
|
|
31627
|
+
};
|
|
31628
|
+
}
|
|
31629
|
+
async function listIndexNowKeys() {
|
|
31630
|
+
const { keys } = await readSavedKeys();
|
|
31631
|
+
return Object.values(keys.sites).map(({ key: _key, ...record }) => record).sort((a, b) => a.host < b.host ? -1 : a.host > b.host ? 1 : 0);
|
|
31632
|
+
}
|
|
31633
|
+
async function removeIndexNowKey(siteValue) {
|
|
31634
|
+
const site = parseIndexNowSite(siteValue);
|
|
31635
|
+
const { keys } = await readSavedKeys();
|
|
31636
|
+
const host = site.hostname.toLowerCase();
|
|
31637
|
+
if (!keys.sites[host]) return false;
|
|
31638
|
+
delete keys.sites[host];
|
|
31639
|
+
if (Object.keys(keys.sites).length === 0) await deleteProviderSecret(INDEXNOW_KEYS_SECRET);
|
|
31640
|
+
else await writeProviderSecret(INDEXNOW_KEYS_SECRET, JSON.stringify(keys));
|
|
31641
|
+
return true;
|
|
31642
|
+
}
|
|
31643
|
+
//#endregion
|
|
31644
|
+
//#region packages/core/src/indexnow/client.ts
|
|
31645
|
+
function uniqueUrls(values, host) {
|
|
31646
|
+
if (values.length === 0) throw new SeoError("INVALID_INPUT", "Pass at least one URL to IndexNow.");
|
|
31647
|
+
if (values.length > 1e3) throw new SeoError("INVALID_INPUT", `IndexNow submissions are limited to ${INDEXNOW_MAX_URLS} URLs per run.`);
|
|
31648
|
+
const urls = /* @__PURE__ */ new Set();
|
|
31649
|
+
for (const value of values) {
|
|
31650
|
+
const url = parseIndexNowSite(value);
|
|
31651
|
+
if (url.toString().length > 2e3) throw new SeoError("INVALID_INPUT", "IndexNow URLs cannot exceed 2,000 characters.");
|
|
31652
|
+
if (url.hostname.toLowerCase() !== host) throw new SeoError("INVALID_INPUT", `Every IndexNow URL must belong to ${host}.`);
|
|
31653
|
+
url.hash = "";
|
|
31654
|
+
urls.add(url.toString());
|
|
31655
|
+
}
|
|
31656
|
+
return [...urls].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
31657
|
+
}
|
|
31658
|
+
async function verifyIndexNowKey(input) {
|
|
31659
|
+
const record = validateIndexNowKeyRecord(input.record);
|
|
31660
|
+
const keyLocation = parseIndexNowSite(record.keyLocation);
|
|
31661
|
+
let response;
|
|
31662
|
+
try {
|
|
31663
|
+
response = await (input.fetchImpl ?? publicHttpFetch)(keyLocation.toString(), {
|
|
31664
|
+
headers: { accept: "text/plain" },
|
|
31665
|
+
redirect: "manual",
|
|
31666
|
+
signal: AbortSignal.timeout(input.timeoutMs ?? 15e3)
|
|
31667
|
+
});
|
|
31668
|
+
} catch (error) {
|
|
31669
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `IndexNow key verification failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
31670
|
+
}
|
|
31671
|
+
const body = await readBoundedResponseText(response, 512, "IndexNow key file");
|
|
31672
|
+
return {
|
|
31673
|
+
verified: response.ok && body.trim() === record.key,
|
|
31674
|
+
status: response.status,
|
|
31675
|
+
keyLocation: keyLocation.toString()
|
|
31676
|
+
};
|
|
31677
|
+
}
|
|
31678
|
+
function indexNowFailure(status) {
|
|
31679
|
+
if (status === 403) return new SeoError("ACCESS_DENIED", "IndexNow could not validate the key. Confirm the public key file contains the saved key.");
|
|
31680
|
+
if (status === 429) return new SeoError("RATE_LIMITED", "IndexNow rate limited this submission. Wait before trying again.");
|
|
31681
|
+
if (status === 400 || status === 422) return new SeoError("INVALID_INPUT", `IndexNow rejected the submission with HTTP ${status}. Check the URL host and key configuration.`);
|
|
31682
|
+
return new SeoError("PROVIDER_UNAVAILABLE", `IndexNow submission failed with HTTP ${status}.`);
|
|
31683
|
+
}
|
|
31684
|
+
async function submitIndexNow(input) {
|
|
31685
|
+
const record = validateIndexNowKeyRecord(input.record);
|
|
31686
|
+
const urls = uniqueUrls(input.urls, record.host);
|
|
31687
|
+
const keyLocation = parseIndexNowSite(record.keyLocation);
|
|
31688
|
+
const base = {
|
|
31689
|
+
schemaVersion: 1,
|
|
31690
|
+
generatedAt: (input.now ?? /* @__PURE__ */ new Date()).toISOString(),
|
|
31691
|
+
endpoint: INDEXNOW_ENDPOINT,
|
|
31692
|
+
host: record.host,
|
|
31693
|
+
keyLocation: keyLocation.toString(),
|
|
31694
|
+
submittedUrls: urls.length,
|
|
31695
|
+
caveats: ["IndexNow confirms receipt, not crawling, indexing, ranking, or traffic.", "Submit URLs only after they are added, updated, or deleted."]
|
|
31696
|
+
};
|
|
31697
|
+
if (input.dryRun) return {
|
|
31698
|
+
...base,
|
|
31699
|
+
dryRun: true,
|
|
31700
|
+
status: "validated"
|
|
31701
|
+
};
|
|
31702
|
+
const verification = await verifyIndexNowKey({
|
|
31703
|
+
record,
|
|
31704
|
+
fetchImpl: input.fetchImpl
|
|
31705
|
+
});
|
|
31706
|
+
if (!verification.verified) throw new SeoError("ACCESS_DENIED", `IndexNow key verification failed at ${verification.keyLocation} with HTTP ${verification.status}. Deploy the generated key file before submitting URLs.`);
|
|
31707
|
+
let response;
|
|
31708
|
+
try {
|
|
31709
|
+
response = await (input.fetchImpl ?? publicHttpFetch)(INDEXNOW_ENDPOINT, {
|
|
31710
|
+
method: "POST",
|
|
31711
|
+
headers: {
|
|
31712
|
+
accept: "application/json",
|
|
31713
|
+
"content-type": "application/json; charset=utf-8"
|
|
31714
|
+
},
|
|
31715
|
+
body: JSON.stringify({
|
|
31716
|
+
host: record.host,
|
|
31717
|
+
key: record.key,
|
|
31718
|
+
keyLocation: keyLocation.toString(),
|
|
31719
|
+
urlList: urls
|
|
31720
|
+
}),
|
|
31721
|
+
signal: AbortSignal.timeout(3e4)
|
|
31722
|
+
});
|
|
31723
|
+
} catch (error) {
|
|
31724
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `IndexNow did not respond: ${error instanceof Error ? error.message : String(error)}`);
|
|
31725
|
+
}
|
|
31726
|
+
await readBoundedResponseText(response, 64e3, "IndexNow response");
|
|
31727
|
+
if (response.status !== 200 && response.status !== 202) throw indexNowFailure(response.status);
|
|
31728
|
+
return {
|
|
31729
|
+
...base,
|
|
31730
|
+
dryRun: false,
|
|
31731
|
+
status: response.status === 200 ? "submitted" : "pending-validation",
|
|
31732
|
+
responseStatus: response.status
|
|
31733
|
+
};
|
|
31734
|
+
}
|
|
31735
|
+
//#endregion
|
|
31736
|
+
//#region packages/core/src/links/normalize.ts
|
|
31737
|
+
const FIELD_ALIASES = {
|
|
31738
|
+
sourceUrl: [
|
|
31739
|
+
"sourceurl",
|
|
31740
|
+
"source",
|
|
31741
|
+
"referringurl",
|
|
31742
|
+
"referringpage",
|
|
31743
|
+
"fromurl"
|
|
31744
|
+
],
|
|
31745
|
+
targetUrl: [
|
|
31746
|
+
"targeturl",
|
|
31747
|
+
"target",
|
|
31748
|
+
"destinationurl",
|
|
31749
|
+
"destination",
|
|
31750
|
+
"tourl"
|
|
31751
|
+
],
|
|
31752
|
+
anchorText: ["anchortext", "anchor"],
|
|
31753
|
+
firstSeenAt: ["firstseenat", "firstseen"],
|
|
31754
|
+
lastSeenAt: ["lastseenat", "lastseen"],
|
|
31755
|
+
nofollow: ["nofollow", "isnofollow"]
|
|
31756
|
+
};
|
|
31757
|
+
function key(value) {
|
|
31758
|
+
return value.toLowerCase().replaceAll(/[^a-z0-9]/g, "");
|
|
31759
|
+
}
|
|
31760
|
+
function valueFor(row, field) {
|
|
31761
|
+
const aliases = new Set(FIELD_ALIASES[field]);
|
|
31762
|
+
for (const [name, value] of Object.entries(row)) if (aliases.has(key(name))) return value;
|
|
31763
|
+
}
|
|
31764
|
+
function httpUrl(value) {
|
|
31765
|
+
if (typeof value !== "string" || !value.trim() || value.trim().length > 2e3) return;
|
|
31766
|
+
try {
|
|
31767
|
+
const url = new URL(value.trim());
|
|
31768
|
+
return url.protocol === "http:" || url.protocol === "https:" ? url : void 0;
|
|
31769
|
+
} catch {
|
|
31770
|
+
return;
|
|
31771
|
+
}
|
|
31772
|
+
}
|
|
31773
|
+
function optionalString(value) {
|
|
31774
|
+
return typeof value === "string" && value.trim() ? value.trim().slice(0, 1e3) : void 0;
|
|
31775
|
+
}
|
|
31776
|
+
function optionalBoolean(value) {
|
|
31777
|
+
if (typeof value === "boolean") return value;
|
|
31778
|
+
if (typeof value !== "string") return void 0;
|
|
31779
|
+
if (/^(true|yes|1)$/i.test(value.trim())) return true;
|
|
31780
|
+
if (/^(false|no|0)$/i.test(value.trim())) return false;
|
|
31781
|
+
}
|
|
31782
|
+
function normalizeLinkEvidenceRow(raw) {
|
|
31783
|
+
const source = httpUrl(valueFor(raw, "sourceUrl"));
|
|
31784
|
+
const target = httpUrl(valueFor(raw, "targetUrl"));
|
|
31785
|
+
if (!source || !target) return void 0;
|
|
31786
|
+
return {
|
|
31787
|
+
sourceUrl: source.toString(),
|
|
31788
|
+
sourceDomain: source.hostname.toLowerCase(),
|
|
31789
|
+
targetUrl: target.toString(),
|
|
31790
|
+
anchorText: optionalString(valueFor(raw, "anchorText")),
|
|
31791
|
+
firstSeenAt: optionalString(valueFor(raw, "firstSeenAt")),
|
|
31792
|
+
lastSeenAt: optionalString(valueFor(raw, "lastSeenAt")),
|
|
31793
|
+
nofollow: optionalBoolean(valueFor(raw, "nofollow"))
|
|
31794
|
+
};
|
|
31795
|
+
}
|
|
31796
|
+
function linkEvidenceKey(row) {
|
|
31797
|
+
return `${row.sourceUrl}\u0000${row.targetUrl}\u0000${row.anchorText ?? ""}`;
|
|
31798
|
+
}
|
|
31799
|
+
function compareLinkEvidence(a, b) {
|
|
31800
|
+
return a.targetUrl.localeCompare(b.targetUrl, "en") || a.sourceUrl.localeCompare(b.sourceUrl, "en") || (a.anchorText ?? "").localeCompare(b.anchorText ?? "", "en");
|
|
31801
|
+
}
|
|
31802
|
+
//#endregion
|
|
31803
|
+
//#region packages/core/src/links/bing.ts
|
|
31804
|
+
const MAX_COUNT_PAGES = 5;
|
|
31805
|
+
const DEFAULT_TARGET_LIMIT = 20;
|
|
31806
|
+
const MAX_TARGET_LIMIT = 50;
|
|
31807
|
+
const DEFAULT_DETAIL_PAGES = 1;
|
|
31808
|
+
const MAX_DETAIL_PAGES = 3;
|
|
31809
|
+
const DEFAULT_ROW_LIMIT$2 = 500;
|
|
31810
|
+
const MAX_ROW_LIMIT$2 = 1e3;
|
|
31811
|
+
const MAX_CONCURRENCY = 3;
|
|
31812
|
+
function integer(value, fallback, max, name) {
|
|
31813
|
+
const resolved = value ?? fallback;
|
|
31814
|
+
if (!Number.isInteger(resolved) || resolved < 1 || resolved > max) throw new SeoError("INVALID_INPUT", `${name} must be between 1 and ${max}.`);
|
|
31815
|
+
return resolved;
|
|
31816
|
+
}
|
|
31817
|
+
async function mapConcurrent(values, concurrency, work) {
|
|
31818
|
+
const output = new Array(values.length);
|
|
31819
|
+
let next = 0;
|
|
31820
|
+
async function worker() {
|
|
31821
|
+
while (next < values.length) {
|
|
31822
|
+
const index = next;
|
|
31823
|
+
next += 1;
|
|
31824
|
+
const value = values[index];
|
|
31825
|
+
if (value !== void 0) output[index] = await work(value);
|
|
31826
|
+
}
|
|
31827
|
+
}
|
|
31828
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, () => worker()));
|
|
31829
|
+
return output;
|
|
31830
|
+
}
|
|
31831
|
+
async function targetDetails(input) {
|
|
31832
|
+
const rows = [];
|
|
31833
|
+
let requestedPages = 0;
|
|
31834
|
+
let invalidRows = 0;
|
|
31835
|
+
let returnedRows = 0;
|
|
31836
|
+
let partial = false;
|
|
31837
|
+
let totalPages = 0;
|
|
31838
|
+
for (let page = 0; page < input.pages; page += 1) {
|
|
31839
|
+
requestedPages += 1;
|
|
31840
|
+
let response;
|
|
31841
|
+
try {
|
|
31842
|
+
response = await input.client.getUrlLinks(input.site, input.target.url, page);
|
|
31843
|
+
} catch (error) {
|
|
31844
|
+
if (error instanceof SeoError && (error.code === "AUTH_REQUIRED" || error.code === "ACCESS_DENIED")) throw error;
|
|
31845
|
+
return {
|
|
31846
|
+
rows,
|
|
31847
|
+
requestedPages,
|
|
31848
|
+
invalidRows,
|
|
31849
|
+
returnedRows,
|
|
31850
|
+
partial: true,
|
|
31851
|
+
failed: true
|
|
31852
|
+
};
|
|
31853
|
+
}
|
|
31854
|
+
invalidRows += response.invalidRows;
|
|
31855
|
+
returnedRows += response.returnedRows;
|
|
31856
|
+
rows.push(...response.rows);
|
|
31857
|
+
totalPages = response.totalPages;
|
|
31858
|
+
partial ||= response.capped;
|
|
31859
|
+
if (page + 1 >= response.totalPages || response.rows.length === 0) break;
|
|
31860
|
+
}
|
|
31861
|
+
partial ||= requestedPages < totalPages;
|
|
31862
|
+
return {
|
|
31863
|
+
rows,
|
|
31864
|
+
requestedPages,
|
|
31865
|
+
invalidRows,
|
|
31866
|
+
returnedRows,
|
|
31867
|
+
partial,
|
|
31868
|
+
failed: false
|
|
31869
|
+
};
|
|
31870
|
+
}
|
|
31871
|
+
async function collectBingLinkEvidence(input) {
|
|
31872
|
+
if (!input.site.trim()) throw new SeoError("INVALID_INPUT", "Pass a Bing Webmaster site URL.");
|
|
31873
|
+
const targetLimit = integer(input.targetLimit, DEFAULT_TARGET_LIMIT, MAX_TARGET_LIMIT, "Bing link target limit");
|
|
31874
|
+
const detailPages = integer(input.detailPagesPerTarget, DEFAULT_DETAIL_PAGES, MAX_DETAIL_PAGES, "Bing detail page limit");
|
|
31875
|
+
const rowLimit = integer(input.rowLimit, DEFAULT_ROW_LIMIT$2, MAX_ROW_LIMIT$2, "Bing link row limit");
|
|
31876
|
+
const client = input.client ?? (await createBingWebmasterClient()).client;
|
|
31877
|
+
const countRows = [];
|
|
31878
|
+
let countPagesRequested = 0;
|
|
31879
|
+
let invalidRows = 0;
|
|
31880
|
+
let countInvalidRows = 0;
|
|
31881
|
+
let countRowsReturned = 0;
|
|
31882
|
+
let partial = false;
|
|
31883
|
+
let totalCountPages = 0;
|
|
31884
|
+
for (let page = 0; page < MAX_COUNT_PAGES; page += 1) {
|
|
31885
|
+
const response = await client.getLinkCounts(input.site, page);
|
|
31886
|
+
countPagesRequested += 1;
|
|
31887
|
+
invalidRows += response.invalidRows;
|
|
31888
|
+
countInvalidRows += response.invalidRows;
|
|
31889
|
+
countRowsReturned += response.returnedRows;
|
|
31890
|
+
countRows.push(...response.rows);
|
|
31891
|
+
totalCountPages = response.totalPages;
|
|
31892
|
+
partial ||= response.capped;
|
|
31893
|
+
if (countRows.length >= targetLimit || page + 1 >= response.totalPages || response.rows.length === 0) break;
|
|
31894
|
+
}
|
|
31895
|
+
partial ||= countPagesRequested < totalCountPages;
|
|
31896
|
+
const countsByUrl = /* @__PURE__ */ new Map();
|
|
31897
|
+
for (const row of countRows) countsByUrl.set(row.url, Math.max(row.count, countsByUrl.get(row.url) ?? 0));
|
|
31898
|
+
const counts = [...countsByUrl].map(([url, count]) => ({
|
|
31899
|
+
url,
|
|
31900
|
+
count
|
|
31901
|
+
}));
|
|
31902
|
+
counts.sort((a, b) => b.count - a.count || a.url.localeCompare(b.url, "en"));
|
|
31903
|
+
const selectedTargets = counts.slice(0, targetLimit);
|
|
31904
|
+
partial ||= counts.length > targetLimit;
|
|
31905
|
+
const details = await mapConcurrent(selectedTargets, MAX_CONCURRENCY, async (target) => targetDetails({
|
|
31906
|
+
client,
|
|
31907
|
+
site: input.site,
|
|
31908
|
+
target,
|
|
31909
|
+
pages: detailPages
|
|
31910
|
+
}));
|
|
31911
|
+
const rows = [];
|
|
31912
|
+
const keys = /* @__PURE__ */ new Set();
|
|
31913
|
+
let duplicateRows = 0;
|
|
31914
|
+
let detailPagesRequested = 0;
|
|
31915
|
+
let detailInvalidRows = 0;
|
|
31916
|
+
let detailRowsReturned = 0;
|
|
31917
|
+
let failedTargets = 0;
|
|
31918
|
+
for (const [index, detail] of details.entries()) {
|
|
31919
|
+
const target = selectedTargets[index];
|
|
31920
|
+
if (!target) continue;
|
|
31921
|
+
detailPagesRequested += detail.requestedPages;
|
|
31922
|
+
if (detail.failed) failedTargets += 1;
|
|
31923
|
+
invalidRows += detail.invalidRows;
|
|
31924
|
+
detailInvalidRows += detail.invalidRows;
|
|
31925
|
+
detailRowsReturned += detail.returnedRows;
|
|
31926
|
+
partial ||= detail.partial;
|
|
31927
|
+
for (const source of detail.rows) {
|
|
31928
|
+
if (rows.length >= rowLimit) {
|
|
31929
|
+
partial = true;
|
|
31930
|
+
break;
|
|
31931
|
+
}
|
|
31932
|
+
const row = normalizeLinkEvidenceRow({
|
|
31933
|
+
sourceUrl: source.url,
|
|
31934
|
+
targetUrl: target.url,
|
|
31935
|
+
anchorText: source.anchorText
|
|
31936
|
+
});
|
|
31937
|
+
if (!row) {
|
|
31938
|
+
invalidRows += 1;
|
|
31939
|
+
continue;
|
|
31940
|
+
}
|
|
31941
|
+
const key = linkEvidenceKey(row);
|
|
31942
|
+
if (keys.has(key)) {
|
|
31943
|
+
duplicateRows += 1;
|
|
31944
|
+
continue;
|
|
31945
|
+
}
|
|
31946
|
+
keys.add(key);
|
|
31947
|
+
rows.push(row);
|
|
31948
|
+
}
|
|
31949
|
+
}
|
|
31950
|
+
rows.sort(compareLinkEvidence);
|
|
31951
|
+
return {
|
|
31952
|
+
rows,
|
|
31953
|
+
targetCounts: selectedTargets.map((target) => ({
|
|
31954
|
+
targetUrl: target.url,
|
|
31955
|
+
providerReportedLinks: target.count,
|
|
31956
|
+
observedLinks: rows.filter((row) => row.targetUrl === target.url).length
|
|
31957
|
+
})),
|
|
31958
|
+
provenance: {
|
|
31959
|
+
provider: "bing-webmaster",
|
|
31960
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
31961
|
+
cached: false,
|
|
31962
|
+
suppliedRows: detailRowsReturned + detailInvalidRows,
|
|
31963
|
+
validRows: rows.length,
|
|
31964
|
+
invalidRows,
|
|
31965
|
+
duplicateRows,
|
|
31966
|
+
capped: partial,
|
|
31967
|
+
rowLimit,
|
|
31968
|
+
completeness: partial ? "partial" : "unknown",
|
|
31969
|
+
providerRequests: {
|
|
31970
|
+
methods: ["GetLinkCounts", "GetUrlLinks"],
|
|
31971
|
+
targetPagesRequested: countPagesRequested,
|
|
31972
|
+
detailPagesRequested,
|
|
31973
|
+
maxConcurrentRequests: MAX_CONCURRENCY
|
|
31974
|
+
},
|
|
31975
|
+
providerCoverage: {
|
|
31976
|
+
targetCountRows: {
|
|
31977
|
+
returnedRows: countRowsReturned,
|
|
31978
|
+
retainedRows: counts.length,
|
|
31979
|
+
invalidRows: countInvalidRows
|
|
31980
|
+
},
|
|
31981
|
+
detailRows: {
|
|
31982
|
+
returnedRows: detailRowsReturned,
|
|
31983
|
+
retainedRows: rows.length,
|
|
31984
|
+
invalidRows: detailInvalidRows
|
|
31985
|
+
}
|
|
31986
|
+
}
|
|
31987
|
+
},
|
|
31988
|
+
warnings: [
|
|
31989
|
+
...countRowsReturned === 0 ? ["Bing returned no target-link rows. This does not prove that the site has no links."] : [],
|
|
31990
|
+
...partial ? ["Bing link evidence was bounded by provider pagination, failed requests, or report limits."] : [],
|
|
31991
|
+
...failedTargets ? [`${failedTargets} target detail request${failedTargets === 1 ? "" : "s"} failed while other evidence was retained.`] : []
|
|
31992
|
+
]
|
|
31993
|
+
};
|
|
30002
31994
|
}
|
|
30003
31995
|
//#endregion
|
|
30004
|
-
//#region packages/core/src/
|
|
30005
|
-
|
|
31996
|
+
//#region packages/core/src/links/import.ts
|
|
31997
|
+
const DEFAULT_ROW_LIMIT$1 = 1e4;
|
|
31998
|
+
const MAX_ROW_LIMIT$1 = 1e5;
|
|
31999
|
+
const MAX_STREAM_BYTES = 5e7;
|
|
32000
|
+
const MAX_JSON_BYTES = 1e7;
|
|
32001
|
+
const MAX_CELL_BYTES = 1e6;
|
|
32002
|
+
function formatFor$1(path, explicit) {
|
|
32003
|
+
if (explicit) return explicit;
|
|
32004
|
+
const extension = extname(path).toLowerCase();
|
|
32005
|
+
if (extension === ".csv") return "csv";
|
|
32006
|
+
if (extension === ".jsonl" || extension === ".ndjson") return "jsonl";
|
|
32007
|
+
if (extension === ".json") return "json";
|
|
32008
|
+
throw new SeoError("INVALID_INPUT", "Link imports must be CSV, JSON, JSONL, or NDJSON.");
|
|
32009
|
+
}
|
|
32010
|
+
function boundedRowLimit(value) {
|
|
32011
|
+
const limit = value ?? DEFAULT_ROW_LIMIT$1;
|
|
32012
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_ROW_LIMIT$1) throw new SeoError("INVALID_INPUT", `Link import row limit must be between 1 and ${MAX_ROW_LIMIT$1}.`);
|
|
32013
|
+
return limit;
|
|
32014
|
+
}
|
|
32015
|
+
async function* csvRecords(path, byteLimit, progress) {
|
|
32016
|
+
const stream = createReadStream(path, { encoding: "utf8" });
|
|
32017
|
+
let record = [];
|
|
32018
|
+
let field = "";
|
|
32019
|
+
let quoted = false;
|
|
32020
|
+
let pendingQuote = false;
|
|
32021
|
+
for await (const chunk of stream) {
|
|
32022
|
+
progress.bytesRead += Buffer.byteLength(chunk);
|
|
32023
|
+
if (progress.bytesRead > byteLimit) {
|
|
32024
|
+
stream.destroy();
|
|
32025
|
+
throw new SeoError("INVALID_INPUT", `Link import exceeds the ${byteLimit}-byte streaming limit.`);
|
|
32026
|
+
}
|
|
32027
|
+
for (const character of chunk) {
|
|
32028
|
+
if (pendingQuote) {
|
|
32029
|
+
if (character === "\"") {
|
|
32030
|
+
field += "\"";
|
|
32031
|
+
pendingQuote = false;
|
|
32032
|
+
continue;
|
|
32033
|
+
}
|
|
32034
|
+
quoted = false;
|
|
32035
|
+
pendingQuote = false;
|
|
32036
|
+
}
|
|
32037
|
+
if (quoted) if (character === "\"") pendingQuote = true;
|
|
32038
|
+
else field += character;
|
|
32039
|
+
else if (character === "\"" && field.length === 0) quoted = true;
|
|
32040
|
+
else if (character === ",") {
|
|
32041
|
+
record.push(field);
|
|
32042
|
+
field = "";
|
|
32043
|
+
} else if (character === "\n") {
|
|
32044
|
+
record.push(field.replace(/\r$/, ""));
|
|
32045
|
+
field = "";
|
|
32046
|
+
if (record.some((value) => value.length > 0)) yield record;
|
|
32047
|
+
record = [];
|
|
32048
|
+
} else field += character;
|
|
32049
|
+
if (Buffer.byteLength(field) > MAX_CELL_BYTES) {
|
|
32050
|
+
stream.destroy();
|
|
32051
|
+
throw new SeoError("INVALID_INPUT", `Link import contains a cell larger than ${MAX_CELL_BYTES} bytes.`);
|
|
32052
|
+
}
|
|
32053
|
+
}
|
|
32054
|
+
}
|
|
32055
|
+
if (quoted && !pendingQuote) throw new SeoError("INVALID_INPUT", "Link CSV contains an unclosed quote.");
|
|
32056
|
+
if (field || record.length) {
|
|
32057
|
+
record.push(field.replace(/\r$/, ""));
|
|
32058
|
+
if (record.some((value) => value.length > 0)) yield record;
|
|
32059
|
+
}
|
|
32060
|
+
}
|
|
32061
|
+
async function* jsonlRecords(path, byteLimit, progress) {
|
|
32062
|
+
const stream = createReadStream(path, { encoding: "utf8" });
|
|
32063
|
+
let remainder = "";
|
|
32064
|
+
for await (const chunk of stream) {
|
|
32065
|
+
progress.bytesRead += Buffer.byteLength(chunk);
|
|
32066
|
+
if (progress.bytesRead > byteLimit) {
|
|
32067
|
+
stream.destroy();
|
|
32068
|
+
throw new SeoError("INVALID_INPUT", `Link import exceeds the ${byteLimit}-byte streaming limit.`);
|
|
32069
|
+
}
|
|
32070
|
+
remainder += chunk;
|
|
32071
|
+
let newline = remainder.indexOf("\n");
|
|
32072
|
+
while (newline >= 0) {
|
|
32073
|
+
const line = remainder.slice(0, newline).trim();
|
|
32074
|
+
remainder = remainder.slice(newline + 1);
|
|
32075
|
+
if (line) yield parseObject(line);
|
|
32076
|
+
newline = remainder.indexOf("\n");
|
|
32077
|
+
}
|
|
32078
|
+
if (Buffer.byteLength(remainder) > MAX_CELL_BYTES) {
|
|
32079
|
+
stream.destroy();
|
|
32080
|
+
throw new SeoError("INVALID_INPUT", `Link import contains a line larger than ${MAX_CELL_BYTES} bytes.`);
|
|
32081
|
+
}
|
|
32082
|
+
}
|
|
32083
|
+
if (remainder.trim()) yield parseObject(remainder);
|
|
32084
|
+
}
|
|
32085
|
+
function parseObject(value) {
|
|
30006
32086
|
try {
|
|
30007
|
-
const
|
|
30008
|
-
if (
|
|
30009
|
-
return
|
|
32087
|
+
const parsed = JSON.parse(value);
|
|
32088
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("not an object");
|
|
32089
|
+
return parsed;
|
|
30010
32090
|
} catch {
|
|
30011
|
-
|
|
32091
|
+
throw new SeoError("INVALID_INPUT", "Link JSONL must contain one JSON object per line.");
|
|
32092
|
+
}
|
|
32093
|
+
}
|
|
32094
|
+
function rowFromCsv(headers, values) {
|
|
32095
|
+
return Object.fromEntries(headers.map((header, index) => [header, values[index]]));
|
|
32096
|
+
}
|
|
32097
|
+
async function importLinkEvidence(input) {
|
|
32098
|
+
const path = resolve(input.file);
|
|
32099
|
+
const file = await stat(path).catch(() => void 0);
|
|
32100
|
+
if (!file?.isFile()) throw new SeoError("INVALID_INPUT", `Link import file was not found: ${path}`);
|
|
32101
|
+
const format = formatFor$1(path, input.format);
|
|
32102
|
+
const rowLimit = boundedRowLimit(input.rowLimit);
|
|
32103
|
+
if (format === "json" && file.size > MAX_JSON_BYTES) throw new SeoError("INVALID_INPUT", `JSON link imports are limited to ${MAX_JSON_BYTES} bytes. Use CSV or JSONL for larger files.`);
|
|
32104
|
+
const progress = { bytesRead: 0 };
|
|
32105
|
+
let records;
|
|
32106
|
+
if (format === "csv") {
|
|
32107
|
+
const csv = csvRecords(path, MAX_STREAM_BYTES, progress);
|
|
32108
|
+
const first = await csv.next();
|
|
32109
|
+
const headers = (Array.isArray(first.value) ? first.value : []).map((value, index) => index === 0 ? value.replace(/^\uFEFF/, "") : value);
|
|
32110
|
+
if (!headers.length) throw new SeoError("INVALID_INPUT", "Link CSV has no header row.");
|
|
32111
|
+
records = (async function* () {
|
|
32112
|
+
for await (const values of csv) yield rowFromCsv(headers, values);
|
|
32113
|
+
})();
|
|
32114
|
+
} else if (format === "jsonl") records = jsonlRecords(path, MAX_STREAM_BYTES, progress);
|
|
32115
|
+
else {
|
|
32116
|
+
const body = await readFile(path, "utf8");
|
|
32117
|
+
progress.bytesRead = Buffer.byteLength(body);
|
|
32118
|
+
let parsed;
|
|
32119
|
+
try {
|
|
32120
|
+
parsed = JSON.parse(body);
|
|
32121
|
+
} catch {
|
|
32122
|
+
throw new SeoError("INVALID_INPUT", "Link JSON is not valid JSON.");
|
|
32123
|
+
}
|
|
32124
|
+
if (!Array.isArray(parsed)) throw new SeoError("INVALID_INPUT", "Link JSON must be an array of objects.");
|
|
32125
|
+
records = (async function* () {
|
|
32126
|
+
for (const row of parsed) yield row && typeof row === "object" && !Array.isArray(row) ? row : {};
|
|
32127
|
+
})();
|
|
30012
32128
|
}
|
|
32129
|
+
const rows = [];
|
|
32130
|
+
const keys = /* @__PURE__ */ new Set();
|
|
32131
|
+
let suppliedRows = 0;
|
|
32132
|
+
let invalidRows = 0;
|
|
32133
|
+
let duplicateRows = 0;
|
|
32134
|
+
let capped = false;
|
|
32135
|
+
for await (const raw of records) {
|
|
32136
|
+
if (suppliedRows >= rowLimit) {
|
|
32137
|
+
capped = true;
|
|
32138
|
+
break;
|
|
32139
|
+
}
|
|
32140
|
+
suppliedRows += 1;
|
|
32141
|
+
const row = normalizeLinkEvidenceRow(raw);
|
|
32142
|
+
if (!row) {
|
|
32143
|
+
invalidRows += 1;
|
|
32144
|
+
continue;
|
|
32145
|
+
}
|
|
32146
|
+
const key = linkEvidenceKey(row);
|
|
32147
|
+
if (keys.has(key)) {
|
|
32148
|
+
duplicateRows += 1;
|
|
32149
|
+
continue;
|
|
32150
|
+
}
|
|
32151
|
+
keys.add(key);
|
|
32152
|
+
rows.push(row);
|
|
32153
|
+
}
|
|
32154
|
+
rows.sort(compareLinkEvidence);
|
|
32155
|
+
const observedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
32156
|
+
return {
|
|
32157
|
+
rows,
|
|
32158
|
+
targetCounts: [],
|
|
32159
|
+
provenance: {
|
|
32160
|
+
provider: `${format}-import`,
|
|
32161
|
+
observedAt,
|
|
32162
|
+
cached: false,
|
|
32163
|
+
suppliedRows,
|
|
32164
|
+
validRows: rows.length,
|
|
32165
|
+
invalidRows,
|
|
32166
|
+
duplicateRows,
|
|
32167
|
+
capped,
|
|
32168
|
+
rowLimit,
|
|
32169
|
+
completeness: capped ? "partial" : "unknown",
|
|
32170
|
+
file: {
|
|
32171
|
+
path,
|
|
32172
|
+
format,
|
|
32173
|
+
bytesRead: progress.bytesRead,
|
|
32174
|
+
fileBytes: file.size
|
|
32175
|
+
}
|
|
32176
|
+
},
|
|
32177
|
+
warnings: capped ? [`Import stopped after ${rowLimit} rows. The file contains more data.`] : []
|
|
32178
|
+
};
|
|
30013
32179
|
}
|
|
30014
|
-
|
|
30015
|
-
|
|
30016
|
-
|
|
30017
|
-
|
|
30018
|
-
|
|
30019
|
-
|
|
30020
|
-
|
|
30021
|
-
|
|
30022
|
-
if (domain) return streamHostname === domain || streamHostname.endsWith(`.${domain}`) ? "domain" : void 0;
|
|
30023
|
-
return hostnameFromUrl(site) === streamHostname ? "hostname" : void 0;
|
|
32180
|
+
//#endregion
|
|
32181
|
+
//#region packages/core/src/links/report.ts
|
|
32182
|
+
const DEFAULT_OUTPUT_LIMIT = 100;
|
|
32183
|
+
const MAX_OUTPUT_LIMIT = 500;
|
|
32184
|
+
function boundedLimit(value) {
|
|
32185
|
+
const limit = value ?? DEFAULT_OUTPUT_LIMIT;
|
|
32186
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_OUTPUT_LIMIT) throw new SeoError("INVALID_INPUT", `Link report limit must be between 1 and ${MAX_OUTPUT_LIMIT}.`);
|
|
32187
|
+
return limit;
|
|
30024
32188
|
}
|
|
30025
|
-
function
|
|
30026
|
-
|
|
30027
|
-
|
|
30028
|
-
|
|
30029
|
-
|
|
30030
|
-
|
|
30031
|
-
|
|
30032
|
-
if (!match) return [];
|
|
30033
|
-
return [{
|
|
30034
|
-
...candidate,
|
|
30035
|
-
defaultUri,
|
|
30036
|
-
hostname,
|
|
30037
|
-
match
|
|
30038
|
-
}];
|
|
30039
|
-
});
|
|
32189
|
+
function targetCounts(rows) {
|
|
32190
|
+
const counts = /* @__PURE__ */ new Map();
|
|
32191
|
+
for (const row of rows) counts.set(row.targetUrl, (counts.get(row.targetUrl) ?? 0) + 1);
|
|
32192
|
+
return [...counts].map(([targetUrl, observedLinks]) => ({
|
|
32193
|
+
targetUrl,
|
|
32194
|
+
observedLinks
|
|
32195
|
+
})).sort((a, b) => b.observedLinks - a.observedLinks || a.targetUrl.localeCompare(b.targetUrl, "en"));
|
|
30040
32196
|
}
|
|
30041
|
-
function
|
|
30042
|
-
const
|
|
30043
|
-
const
|
|
30044
|
-
|
|
32197
|
+
function linkEvidenceReport(input) {
|
|
32198
|
+
const limit = boundedLimit(input.limit);
|
|
32199
|
+
const rows = input.evidence.rows.slice(0, limit);
|
|
32200
|
+
const domains = new Set(input.evidence.rows.map((row) => row.sourceDomain));
|
|
32201
|
+
const observedTargets = targetCounts(input.evidence.rows);
|
|
32202
|
+
const counts = input.evidence.targetCounts.length ? input.evidence.targetCounts : observedTargets;
|
|
32203
|
+
return {
|
|
32204
|
+
schemaVersion: 1,
|
|
32205
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
32206
|
+
dataStatus: input.evidence.provenance.completeness === "complete" ? "complete" : "partial",
|
|
32207
|
+
summary: {
|
|
32208
|
+
observedLinks: input.evidence.rows.length,
|
|
32209
|
+
referringDomains: domains.size,
|
|
32210
|
+
targetPages: new Set(input.evidence.rows.map((row) => row.targetUrl)).size,
|
|
32211
|
+
providerTargetPages: counts.length
|
|
32212
|
+
},
|
|
32213
|
+
provenance: input.evidence.provenance,
|
|
32214
|
+
selection: {
|
|
32215
|
+
availableRows: input.evidence.rows.length,
|
|
32216
|
+
returnedRows: rows.length,
|
|
32217
|
+
omittedRows: Math.max(0, input.evidence.rows.length - rows.length),
|
|
32218
|
+
limit
|
|
32219
|
+
},
|
|
32220
|
+
targetCounts: counts.slice(0, 100),
|
|
32221
|
+
links: rows,
|
|
32222
|
+
warnings: input.evidence.warnings,
|
|
32223
|
+
caveats: [
|
|
32224
|
+
"This is bounded provider or imported evidence, not a complete backlink index.",
|
|
32225
|
+
"A missing link is not proof that no link exists, and an observed link is not a quality or ranking verdict.",
|
|
32226
|
+
"Referring-domain counts describe the retained rows only."
|
|
32227
|
+
]
|
|
32228
|
+
};
|
|
30045
32229
|
}
|
|
30046
32230
|
//#endregion
|
|
30047
32231
|
//#region packages/core/src/storage/log-retention.ts
|
|
@@ -30418,6 +32602,472 @@ var JsonFileStorageAdapter = class {
|
|
|
30418
32602
|
};
|
|
30419
32603
|
const localCredentialsProvider = new LocalCredentialsProvider();
|
|
30420
32604
|
//#endregion
|
|
32605
|
+
//#region packages/core/src/server-logs/parse.ts
|
|
32606
|
+
const MONTHS = {
|
|
32607
|
+
Jan: 0,
|
|
32608
|
+
Feb: 1,
|
|
32609
|
+
Mar: 2,
|
|
32610
|
+
Apr: 3,
|
|
32611
|
+
May: 4,
|
|
32612
|
+
Jun: 5,
|
|
32613
|
+
Jul: 6,
|
|
32614
|
+
Aug: 7,
|
|
32615
|
+
Sep: 8,
|
|
32616
|
+
Oct: 9,
|
|
32617
|
+
Nov: 10,
|
|
32618
|
+
Dec: 11
|
|
32619
|
+
};
|
|
32620
|
+
const CRAWLERS = [
|
|
32621
|
+
{
|
|
32622
|
+
family: "Googlebot",
|
|
32623
|
+
category: "search",
|
|
32624
|
+
patterns: ["googlebot", "google-inspectiontool"]
|
|
32625
|
+
},
|
|
32626
|
+
{
|
|
32627
|
+
family: "Bingbot",
|
|
32628
|
+
category: "search",
|
|
32629
|
+
patterns: ["bingbot", "bingpreview"]
|
|
32630
|
+
},
|
|
32631
|
+
{
|
|
32632
|
+
family: "Applebot",
|
|
32633
|
+
category: "search",
|
|
32634
|
+
patterns: ["applebot"]
|
|
32635
|
+
},
|
|
32636
|
+
{
|
|
32637
|
+
family: "DuckDuckBot",
|
|
32638
|
+
category: "search",
|
|
32639
|
+
patterns: ["duckduckbot"]
|
|
32640
|
+
},
|
|
32641
|
+
{
|
|
32642
|
+
family: "YandexBot",
|
|
32643
|
+
category: "search",
|
|
32644
|
+
patterns: ["yandexbot"]
|
|
32645
|
+
},
|
|
32646
|
+
{
|
|
32647
|
+
family: "Baiduspider",
|
|
32648
|
+
category: "search",
|
|
32649
|
+
patterns: ["baiduspider"]
|
|
32650
|
+
},
|
|
32651
|
+
{
|
|
32652
|
+
family: "PetalBot",
|
|
32653
|
+
category: "search",
|
|
32654
|
+
patterns: ["petalbot"]
|
|
32655
|
+
},
|
|
32656
|
+
{
|
|
32657
|
+
family: "OpenAI",
|
|
32658
|
+
category: "ai",
|
|
32659
|
+
patterns: [
|
|
32660
|
+
"gptbot",
|
|
32661
|
+
"chatgpt-user",
|
|
32662
|
+
"oai-searchbot"
|
|
32663
|
+
]
|
|
32664
|
+
},
|
|
32665
|
+
{
|
|
32666
|
+
family: "Anthropic",
|
|
32667
|
+
category: "ai",
|
|
32668
|
+
patterns: [
|
|
32669
|
+
"claudebot",
|
|
32670
|
+
"claude-searchbot",
|
|
32671
|
+
"claude-user"
|
|
32672
|
+
]
|
|
32673
|
+
},
|
|
32674
|
+
{
|
|
32675
|
+
family: "Perplexity",
|
|
32676
|
+
category: "ai",
|
|
32677
|
+
patterns: ["perplexitybot", "perplexity-user"]
|
|
32678
|
+
},
|
|
32679
|
+
{
|
|
32680
|
+
family: "Amazonbot",
|
|
32681
|
+
category: "ai",
|
|
32682
|
+
patterns: ["amazonbot"]
|
|
32683
|
+
},
|
|
32684
|
+
{
|
|
32685
|
+
family: "Bytespider",
|
|
32686
|
+
category: "ai",
|
|
32687
|
+
patterns: ["bytespider"]
|
|
32688
|
+
},
|
|
32689
|
+
{
|
|
32690
|
+
family: "CCBot",
|
|
32691
|
+
category: "ai",
|
|
32692
|
+
patterns: ["ccbot"]
|
|
32693
|
+
},
|
|
32694
|
+
{
|
|
32695
|
+
family: "Cohere",
|
|
32696
|
+
category: "ai",
|
|
32697
|
+
patterns: ["cohere-ai"]
|
|
32698
|
+
}
|
|
32699
|
+
];
|
|
32700
|
+
const COMBINED_LOG = /^\S+\s+\S+\s+\S+\s+\[([^\]]+)]\s+"([^"\s]+)\s+([^"\s]+)(?:\s+[^"\s]+)?"\s+(\d{3})\s+(\d+|-)\s*(?:"[^"]*"\s+"([^"]*)")?\s*$/;
|
|
32701
|
+
function classifyCrawler(userAgent) {
|
|
32702
|
+
if (!userAgent) return void 0;
|
|
32703
|
+
const value = userAgent.toLowerCase();
|
|
32704
|
+
const match = CRAWLERS.find((crawler) => crawler.patterns.some((pattern) => value.includes(pattern)));
|
|
32705
|
+
return match ? {
|
|
32706
|
+
family: match.family,
|
|
32707
|
+
category: match.category
|
|
32708
|
+
} : void 0;
|
|
32709
|
+
}
|
|
32710
|
+
function logTimestamp(value) {
|
|
32711
|
+
const match = /^(\d{2})\/([A-Z][a-z]{2})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})$/.exec(value);
|
|
32712
|
+
if (!match) return void 0;
|
|
32713
|
+
const month = MONTHS[match[2] ?? ""];
|
|
32714
|
+
if (month === void 0) return void 0;
|
|
32715
|
+
const utc = Date.UTC(Number(match[3]), month, Number(match[1]), Number(match[4]), Number(match[5]), Number(match[6]));
|
|
32716
|
+
const offsetMinutes = Number(match[8]) * 60 + Number(match[9]);
|
|
32717
|
+
const adjusted = utc + (match[7] === "+" ? -1 : 1) * offsetMinutes * 6e4;
|
|
32718
|
+
const date = new Date(adjusted);
|
|
32719
|
+
return Number.isNaN(date.getTime()) ? void 0 : date.toISOString();
|
|
32720
|
+
}
|
|
32721
|
+
function normalizedPath(value) {
|
|
32722
|
+
if (typeof value !== "string" || value.length < 1 || value.length > 2e3) return;
|
|
32723
|
+
try {
|
|
32724
|
+
if (value.startsWith("/")) return new URL(value, "https://server-log.invalid").pathname;
|
|
32725
|
+
const url = new URL(value);
|
|
32726
|
+
return ["http:", "https:"].includes(url.protocol) ? url.pathname : void 0;
|
|
32727
|
+
} catch {
|
|
32728
|
+
return;
|
|
32729
|
+
}
|
|
32730
|
+
}
|
|
32731
|
+
function normalizedMethod(value) {
|
|
32732
|
+
if (typeof value !== "string" || !/^[A-Za-z]{1,16}$/.test(value)) return;
|
|
32733
|
+
return value.toUpperCase();
|
|
32734
|
+
}
|
|
32735
|
+
function normalizedStatus(value) {
|
|
32736
|
+
const status = typeof value === "number" ? value : Number(value);
|
|
32737
|
+
return Number.isInteger(status) && status >= 100 && status <= 599 ? status : void 0;
|
|
32738
|
+
}
|
|
32739
|
+
function normalizedBytes(value) {
|
|
32740
|
+
if (value === void 0 || value === null || value === "" || value === "-") return;
|
|
32741
|
+
const bytes = typeof value === "number" ? value : Number(value);
|
|
32742
|
+
return Number.isSafeInteger(bytes) && bytes >= 0 ? bytes : void 0;
|
|
32743
|
+
}
|
|
32744
|
+
function record(input) {
|
|
32745
|
+
let timestamp;
|
|
32746
|
+
if (typeof input.timestamp === "string") {
|
|
32747
|
+
const date = new Date(input.timestamp);
|
|
32748
|
+
if (!Number.isNaN(date.getTime())) timestamp = date.toISOString();
|
|
32749
|
+
}
|
|
32750
|
+
const method = normalizedMethod(input.method);
|
|
32751
|
+
const path = normalizedPath(input.path);
|
|
32752
|
+
const status = normalizedStatus(input.status);
|
|
32753
|
+
if (!timestamp || !method || !path || status === void 0) return void 0;
|
|
32754
|
+
const userAgent = typeof input.userAgent === "string" && input.userAgent.length <= 2e3 ? input.userAgent : void 0;
|
|
32755
|
+
return {
|
|
32756
|
+
timestamp,
|
|
32757
|
+
method,
|
|
32758
|
+
path,
|
|
32759
|
+
status,
|
|
32760
|
+
bytes: normalizedBytes(input.bytes),
|
|
32761
|
+
userAgent,
|
|
32762
|
+
crawler: classifyCrawler(userAgent)
|
|
32763
|
+
};
|
|
32764
|
+
}
|
|
32765
|
+
function parseCombinedLogLine(line) {
|
|
32766
|
+
const match = COMBINED_LOG.exec(line);
|
|
32767
|
+
if (!match) return void 0;
|
|
32768
|
+
const timestamp = logTimestamp(match[1] ?? "");
|
|
32769
|
+
if (!timestamp) return void 0;
|
|
32770
|
+
return record({
|
|
32771
|
+
timestamp,
|
|
32772
|
+
method: match[2],
|
|
32773
|
+
path: match[3],
|
|
32774
|
+
status: match[4],
|
|
32775
|
+
bytes: match[5],
|
|
32776
|
+
userAgent: match[6]
|
|
32777
|
+
});
|
|
32778
|
+
}
|
|
32779
|
+
function firstValue(value, names) {
|
|
32780
|
+
for (const name of names) if (value[name] !== void 0) return value[name];
|
|
32781
|
+
}
|
|
32782
|
+
function parseJsonLogLine(line) {
|
|
32783
|
+
let value;
|
|
32784
|
+
try {
|
|
32785
|
+
value = JSON.parse(line);
|
|
32786
|
+
} catch {
|
|
32787
|
+
return;
|
|
32788
|
+
}
|
|
32789
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return;
|
|
32790
|
+
const row = value;
|
|
32791
|
+
return record({
|
|
32792
|
+
timestamp: firstValue(row, [
|
|
32793
|
+
"timestamp",
|
|
32794
|
+
"time",
|
|
32795
|
+
"datetime",
|
|
32796
|
+
"@timestamp"
|
|
32797
|
+
]),
|
|
32798
|
+
method: firstValue(row, [
|
|
32799
|
+
"method",
|
|
32800
|
+
"requestMethod",
|
|
32801
|
+
"http_method"
|
|
32802
|
+
]),
|
|
32803
|
+
path: firstValue(row, [
|
|
32804
|
+
"path",
|
|
32805
|
+
"url",
|
|
32806
|
+
"requestUri",
|
|
32807
|
+
"request_uri"
|
|
32808
|
+
]),
|
|
32809
|
+
status: firstValue(row, [
|
|
32810
|
+
"status",
|
|
32811
|
+
"statusCode",
|
|
32812
|
+
"status_code"
|
|
32813
|
+
]),
|
|
32814
|
+
bytes: firstValue(row, [
|
|
32815
|
+
"bytes",
|
|
32816
|
+
"bodyBytesSent",
|
|
32817
|
+
"body_bytes_sent"
|
|
32818
|
+
]),
|
|
32819
|
+
userAgent: firstValue(row, [
|
|
32820
|
+
"userAgent",
|
|
32821
|
+
"user_agent",
|
|
32822
|
+
"httpUserAgent",
|
|
32823
|
+
"http_user_agent"
|
|
32824
|
+
])
|
|
32825
|
+
});
|
|
32826
|
+
}
|
|
32827
|
+
function emptyStatusBreakdown() {
|
|
32828
|
+
return {
|
|
32829
|
+
success: 0,
|
|
32830
|
+
redirect: 0,
|
|
32831
|
+
clientError: 0,
|
|
32832
|
+
serverError: 0,
|
|
32833
|
+
other: 0
|
|
32834
|
+
};
|
|
32835
|
+
}
|
|
32836
|
+
function addStatus(breakdown, status) {
|
|
32837
|
+
if (status >= 200 && status < 300) breakdown.success += 1;
|
|
32838
|
+
else if (status >= 300 && status < 400) breakdown.redirect += 1;
|
|
32839
|
+
else if (status >= 400 && status < 500) breakdown.clientError += 1;
|
|
32840
|
+
else if (status >= 500) breakdown.serverError += 1;
|
|
32841
|
+
else breakdown.other += 1;
|
|
32842
|
+
}
|
|
32843
|
+
//#endregion
|
|
32844
|
+
//#region packages/core/src/server-logs/import.ts
|
|
32845
|
+
const DEFAULT_ROW_LIMIT = 1e6;
|
|
32846
|
+
const MAX_ROW_LIMIT = 1e7;
|
|
32847
|
+
const DEFAULT_PATH_LIMIT = 25e3;
|
|
32848
|
+
const MAX_PATH_LIMIT = 1e5;
|
|
32849
|
+
const BYTE_LIMIT = 1e9;
|
|
32850
|
+
const MAX_LINE_BYTES = 65536;
|
|
32851
|
+
function boundedInteger(value, fallback, maximum, label) {
|
|
32852
|
+
const result = value ?? fallback;
|
|
32853
|
+
if (!Number.isInteger(result) || result < 1 || result > maximum) throw new SeoError("INVALID_INPUT", `${label} must be between 1 and ${maximum}.`);
|
|
32854
|
+
return result;
|
|
32855
|
+
}
|
|
32856
|
+
function formatFor(path, explicit) {
|
|
32857
|
+
if (explicit) return explicit;
|
|
32858
|
+
const extension = extname(path).toLowerCase();
|
|
32859
|
+
return [".jsonl", ".ndjson"].includes(extension) ? "jsonl" : "combined";
|
|
32860
|
+
}
|
|
32861
|
+
async function* lines(path, progress) {
|
|
32862
|
+
const stream = createReadStream(path, {
|
|
32863
|
+
encoding: "utf8",
|
|
32864
|
+
highWaterMark: 64 * 1024
|
|
32865
|
+
});
|
|
32866
|
+
let remainder = "";
|
|
32867
|
+
let discarding = false;
|
|
32868
|
+
for await (const chunk of stream) {
|
|
32869
|
+
const remainingBytes = BYTE_LIMIT - progress.bytesRead;
|
|
32870
|
+
if (remainingBytes <= 0) {
|
|
32871
|
+
progress.bytesCapped = true;
|
|
32872
|
+
break;
|
|
32873
|
+
}
|
|
32874
|
+
const chunkBytes = Buffer.byteLength(chunk);
|
|
32875
|
+
const accepted = chunkBytes > remainingBytes ? Buffer.from(chunk).subarray(0, remainingBytes).toString("utf8") : chunk;
|
|
32876
|
+
progress.bytesRead += Buffer.byteLength(accepted);
|
|
32877
|
+
if (chunkBytes > remainingBytes) progress.bytesCapped = true;
|
|
32878
|
+
remainder += accepted;
|
|
32879
|
+
let newline = remainder.indexOf("\n");
|
|
32880
|
+
while (newline >= 0) {
|
|
32881
|
+
const line = remainder.slice(0, newline).replace(/\r$/, "");
|
|
32882
|
+
remainder = remainder.slice(newline + 1);
|
|
32883
|
+
if (discarding || Buffer.byteLength(line) > MAX_LINE_BYTES) {
|
|
32884
|
+
yield { tooLong: true };
|
|
32885
|
+
discarding = false;
|
|
32886
|
+
} else if (line.trim()) yield { line };
|
|
32887
|
+
newline = remainder.indexOf("\n");
|
|
32888
|
+
}
|
|
32889
|
+
if (Buffer.byteLength(remainder) > MAX_LINE_BYTES) {
|
|
32890
|
+
remainder = "";
|
|
32891
|
+
discarding = true;
|
|
32892
|
+
}
|
|
32893
|
+
if (progress.bytesCapped) break;
|
|
32894
|
+
}
|
|
32895
|
+
if (discarding || Buffer.byteLength(remainder) > MAX_LINE_BYTES) yield { tooLong: true };
|
|
32896
|
+
else if (remainder.trim()) yield { line: remainder.replace(/\r$/, "") };
|
|
32897
|
+
}
|
|
32898
|
+
function compareCrawler(a, b) {
|
|
32899
|
+
return b.requests - a.requests || a.family.localeCompare(b.family, "en");
|
|
32900
|
+
}
|
|
32901
|
+
function compareCrawlerPath(a, b) {
|
|
32902
|
+
return b.requests - a.requests || a.family.localeCompare(b.family, "en") || a.path.localeCompare(b.path, "en");
|
|
32903
|
+
}
|
|
32904
|
+
function updatePeriod(timestamp, period) {
|
|
32905
|
+
if (!period.firstSeenAt || timestamp < period.firstSeenAt) period.firstSeenAt = timestamp;
|
|
32906
|
+
if (!period.lastSeenAt || timestamp > period.lastSeenAt) period.lastSeenAt = timestamp;
|
|
32907
|
+
}
|
|
32908
|
+
function newCrawler(record) {
|
|
32909
|
+
const crawler = record.crawler;
|
|
32910
|
+
if (!crawler) throw new Error("Crawler evidence is required.");
|
|
32911
|
+
return {
|
|
32912
|
+
family: crawler.family,
|
|
32913
|
+
category: crawler.category,
|
|
32914
|
+
requests: 0,
|
|
32915
|
+
lastSeenAt: record.timestamp,
|
|
32916
|
+
...emptyStatusBreakdown()
|
|
32917
|
+
};
|
|
32918
|
+
}
|
|
32919
|
+
function newCrawlerPath(record) {
|
|
32920
|
+
return {
|
|
32921
|
+
...newCrawler(record),
|
|
32922
|
+
path: record.path
|
|
32923
|
+
};
|
|
32924
|
+
}
|
|
32925
|
+
async function importServerLog(input) {
|
|
32926
|
+
const path = resolve(input.file);
|
|
32927
|
+
const file = await stat(path).catch(() => void 0);
|
|
32928
|
+
if (!file?.isFile()) throw new SeoError("INVALID_INPUT", `Server log was not found: ${path}`);
|
|
32929
|
+
const format = formatFor(path, input.format);
|
|
32930
|
+
const rowLimit = boundedInteger(input.rowLimit, DEFAULT_ROW_LIMIT, MAX_ROW_LIMIT, "Server log row limit");
|
|
32931
|
+
const pathLimit = boundedInteger(input.pathLimit, DEFAULT_PATH_LIMIT, MAX_PATH_LIMIT, "Server log path limit");
|
|
32932
|
+
const parse = format === "jsonl" ? parseJsonLogLine : parseCombinedLogLine;
|
|
32933
|
+
const progress = {
|
|
32934
|
+
bytesRead: 0,
|
|
32935
|
+
bytesCapped: false
|
|
32936
|
+
};
|
|
32937
|
+
const period = {};
|
|
32938
|
+
const statusCodes = /* @__PURE__ */ new Map();
|
|
32939
|
+
const crawlers = /* @__PURE__ */ new Map();
|
|
32940
|
+
const crawlerPaths = /* @__PURE__ */ new Map();
|
|
32941
|
+
let suppliedRows = 0;
|
|
32942
|
+
let parsedRows = 0;
|
|
32943
|
+
let invalidRows = 0;
|
|
32944
|
+
let crawlerRows = 0;
|
|
32945
|
+
let responseBytes = 0;
|
|
32946
|
+
let rowsCapped = false;
|
|
32947
|
+
let pathsCapped = false;
|
|
32948
|
+
let untrackedCrawlerPathRows = 0;
|
|
32949
|
+
for await (const item of lines(path, progress)) {
|
|
32950
|
+
if (suppliedRows >= rowLimit) {
|
|
32951
|
+
rowsCapped = true;
|
|
32952
|
+
break;
|
|
32953
|
+
}
|
|
32954
|
+
suppliedRows += 1;
|
|
32955
|
+
const parsed = item.line ? parse(item.line) : void 0;
|
|
32956
|
+
if (!parsed) {
|
|
32957
|
+
invalidRows += 1;
|
|
32958
|
+
continue;
|
|
32959
|
+
}
|
|
32960
|
+
parsedRows += 1;
|
|
32961
|
+
responseBytes += parsed.bytes ?? 0;
|
|
32962
|
+
updatePeriod(parsed.timestamp, period);
|
|
32963
|
+
statusCodes.set(parsed.status, (statusCodes.get(parsed.status) ?? 0) + 1);
|
|
32964
|
+
if (!parsed.crawler) continue;
|
|
32965
|
+
crawlerRows += 1;
|
|
32966
|
+
const crawler = crawlers.get(parsed.crawler.family) ?? newCrawler(parsed);
|
|
32967
|
+
crawler.requests += 1;
|
|
32968
|
+
if (parsed.timestamp > crawler.lastSeenAt) crawler.lastSeenAt = parsed.timestamp;
|
|
32969
|
+
addStatus(crawler, parsed.status);
|
|
32970
|
+
crawlers.set(parsed.crawler.family, crawler);
|
|
32971
|
+
const key = `${parsed.crawler.family}\u0000${parsed.path}`;
|
|
32972
|
+
let crawlerPath = crawlerPaths.get(key);
|
|
32973
|
+
if (!crawlerPath && crawlerPaths.size >= pathLimit) {
|
|
32974
|
+
pathsCapped = true;
|
|
32975
|
+
untrackedCrawlerPathRows += 1;
|
|
32976
|
+
continue;
|
|
32977
|
+
}
|
|
32978
|
+
crawlerPath ??= newCrawlerPath(parsed);
|
|
32979
|
+
crawlerPath.requests += 1;
|
|
32980
|
+
if (parsed.timestamp > crawlerPath.lastSeenAt) crawlerPath.lastSeenAt = parsed.timestamp;
|
|
32981
|
+
addStatus(crawlerPath, parsed.status);
|
|
32982
|
+
crawlerPaths.set(key, crawlerPath);
|
|
32983
|
+
}
|
|
32984
|
+
const fileReadCompletely = !rowsCapped && !progress.bytesCapped && progress.bytesRead >= file.size;
|
|
32985
|
+
const completeness = fileReadCompletely && invalidRows === 0 && !pathsCapped ? "complete" : "partial";
|
|
32986
|
+
const warnings = [];
|
|
32987
|
+
if (rowsCapped) warnings.push(`Analysis stopped after ${rowLimit} rows.`);
|
|
32988
|
+
if (progress.bytesCapped) warnings.push(`Analysis stopped after ${BYTE_LIMIT} input bytes.`);
|
|
32989
|
+
if (invalidRows) warnings.push(`${invalidRows} malformed or unsupported rows were skipped.`);
|
|
32990
|
+
if (pathsCapped) warnings.push(`Crawler path tracking stopped at ${pathLimit} unique crawler and path pairs. Overall crawler and status totals still include later valid rows.`);
|
|
32991
|
+
return {
|
|
32992
|
+
summary: {
|
|
32993
|
+
suppliedRows,
|
|
32994
|
+
parsedRows,
|
|
32995
|
+
invalidRows,
|
|
32996
|
+
crawlerRows,
|
|
32997
|
+
nonCrawlerRows: parsedRows - crawlerRows,
|
|
32998
|
+
responseBytes,
|
|
32999
|
+
...period
|
|
33000
|
+
},
|
|
33001
|
+
statusCodes: [...statusCodes].map(([status, requests]) => ({
|
|
33002
|
+
status,
|
|
33003
|
+
requests
|
|
33004
|
+
})).sort((a, b) => a.status - b.status),
|
|
33005
|
+
crawlers: [...crawlers.values()].sort(compareCrawler),
|
|
33006
|
+
crawlerPaths: [...crawlerPaths.values()].sort(compareCrawlerPath),
|
|
33007
|
+
provenance: {
|
|
33008
|
+
source: "local-server-log",
|
|
33009
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
33010
|
+
cached: false,
|
|
33011
|
+
file: {
|
|
33012
|
+
path,
|
|
33013
|
+
format,
|
|
33014
|
+
bytesRead: progress.bytesRead,
|
|
33015
|
+
fileBytes: file.size
|
|
33016
|
+
},
|
|
33017
|
+
limits: {
|
|
33018
|
+
rowLimit,
|
|
33019
|
+
pathLimit,
|
|
33020
|
+
byteLimit: BYTE_LIMIT,
|
|
33021
|
+
maxLineBytes: MAX_LINE_BYTES
|
|
33022
|
+
},
|
|
33023
|
+
coverage: {
|
|
33024
|
+
fileReadCompletely,
|
|
33025
|
+
rowsCapped,
|
|
33026
|
+
bytesCapped: progress.bytesCapped,
|
|
33027
|
+
pathsCapped,
|
|
33028
|
+
untrackedCrawlerPathRows
|
|
33029
|
+
},
|
|
33030
|
+
completeness
|
|
33031
|
+
},
|
|
33032
|
+
warnings
|
|
33033
|
+
};
|
|
33034
|
+
}
|
|
33035
|
+
//#endregion
|
|
33036
|
+
//#region packages/core/src/server-logs/report.ts
|
|
33037
|
+
const DEFAULT_LIMIT = 100;
|
|
33038
|
+
const MAX_LIMIT = 500;
|
|
33039
|
+
function outputLimit(value) {
|
|
33040
|
+
const limit = value ?? DEFAULT_LIMIT;
|
|
33041
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) throw new SeoError("INVALID_INPUT", `Server log output limit must be between 1 and ${MAX_LIMIT}.`);
|
|
33042
|
+
return limit;
|
|
33043
|
+
}
|
|
33044
|
+
function serverLogReport(input) {
|
|
33045
|
+
const limit = outputLimit(input.limit);
|
|
33046
|
+
const crawlerPaths = input.evidence.crawlerPaths.slice(0, limit);
|
|
33047
|
+
return {
|
|
33048
|
+
schemaVersion: 1,
|
|
33049
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
33050
|
+
dataStatus: input.evidence.provenance.completeness,
|
|
33051
|
+
summary: input.evidence.summary,
|
|
33052
|
+
statusCodes: input.evidence.statusCodes,
|
|
33053
|
+
crawlers: input.evidence.crawlers,
|
|
33054
|
+
crawlerPaths,
|
|
33055
|
+
selection: {
|
|
33056
|
+
availableCrawlerPaths: input.evidence.crawlerPaths.length,
|
|
33057
|
+
returnedCrawlerPaths: crawlerPaths.length,
|
|
33058
|
+
omittedCrawlerPaths: Math.max(0, input.evidence.crawlerPaths.length - crawlerPaths.length),
|
|
33059
|
+
limit
|
|
33060
|
+
},
|
|
33061
|
+
provenance: input.evidence.provenance,
|
|
33062
|
+
warnings: input.evidence.warnings,
|
|
33063
|
+
caveats: [
|
|
33064
|
+
"Crawler families are identified from user-agent strings, which can be missing or spoofed.",
|
|
33065
|
+
"This report describes only the supplied log file and cannot prove that every server or time period is represented.",
|
|
33066
|
+
"Crawler path rankings describe retained path aggregates when the path limit is reached."
|
|
33067
|
+
]
|
|
33068
|
+
};
|
|
33069
|
+
}
|
|
33070
|
+
//#endregion
|
|
30421
33071
|
//#region packages/core/src/telemetry.ts
|
|
30422
33072
|
const TELEMETRY_ENDPOINT = "https://seoskill.dev/api/t";
|
|
30423
33073
|
const TELEMETRY_SCHEMA_VERSION = 1;
|
|
@@ -30455,6 +33105,7 @@ const TELEMETRY_REPORTS = [
|
|
|
30455
33105
|
"ai-search-scorecard",
|
|
30456
33106
|
"audit-page",
|
|
30457
33107
|
"audit-urls",
|
|
33108
|
+
"bing-webmaster-overview",
|
|
30458
33109
|
"cannibalisation",
|
|
30459
33110
|
"community-intent",
|
|
30460
33111
|
"compare-crawls",
|
|
@@ -30473,6 +33124,8 @@ const TELEMETRY_REPORTS = [
|
|
|
30473
33124
|
"index-monitor",
|
|
30474
33125
|
"index-watch",
|
|
30475
33126
|
"internal-links",
|
|
33127
|
+
"link-evidence",
|
|
33128
|
+
"server-log-analysis",
|
|
30476
33129
|
"link-recovery",
|
|
30477
33130
|
"crawl-history",
|
|
30478
33131
|
"crawler-rules",
|
|
@@ -30814,6 +33467,6 @@ function isoDate(daysAgo = 0) {
|
|
|
30814
33467
|
return date.toISOString().slice(0, 10);
|
|
30815
33468
|
}
|
|
30816
33469
|
//#endregion
|
|
30817
|
-
export { AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, AI_SEARCH_SCORECARD_METHODOLOGY_ID, AI_SEARCH_SCORECARD_METHODOLOGY_VERSION, AUDIT_PAGE_RULE_IDS, AUTOMATIC_BASELINE_SITE_RETENTION, AUTOMATIC_BASELINE_TOTAL_RETENTION, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CRAWL_REPORT_STORAGE_VERSION, CRAWL_RUN_RETENTION, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_DEFAULT_MIN_IMPRESSIONS, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DataForSeoProvider, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, INDEX_WATCH_ATTEMPT_RETENTION, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LINK_RECOVER_RUN_RETENTION, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, OKF_DEFAULT_CONCEPTS, OKF_MAX_CONCEPTS, REPORT_BASELINE_MAX_AGE_MS, REPORT_BASELINE_MAX_DEPTH, REPORT_BASELINE_MAX_PAGES, ResponseSizeLimitError, SCORECARD_CHECK_WEIGHTS, SCORECARD_STATUS_CREDIT, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SemrushProvider, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_NOTICE, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, affectedUrls, agentReadiness, aiPromptsForQuery, aiReadiness, aiReferralSourceForValue, aiReferralsReport, aiSearchScorecard, allocateIndexUrlsToProperties, analyticsConnectionsSchema, analyzeAiReferralRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, attachCrawlRecommendations, auditCrawlPages, auditCrawlRequests, auditLlmsTxt, auditPage, authStatus, boundedSitemapInventory, buildOkfBundle, buildPseoAuditReportFromRows, buildTelemetryPayload, cannibalReport, changeMeasurementCsvRows, checkDatabaseReadiness, classifyCommunityIntent, clearCache, clientProfileSchema, clusterPseoTemplates, clusterQueryRows, collectAgentDiscovery, combineRobotsValues, communityIntentReport, compareCrawlPages, compareCrawlReports, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlDiff, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportStore, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createContentGroup, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, decayClusterDrafts, decayComparisonRange, decayingReport, deleteChange, deleteClient, deleteContentGroup, deleteCrawlReport, deleteTokens, deriveBrandTerms, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensureSeoCliDirs, entityReadiness, estimateSerpTitleWidth, expectedCtrForPosition, explainOkfValidation, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fetchText, fileSize, finalGscDateRange, findOverlappingSearchUpdates, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, generateLlmsTxt, geoGaps, geoGapsReport, getAuthModeStatus, getCacheStats, getChange, getClient, getClientConfig, getContentGroup, getDb, getKeywordProvider, getPreviousRun, getPrivacySnapshot, getRepeatedLinkRecoverUrls, getRunPages, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleAnalyticsPropertyId, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoveragePlan, indexCoverageSignals, indexMonitor, indexWatch, inferPseoTemplate, inferTemplateMovement, initializeTelemetry, insertCrawlRun, insertLinkRecoverRun, inspectUrl, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isCompatibleTechnicalBaseline, isCurrentTechnicalBaseline, isEntityEvidenceSchemaType, isSiteIdentitySchemaType, isSkippableReportError, isTelemetryCi, isTelemetryReport, isoDate, isoWeek, latestCrawlReport, latestCrawlSummaries, latestFinalGscDate, latestIndexWatchSummary, latestLinkRecoverSummary, linkEntries, linkRecover, linkRecoverRecommendation, listChanges, listClients, listContentGroups, listCrawlReports, listGa4AccountSummaries, listGa4DataStreams, listRules, listSearchUpdates, listSites, loadCrawlReport, localCredentialsProvider, loginWithLoopback, makeRunId, matchGa4WebStreams, measureChange, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, monitoringStatus, monthlyReport, monthlyReportWorkflow, narrativeCsvFiles, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, noteCacheWrite, okfConceptLimit, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, parsePseoPath, performanceAudit, performanceReportIsCacheable, planDueIndexUrls, planIndexCoverageFromUrls, plural, propertyMatchesUrl, providerPreferenceSchema, pruneIndexWatchSnapshots, pruneLogs, pseoAuditOptions, pseoAuditReport, pseoCsvFiles, pseoIndexStatus, pseoPresentation, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, readBoundedResponseText, readConfig, readOauthClient, readTokens, recommendCrawlDiffItem, recommendationForRule, recordChange, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderPseoMarkdown, renderWorkflowMarkdown, reportNarrative, reportPresentation, requestHeaders, resolveSegmentRanges, resolveSeoCliPaths, resolveTechnicalBaseline, reviewObservations, roundedPosition, runDoctor, runGa4Report, safeError, saveClient, saveCrawlReport, secondPage, segmentComparisonRange, segmentImpact, segmentRangeDays, selectAffectedUrls, selectDueIndexUrls, seoErrorEnvelope, seoToAiQueryReport, sessionSourceInListFilter, setDefaultClient, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteSchema, sqliteCrawlReportStore, strikingDistance, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, technicalWatchWorkflow, telemetryErrorCategory, templateForUrl, toSeoError, tokenSchema, topCrawlRecommendations, topFixes, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, unavailableSegmentImpactReport, unicodeTokens, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateOkfFiles, verifyQueryContent, workflowPresentation, writeConfig, writeOauthClient, writeTokens, xRobotsDirectives };
|
|
33470
|
+
export { AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, AI_SEARCH_SCORECARD_METHODOLOGY_ID, AI_SEARCH_SCORECARD_METHODOLOGY_VERSION, AUDIT_PAGE_RULE_IDS, AUTOMATIC_BASELINE_SITE_RETENTION, AUTOMATIC_BASELINE_TOTAL_RETENTION, BING_API_KEY_ENV, BING_API_KEY_SECRET, BROWSER_USER_AGENT, BingWebmasterClient, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CRAWL_REPORT_STORAGE_VERSION, CRAWL_RUN_RETENTION, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_DEFAULT_MIN_IMPRESSIONS, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DataForSeoProvider, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, INDEXNOW_ENDPOINT, INDEXNOW_KEYS_SECRET, INDEXNOW_KEY_ENV, INDEXNOW_MAX_URLS, INDEX_WATCH_ATTEMPT_RETENTION, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LINK_RECOVER_RUN_RETENTION, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, OKF_DEFAULT_CONCEPTS, OKF_MAX_CONCEPTS, REPORT_BASELINE_MAX_AGE_MS, REPORT_BASELINE_MAX_DEPTH, REPORT_BASELINE_MAX_PAGES, ResponseSizeLimitError, SCORECARD_CHECK_WEIGHTS, SCORECARD_STATUS_CREDIT, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SemrushProvider, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_NOTICE, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, addStatus, affectedUrls, agentReadiness, aiPromptsForQuery, aiReadiness, aiReferralSourceForValue, aiReferralsReport, aiSearchScorecard, allocateIndexUrlsToProperties, analyticsConnectionsSchema, analyzeAiReferralRows, analyzeBingCrawl, analyzeBingDimensions, analyzeBingTraffic, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, attachCrawlRecommendations, auditCrawlPages, auditCrawlRequests, auditLlmsTxt, auditPage, authStatus, bingAnalysisDefaults, bingWebmasterOverview, bingWebmasterSiteUrl, boundedSitemapInventory, buildOkfBundle, buildPseoAuditReportFromRows, buildTelemetryPayload, cannibalReport, changeMeasurementCsvRows, checkDatabaseReadiness, classifyCommunityIntent, classifyCrawler, clearCache, clientProfileSchema, clusterPseoTemplates, clusterQueryRows, collectAgentDiscovery, collectBingLinkEvidence, combineRobotsValues, communityIntentReport, compareCrawlPages, compareCrawlReports, compareLinkEvidence, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlDiff, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportStore, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createBingWebmasterClient, createContentGroup, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createIndexNowKey, createIndexNowKeyRecord, createLogger, createPageRenderer, ctrUnderperformersReport, decayClusterDrafts, decayComparisonRange, decayingReport, deleteChange, deleteClient, deleteContentGroup, deleteCrawlReport, deleteProviderSecret, deleteTokens, deriveBrandTerms, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, emptyStatusBreakdown, ensureSeoCliDirs, entityReadiness, estimateSerpTitleWidth, expectedCtrForPosition, explainOkfValidation, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fetchText, fileSize, finalGscDateRange, findOverlappingSearchUpdates, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, generateLlmsTxt, geoGaps, geoGapsReport, getAuthModeStatus, getCacheStats, getChange, getClient, getClientConfig, getContentGroup, getDb, getKeywordProvider, getPreviousRun, getPrivacySnapshot, getRepeatedLinkRecoverUrls, getRunPages, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleAnalyticsPropertyId, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, importLinkEvidence, importServerLog, indexCoveragePlan, indexCoverageSignals, indexMonitor, indexWatch, inferPseoTemplate, inferTemplateMovement, initializeTelemetry, insertCrawlRun, insertLinkRecoverRun, inspectUrl, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isCompatibleTechnicalBaseline, isCurrentTechnicalBaseline, isEntityEvidenceSchemaType, isSiteIdentitySchemaType, isSkippableReportError, isTelemetryCi, isTelemetryReport, isoDate, isoWeek, latestCrawlReport, latestCrawlSummaries, latestFinalGscDate, latestIndexWatchSummary, latestLinkRecoverSummary, linkEntries, linkEvidenceKey, linkEvidenceReport, linkRecover, linkRecoverRecommendation, listChanges, listClients, listContentGroups, listCrawlReports, listGa4AccountSummaries, listGa4DataStreams, listIndexNowKeys, listRules, listSearchUpdates, listSites, loadCrawlReport, localCredentialsProvider, loginWithLoopback, makeRunId, matchGa4WebStreams, measureChange, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, monitoringStatus, monthlyReport, monthlyReportWorkflow, narrativeCsvFiles, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLinkEvidenceRow, normalizeLoadedCrawlReport, noteCacheWrite, okfConceptLimit, pageEvidenceRecommendation, pageOpportunitiesReport, parseCombinedLogLine, parseIndexNowSite, parseJsonLogLine, parseOlderThan, parsePseoPath, performanceAudit, performanceReportIsCacheable, planDueIndexUrls, planIndexCoverageFromUrls, plural, propertyMatchesUrl, providerPreferenceSchema, pruneIndexWatchSnapshots, pruneLogs, pseoAuditOptions, pseoAuditReport, pseoCsvFiles, pseoIndexStatus, pseoPresentation, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, readBoundedResponseText, readConfig, readOauthClient, readProviderSecret, readTokens, recommendCrawlDiffItem, recommendationForRule, recordChange, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, removeIndexNowKey, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderPseoMarkdown, renderReportHtml, renderWorkflowMarkdown, reportNarrative, reportPresentation, requestHeaders, resolveIndexNowKey, resolveSegmentRanges, resolveSeoCliPaths, resolveTechnicalBaseline, reviewObservations, roundedPosition, runDoctor, runGa4Report, safeError, saveClient, saveCrawlReport, saveIndexNowKey, secondPage, segmentComparisonRange, segmentImpact, segmentRangeDays, selectAffectedUrls, selectDueIndexUrls, seoErrorEnvelope, seoToAiQueryReport, serverLogReport, sessionSourceInListFilter, setClientBingSite, setDefaultClient, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteSchema, sqliteCrawlReportStore, strikingDistance, submitIndexNow, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, technicalWatchWorkflow, telemetryErrorCategory, templateForUrl, toSeoError, tokenSchema, topCrawlRecommendations, topFixes, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, unavailableSegmentImpactReport, unicodeTokens, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateIndexNowKey, validateIndexNowKeyRecord, validateOkfFiles, verifyIndexNowKey, verifyQueryContent, workflowPresentation, writeConfig, writeOauthClient, writeProviderSecret, writeTokens, xRobotsDirectives };
|
|
30818
33471
|
|
|
30819
33472
|
//# sourceMappingURL=index.js.map
|