seo 0.2.10 → 0.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -1
- package/dist/cli.js +217 -112
- package/dist/cli.js.map +1 -1
- package/dist/{dist-CBY8QOnv.js → dist-D0EI_L8C.js} +1 -1
- package/dist/{dist-w2XtKwjW.js → dist-DaHrxVkW.js} +85 -80
- package/dist/dist-DaHrxVkW.js.map +1 -0
- package/dist/index.d.ts +1185 -61
- package/dist/index.js +2571 -320
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +2322 -274
- package/dist/mcp.js.map +1 -1
- package/dist/{package-92W1Zc-o.js → package-B-M-4PZ_.js} +2 -2
- package/dist/{package-92W1Zc-o.js.map → package-B-M-4PZ_.js.map} +1 -1
- package/evals/bing-webmaster-overview.json +31 -0
- package/evals/link-evidence.json +29 -0
- package/evals/seo.json +16 -0
- package/evals/server-log-analysis.json +29 -0
- package/package.json +3 -3
- package/skills/seo/SKILL.md +56 -67
- package/dist/dist-w2XtKwjW.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { n as load, t as version } from "./package-
|
|
2
|
+
import { n as load, t as version } from "./package-B-M-4PZ_.js";
|
|
3
3
|
import { Headers, fetch as fetch$1 } from "undici";
|
|
4
|
-
import { chmodSync, createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { chmodSync, createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
5
5
|
import { arch, homedir, platform, totalmem } from "node:os";
|
|
6
|
-
import { basename, dirname, join } from "node:path";
|
|
6
|
+
import { basename, dirname, extname, join, resolve } from "node:path";
|
|
7
7
|
import { z } from "zod";
|
|
8
8
|
import lockfile from "proper-lockfile";
|
|
9
|
-
import crypto, { createHash, randomUUID, sign } from "node:crypto";
|
|
9
|
+
import crypto, { createHash, randomBytes, randomUUID, sign } from "node:crypto";
|
|
10
10
|
import http from "node:http";
|
|
11
11
|
import open from "open";
|
|
12
12
|
import Libsql from "libsql";
|
|
@@ -21,9 +21,9 @@ import { gunzipSync } from "node:zlib";
|
|
|
21
21
|
import { execFile } from "node:child_process";
|
|
22
22
|
import { fileURLToPath } from "node:url";
|
|
23
23
|
import { promisify } from "node:util";
|
|
24
|
+
import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
24
25
|
import pino from "pino";
|
|
25
26
|
import pretty from "pino-pretty";
|
|
26
|
-
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
27
27
|
//#region packages/core/src/errors.ts
|
|
28
28
|
const errorMetadata = {
|
|
29
29
|
ACCESS_DENIED: {
|
|
@@ -83,12 +83,12 @@ var SeoError = class extends Error {
|
|
|
83
83
|
this.retryable = errorMetadata[code].retryable;
|
|
84
84
|
}
|
|
85
85
|
};
|
|
86
|
-
function errorMessage$
|
|
86
|
+
function errorMessage$3(error) {
|
|
87
87
|
return error instanceof Error ? error.message : String(error);
|
|
88
88
|
}
|
|
89
89
|
function toSeoError(error) {
|
|
90
90
|
if (error instanceof SeoError) return error;
|
|
91
|
-
const message = errorMessage$
|
|
91
|
+
const message = errorMessage$3(error);
|
|
92
92
|
if (/not logged in/i.test(message)) return new SeoError("AUTH_REQUIRED", message);
|
|
93
93
|
if (/oauth client config.*missing|shared seo google app/i.test(message)) return new SeoError("AUTH_CONFIG_REQUIRED", message);
|
|
94
94
|
if (/invalid_grant|refresh token.*no longer valid/i.test(message)) return new SeoError("AUTH_EXPIRED", message);
|
|
@@ -145,6 +145,7 @@ function resolveSeoCliPaths(input = {}) {
|
|
|
145
145
|
configFile: join(configDir, "config.json"),
|
|
146
146
|
tokensFile: join(configDir, "tokens.json"),
|
|
147
147
|
oauthClientFile: join(configDir, "oauth-client.json"),
|
|
148
|
+
providerSecretsFile: join(configDir, "provider-secrets.json"),
|
|
148
149
|
telemetryStateFile: join(configDir, "telemetry.json"),
|
|
149
150
|
cacheDbFile: join(cacheDir, "cache.db")
|
|
150
151
|
};
|
|
@@ -231,6 +232,7 @@ const clientProfileSchema = z.object({
|
|
|
231
232
|
watchUrls: z.array(z.string()).default([]),
|
|
232
233
|
brandTerms: z.array(z.string()).default([]),
|
|
233
234
|
analytics: analyticsConnectionsSchema,
|
|
235
|
+
searchEngines: z.object({ bing: z.object({ siteUrl: z.string() }).optional() }).optional(),
|
|
234
236
|
reportDay: z.number().int().min(1).max(31).optional(),
|
|
235
237
|
technicalWeekday: z.number().int().min(0).max(7).optional(),
|
|
236
238
|
isDefault: z.boolean().optional(),
|
|
@@ -316,11 +318,11 @@ async function deleteKeyringPassword(service, account) {
|
|
|
316
318
|
}
|
|
317
319
|
//#endregion
|
|
318
320
|
//#region packages/core/src/storage/config.ts
|
|
319
|
-
const KEYRING_SERVICE = "seo";
|
|
320
|
-
const PRIVATE_FILE_MODE = 384;
|
|
321
|
+
const KEYRING_SERVICE$1 = "seo";
|
|
322
|
+
const PRIVATE_FILE_MODE$1 = 384;
|
|
321
323
|
function tightenConfigPermissions(path) {
|
|
322
324
|
try {
|
|
323
|
-
chmodSync(path, PRIVATE_FILE_MODE);
|
|
325
|
+
chmodSync(path, PRIVATE_FILE_MODE$1);
|
|
324
326
|
} catch (error) {
|
|
325
327
|
if (error.code !== "ENOENT") throw error;
|
|
326
328
|
}
|
|
@@ -334,7 +336,7 @@ function readConfig() {
|
|
|
334
336
|
}
|
|
335
337
|
function writeConfig(config) {
|
|
336
338
|
ensureSeoCliDirs();
|
|
337
|
-
writeJsonAtomic(getSeoCliPaths().configFile, configSchema.parse(config), PRIVATE_FILE_MODE);
|
|
339
|
+
writeJsonAtomic(getSeoCliPaths().configFile, configSchema.parse(config), PRIVATE_FILE_MODE$1);
|
|
338
340
|
}
|
|
339
341
|
function getKeyringAccount(tokens) {
|
|
340
342
|
return `google:${tokens.account_email}`;
|
|
@@ -354,7 +356,7 @@ function tokenMetadata(tokens) {
|
|
|
354
356
|
}
|
|
355
357
|
async function readKeyringTokens(tokens) {
|
|
356
358
|
const account = getKeyringAccount(tokens);
|
|
357
|
-
const [accessToken, refreshToken] = await Promise.all([getKeyringPassword(KEYRING_SERVICE, `${account}:access`), getKeyringPassword(KEYRING_SERVICE, `${account}:refresh`)]);
|
|
359
|
+
const [accessToken, refreshToken] = await Promise.all([getKeyringPassword(KEYRING_SERVICE$1, `${account}:access`), getKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`)]);
|
|
358
360
|
return {
|
|
359
361
|
accessToken: accessToken ?? void 0,
|
|
360
362
|
refreshToken: refreshToken ?? void 0
|
|
@@ -362,14 +364,14 @@ async function readKeyringTokens(tokens) {
|
|
|
362
364
|
}
|
|
363
365
|
async function writeKeyringTokens(tokens) {
|
|
364
366
|
const account = getKeyringAccount(tokens);
|
|
365
|
-
if (tokens.access_token) await setKeyringPassword(KEYRING_SERVICE, `${account}:access`, tokens.access_token);
|
|
366
|
-
else await deleteKeyringPassword(KEYRING_SERVICE, `${account}:access`);
|
|
367
|
-
if (tokens.refresh_token) await setKeyringPassword(KEYRING_SERVICE, `${account}:refresh`, tokens.refresh_token);
|
|
368
|
-
else await deleteKeyringPassword(KEYRING_SERVICE, `${account}:refresh`);
|
|
367
|
+
if (tokens.access_token) await setKeyringPassword(KEYRING_SERVICE$1, `${account}:access`, tokens.access_token);
|
|
368
|
+
else await deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:access`);
|
|
369
|
+
if (tokens.refresh_token) await setKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`, tokens.refresh_token);
|
|
370
|
+
else await deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`);
|
|
369
371
|
}
|
|
370
372
|
async function deleteKeyringTokens(tokens) {
|
|
371
373
|
const account = getKeyringAccount(tokens);
|
|
372
|
-
await Promise.all([deleteKeyringPassword(KEYRING_SERVICE, `${account}:access`), deleteKeyringPassword(KEYRING_SERVICE, `${account}:refresh`)]);
|
|
374
|
+
await Promise.all([deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:access`), deleteKeyringPassword(KEYRING_SERVICE$1, `${account}:refresh`)]);
|
|
373
375
|
}
|
|
374
376
|
async function readTokens() {
|
|
375
377
|
ensureSeoCliDirs();
|
|
@@ -381,7 +383,7 @@ async function readTokens() {
|
|
|
381
383
|
const keyringTokens = await readKeyringTokens(parsed);
|
|
382
384
|
if (hasTokenSecrets(parsed)) {
|
|
383
385
|
await writeKeyringTokens(parsed);
|
|
384
|
-
writeJsonAtomic(getSeoCliPaths().tokensFile, tokenMetadata(parsed), PRIVATE_FILE_MODE);
|
|
386
|
+
writeJsonAtomic(getSeoCliPaths().tokensFile, tokenMetadata(parsed), PRIVATE_FILE_MODE$1);
|
|
385
387
|
return parsed;
|
|
386
388
|
}
|
|
387
389
|
return {
|
|
@@ -398,10 +400,10 @@ async function writeTokens(tokens) {
|
|
|
398
400
|
const parsed = tokenSchema.parse(tokens);
|
|
399
401
|
if (readConfig().security.useKeychain) try {
|
|
400
402
|
await writeKeyringTokens(parsed);
|
|
401
|
-
writeJsonAtomic(getSeoCliPaths().tokensFile, tokenMetadata(parsed), PRIVATE_FILE_MODE);
|
|
403
|
+
writeJsonAtomic(getSeoCliPaths().tokensFile, tokenMetadata(parsed), PRIVATE_FILE_MODE$1);
|
|
402
404
|
return;
|
|
403
405
|
} catch {}
|
|
404
|
-
writeJsonAtomic(getSeoCliPaths().tokensFile, parsed, PRIVATE_FILE_MODE);
|
|
406
|
+
writeJsonAtomic(getSeoCliPaths().tokensFile, parsed, PRIVATE_FILE_MODE$1);
|
|
405
407
|
}
|
|
406
408
|
async function deleteTokens() {
|
|
407
409
|
const tokens = await readTokens();
|
|
@@ -493,6 +495,12 @@ function getPrivacySnapshot() {
|
|
|
493
495
|
mode: fileMode(paths.oauthClientFile),
|
|
494
496
|
sizeBytes: sizeOf(paths.oauthClientFile)
|
|
495
497
|
},
|
|
498
|
+
{
|
|
499
|
+
label: "Provider secrets",
|
|
500
|
+
path: paths.providerSecretsFile,
|
|
501
|
+
mode: fileMode(paths.providerSecretsFile),
|
|
502
|
+
sizeBytes: sizeOf(paths.providerSecretsFile)
|
|
503
|
+
},
|
|
496
504
|
{
|
|
497
505
|
label: "Telemetry state",
|
|
498
506
|
path: paths.telemetryStateFile,
|
|
@@ -3157,7 +3165,7 @@ function absoluteUrl$2(value, base) {
|
|
|
3157
3165
|
return;
|
|
3158
3166
|
}
|
|
3159
3167
|
}
|
|
3160
|
-
function httpUrl(value, base) {
|
|
3168
|
+
function httpUrl$1(value, base) {
|
|
3161
3169
|
const resolved = absoluteUrl$2(value, base);
|
|
3162
3170
|
if (!resolved) return void 0;
|
|
3163
3171
|
const protocol = new URL(resolved).protocol;
|
|
@@ -3309,14 +3317,14 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
3309
3317
|
let invalidHreflangCount = 0;
|
|
3310
3318
|
for (const element of $("link[rel~=\"alternate\"][hreflang][href]").toArray()) {
|
|
3311
3319
|
const language = $(element).attr("hreflang") ?? "";
|
|
3312
|
-
const href = httpUrl($(element).attr("href"), fetchResult.finalUrl);
|
|
3320
|
+
const href = httpUrl$1($(element).attr("href"), fetchResult.finalUrl);
|
|
3313
3321
|
if (language && href) hreflang.push({
|
|
3314
3322
|
hreflang: language,
|
|
3315
3323
|
href
|
|
3316
3324
|
});
|
|
3317
3325
|
else invalidHreflangCount += 1;
|
|
3318
3326
|
}
|
|
3319
|
-
const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
|
|
3327
|
+
const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl$1($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
|
|
3320
3328
|
const openGraph = Object.fromEntries($("meta[property^=\"og:\"]").toArray().map((element) => [$(element).attr("property") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
3321
3329
|
const twitter = Object.fromEntries($("meta[name^=\"twitter:\"]").toArray().map((element) => [$(element).attr("name") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
|
|
3322
3330
|
const structuredData = extractStructuredData($, fetchResult.finalUrl, { retainJsonLd: extractor !== "crawler" });
|
|
@@ -4735,7 +4743,7 @@ function urlInspectionQuotaDate(now) {
|
|
|
4735
4743
|
function urlInspectionQuotaResetAt(now) {
|
|
4736
4744
|
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1));
|
|
4737
4745
|
}
|
|
4738
|
-
function boundedLimit(value) {
|
|
4746
|
+
function boundedLimit$1(value) {
|
|
4739
4747
|
const limit = value ?? 2e3;
|
|
4740
4748
|
if (!Number.isInteger(limit) || limit < 1 || limit > 2e3) throw new SeoError("INVALID_INPUT", "URL Inspection daily limit must be an integer from 1 to 2000.");
|
|
4741
4749
|
return limit;
|
|
@@ -4768,7 +4776,7 @@ function reserveUrlInspectionQuota(input) {
|
|
|
4768
4776
|
const database = input.database ?? getDb();
|
|
4769
4777
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
4770
4778
|
const nowMs = now.getTime();
|
|
4771
|
-
const requestedLimit = boundedLimit(input.limit);
|
|
4779
|
+
const requestedLimit = boundedLimit$1(input.limit);
|
|
4772
4780
|
const quotaDate = urlInspectionQuotaDate(now);
|
|
4773
4781
|
const resetAt = urlInspectionQuotaResetAt(now);
|
|
4774
4782
|
const result = database.transaction(() => {
|
|
@@ -6674,13 +6682,13 @@ function isLowActionabilityQuery(query) {
|
|
|
6674
6682
|
//#endregion
|
|
6675
6683
|
//#region packages/core/src/analyze/page-opportunities-selection.ts
|
|
6676
6684
|
const DEFAULT_MIN_IMPRESSIONS$3 = 10;
|
|
6677
|
-
const DEFAULT_LIMIT$
|
|
6678
|
-
const MAX_LIMIT$
|
|
6685
|
+
const DEFAULT_LIMIT$4 = 25;
|
|
6686
|
+
const MAX_LIMIT$4 = 100;
|
|
6679
6687
|
function integerInRange$2(value, fallback, minimum, maximum) {
|
|
6680
6688
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
6681
6689
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
6682
6690
|
}
|
|
6683
|
-
function isHttpUrl$
|
|
6691
|
+
function isHttpUrl$2(value) {
|
|
6684
6692
|
try {
|
|
6685
6693
|
const parsed = new URL(value);
|
|
6686
6694
|
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
@@ -6702,7 +6710,7 @@ function isBrandRow(input) {
|
|
|
6702
6710
|
function normalizePageOpportunityOptions(input) {
|
|
6703
6711
|
return {
|
|
6704
6712
|
minImpressions: integerInRange$2(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$3, 1, Number.MAX_SAFE_INTEGER),
|
|
6705
|
-
limit: integerInRange$2(input.limit, DEFAULT_LIMIT$
|
|
6713
|
+
limit: integerInRange$2(input.limit, DEFAULT_LIMIT$4, 1, MAX_LIMIT$4)
|
|
6706
6714
|
};
|
|
6707
6715
|
}
|
|
6708
6716
|
function selectTargetRows(input) {
|
|
@@ -6721,7 +6729,7 @@ function selectTargetRows(input) {
|
|
|
6721
6729
|
for (const row of input.rows) {
|
|
6722
6730
|
const query = row.keys[0]?.trim() ?? "";
|
|
6723
6731
|
const page = row.keys[1] ?? "";
|
|
6724
|
-
if (row.keys.length < 2 || !query || !isHttpUrl$
|
|
6732
|
+
if (row.keys.length < 2 || !query || !isHttpUrl$2(page) || !hasValidMetrics(row)) {
|
|
6725
6733
|
selection.invalidRows++;
|
|
6726
6734
|
continue;
|
|
6727
6735
|
}
|
|
@@ -6757,7 +6765,7 @@ function selectTargetRows(input) {
|
|
|
6757
6765
|
function selectBenchmarkRows$1(input) {
|
|
6758
6766
|
return input.rows.filter((row) => {
|
|
6759
6767
|
const query = row.keys[0]?.trim() ?? "";
|
|
6760
|
-
return row.keys.length >= 2 && isHttpUrl$
|
|
6768
|
+
return row.keys.length >= 2 && isHttpUrl$2(row.keys[1] ?? "") && hasValidMetrics(row) && row.position >= 1 && row.position <= 10 && Boolean(query) && !isLowActionabilityQuery(query) && !isBrandRow({
|
|
6761
6769
|
query,
|
|
6762
6770
|
site: input.site,
|
|
6763
6771
|
brandTerms: input.brandTerms,
|
|
@@ -6939,7 +6947,7 @@ const defaultDependencies$15 = {
|
|
|
6939
6947
|
extract: extractPage,
|
|
6940
6948
|
now: () => /* @__PURE__ */ new Date()
|
|
6941
6949
|
};
|
|
6942
|
-
function errorMessage$
|
|
6950
|
+
function errorMessage$2(error) {
|
|
6943
6951
|
return error instanceof Error ? error.message : String(error);
|
|
6944
6952
|
}
|
|
6945
6953
|
function unique$4(values) {
|
|
@@ -6960,7 +6968,7 @@ async function verifyPage(input) {
|
|
|
6960
6968
|
});
|
|
6961
6969
|
warnings.push(...fetched.warnings);
|
|
6962
6970
|
} catch (error) {
|
|
6963
|
-
const reason = `Page fetch failed: ${errorMessage$
|
|
6971
|
+
const reason = `Page fetch failed: ${errorMessage$2(error)}`;
|
|
6964
6972
|
return {
|
|
6965
6973
|
verification: {
|
|
6966
6974
|
status: "failed",
|
|
@@ -6982,7 +6990,7 @@ async function verifyPage(input) {
|
|
|
6982
6990
|
warnings: unique$4(warnings)
|
|
6983
6991
|
};
|
|
6984
6992
|
} catch (error) {
|
|
6985
|
-
const reason = `Page extraction failed: ${errorMessage$
|
|
6993
|
+
const reason = `Page extraction failed: ${errorMessage$2(error)}`;
|
|
6986
6994
|
return {
|
|
6987
6995
|
fetched,
|
|
6988
6996
|
verification: {
|
|
@@ -8177,7 +8185,7 @@ function check$1(section, input) {
|
|
|
8177
8185
|
...input
|
|
8178
8186
|
};
|
|
8179
8187
|
}
|
|
8180
|
-
function section$
|
|
8188
|
+
function section$2(id, title, checks) {
|
|
8181
8189
|
return {
|
|
8182
8190
|
id,
|
|
8183
8191
|
title,
|
|
@@ -8615,19 +8623,19 @@ function agentReadiness(report) {
|
|
|
8615
8623
|
headline: "No agent-discovery evidence is stored in this crawl. Run a fresh focused check.",
|
|
8616
8624
|
profileApplicability: profiles,
|
|
8617
8625
|
summary: summariseChecks(checks),
|
|
8618
|
-
sections: [section$
|
|
8626
|
+
sections: [section$2("collection", "Evidence collection", [missing]), section$2("profile", "Profile scope", scopeChecks)],
|
|
8619
8627
|
checks,
|
|
8620
8628
|
topActions: [missing],
|
|
8621
8629
|
caveats: ["Not evaluated does not mean absent or broken.", "This report does not measure rankings, AI mentions, citations, or selection."]
|
|
8622
8630
|
};
|
|
8623
8631
|
}
|
|
8624
8632
|
const sections = [
|
|
8625
|
-
section$
|
|
8626
|
-
section$
|
|
8627
|
-
section$
|
|
8628
|
-
section$
|
|
8629
|
-
section$
|
|
8630
|
-
section$
|
|
8633
|
+
section$2("profile", "Profile scope", profileChecks(discovery)),
|
|
8634
|
+
section$2("access", "Crawler and protocol access", accessChecks(report, discovery)),
|
|
8635
|
+
section$2("representations", "HTML and Markdown representations", markdownChecks(discovery)),
|
|
8636
|
+
section$2("discovery", "Agent discovery files", discoveryChecks(discovery, report.ai?.robotsTxt?.contentSignals)),
|
|
8637
|
+
section$2("endpoints", "Agent endpoints and auth discovery", endpointChecks(discovery)),
|
|
8638
|
+
section$2("identity", "Identity evidence", identityChecks(report))
|
|
8631
8639
|
];
|
|
8632
8640
|
const checks = sections.flatMap((item) => item.checks);
|
|
8633
8641
|
const actionable = checks.filter((item) => [
|
|
@@ -8814,7 +8822,7 @@ function status(score, maxScore) {
|
|
|
8814
8822
|
if (ratio >= .5) return "warning";
|
|
8815
8823
|
return "fail";
|
|
8816
8824
|
}
|
|
8817
|
-
function section(id, title, checks) {
|
|
8825
|
+
function section$1(id, title, checks) {
|
|
8818
8826
|
return {
|
|
8819
8827
|
id,
|
|
8820
8828
|
title,
|
|
@@ -8872,7 +8880,7 @@ function aiReadiness(report) {
|
|
|
8872
8880
|
const blockedSnippets = restrictedSnippets.filter(({ control }) => control.status === "blocked");
|
|
8873
8881
|
const limitedSnippets = restrictedSnippets.filter(({ control }) => control.status === "limited");
|
|
8874
8882
|
const sections = [
|
|
8875
|
-
section("agent-access", "Agent access", [
|
|
8883
|
+
section$1("agent-access", "Agent access", [
|
|
8876
8884
|
check({
|
|
8877
8885
|
id: "robots-ai-bots",
|
|
8878
8886
|
section: "agent-access",
|
|
@@ -8915,7 +8923,7 @@ function aiReadiness(report) {
|
|
|
8915
8923
|
evidence: { resources: report.ai?.agentResources ?? [] }
|
|
8916
8924
|
})
|
|
8917
8925
|
]),
|
|
8918
|
-
section("machine-readable", "Machine-readable signals", [check({
|
|
8926
|
+
section$1("machine-readable", "Machine-readable signals", [check({
|
|
8919
8927
|
id: "llms-txt",
|
|
8920
8928
|
section: "machine-readable",
|
|
8921
8929
|
maxScore: 0,
|
|
@@ -8940,7 +8948,7 @@ function aiReadiness(report) {
|
|
|
8940
8948
|
action: invalidJsonLdPages.length ? "Fix invalid JSON-LD before relying on its structured-data claims." : "Use structured data only where it accurately represents the visible page and a supported use case.",
|
|
8941
8949
|
urls: invalidJsonLdPages.slice(0, 10).map((page) => page.finalUrl)
|
|
8942
8950
|
})]),
|
|
8943
|
-
section("content-clarity", "Content clarity", [
|
|
8951
|
+
section$1("content-clarity", "Content clarity", [
|
|
8944
8952
|
check({
|
|
8945
8953
|
id: "answerable-content",
|
|
8946
8954
|
section: "content-clarity",
|
|
@@ -8990,7 +8998,7 @@ function aiReadiness(report) {
|
|
|
8990
8998
|
action: "Set the html lang attribute so crawlers and assistive tech understand the page language."
|
|
8991
8999
|
})
|
|
8992
9000
|
]),
|
|
8993
|
-
section("entity-signals", "Entity signals", [check({
|
|
9001
|
+
section$1("entity-signals", "Entity signals", [check({
|
|
8994
9002
|
id: "entity-schema",
|
|
8995
9003
|
section: "entity-signals",
|
|
8996
9004
|
maxScore: 0,
|
|
@@ -9009,7 +9017,7 @@ function aiReadiness(report) {
|
|
|
9009
9017
|
plainEnglish: "Author and date signals help agents judge freshness and accountability.",
|
|
9010
9018
|
action: "Add visible authors, reviewed dates, and updated dates to content where trust and freshness matter."
|
|
9011
9019
|
})]),
|
|
9012
|
-
section("technical-ux", "Technical UX", [
|
|
9020
|
+
section$1("technical-ux", "Technical UX", [
|
|
9013
9021
|
check({
|
|
9014
9022
|
id: "snippet-controls",
|
|
9015
9023
|
section: "technical-ux",
|
|
@@ -9047,7 +9055,7 @@ function aiReadiness(report) {
|
|
|
9047
9055
|
action: "Fix 4xx/5xx pages, accidental noindex, and blocked internal URLs before expanding AI discovery work."
|
|
9048
9056
|
})
|
|
9049
9057
|
]),
|
|
9050
|
-
section("agent-content", "Agent content observations", [
|
|
9058
|
+
section$1("agent-content", "Agent content observations", [
|
|
9051
9059
|
check({
|
|
9052
9060
|
id: "markdown-alternates",
|
|
9053
9061
|
section: "agent-content",
|
|
@@ -9087,7 +9095,7 @@ function aiReadiness(report) {
|
|
|
9087
9095
|
evidence: agentDiscovery?.profileApplicability
|
|
9088
9096
|
})
|
|
9089
9097
|
]),
|
|
9090
|
-
section("crawl-completeness", "Crawl completeness", [check({
|
|
9098
|
+
section$1("crawl-completeness", "Crawl completeness", [check({
|
|
9091
9099
|
id: "crawl-depth",
|
|
9092
9100
|
section: "crawl-completeness",
|
|
9093
9101
|
maxScore: 8,
|
|
@@ -15603,19 +15611,19 @@ function renderCrawlHtml(report, fixes = topFixes(report)) {
|
|
|
15603
15611
|
const reviews = reviewObservations(report);
|
|
15604
15612
|
const issueRows = report.issueGroups.map((group) => {
|
|
15605
15613
|
const rule = explainRule(group.ruleId);
|
|
15606
|
-
return `<tr><td>${escapeHtml(group.severity)}</td><td>${escapeHtml(group.title)}</td><td>${group.count}</td><td>${escapeHtml(rule?.howToFix ?? "")}</td><td>${escapeHtml(group.sampleUrls[0] ?? "")}</td></tr>`;
|
|
15614
|
+
return `<tr><td>${escapeHtml$1(group.severity)}</td><td>${escapeHtml$1(group.title)}</td><td>${group.count}</td><td>${escapeHtml$1(rule?.howToFix ?? "")}</td><td>${escapeHtml$1(group.sampleUrls[0] ?? "")}</td></tr>`;
|
|
15607
15615
|
}).join("\n");
|
|
15608
|
-
const fixItems = fixes.slice(0, 10).map((fix) => `<li><strong>${escapeHtml(fix.title)}</strong><br>${escapeHtml(fix.howToFix)}<br><span>Verify: ${escapeHtml(fix.howToVerify)}</span><br><code>${escapeHtml(fix.verification.command)}</code></li>`).join("\n");
|
|
15609
|
-
const reviewItems = reviews.slice(0, 10).map((observation) => `<li><strong>${escapeHtml(observation.title)}</strong><br>${escapeHtml(observation.howToFix)}<br><span>Check before scheduling work. ${escapeHtml(observation.howToVerify)}</span></li>`).join("\n");
|
|
15610
|
-
const caveatItems = report.caveats.map((caveat) => `<li>${escapeHtml(caveat)}</li>`).join("\n");
|
|
15611
|
-
const warningItems = report.warnings.map((warning) => `<li>${escapeHtml(warning)}</li>`).join("\n");
|
|
15612
|
-
const accessItems = report.access.samples.map((sample) => `<li><strong>${escapeHtml(sample.url)}</strong><br>${escapeHtml(sample.evidence.guidance.summary)}<br>${escapeHtml(sample.evidence.guidance.recommendedAction)}<br><span>${escapeHtml(sample.evidence.guidance.securityNote)}</span></li>`).join("\n");
|
|
15616
|
+
const fixItems = fixes.slice(0, 10).map((fix) => `<li><strong>${escapeHtml$1(fix.title)}</strong><br>${escapeHtml$1(fix.howToFix)}<br><span>Verify: ${escapeHtml$1(fix.howToVerify)}</span><br><code>${escapeHtml$1(fix.verification.command)}</code></li>`).join("\n");
|
|
15617
|
+
const reviewItems = reviews.slice(0, 10).map((observation) => `<li><strong>${escapeHtml$1(observation.title)}</strong><br>${escapeHtml$1(observation.howToFix)}<br><span>Check before scheduling work. ${escapeHtml$1(observation.howToVerify)}</span></li>`).join("\n");
|
|
15618
|
+
const caveatItems = report.caveats.map((caveat) => `<li>${escapeHtml$1(caveat)}</li>`).join("\n");
|
|
15619
|
+
const warningItems = report.warnings.map((warning) => `<li>${escapeHtml$1(warning)}</li>`).join("\n");
|
|
15620
|
+
const accessItems = report.access.samples.map((sample) => `<li><strong>${escapeHtml$1(sample.url)}</strong><br>${escapeHtml$1(sample.evidence.guidance.summary)}<br>${escapeHtml$1(sample.evidence.guidance.recommendedAction)}<br><span>${escapeHtml$1(sample.evidence.guidance.securityNote)}</span></li>`).join("\n");
|
|
15613
15621
|
return `<!doctype html>
|
|
15614
15622
|
<html lang="en">
|
|
15615
15623
|
<head>
|
|
15616
15624
|
<meta charset="utf-8">
|
|
15617
15625
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
15618
|
-
<title>Crawl report for ${escapeHtml(report.config.url)}</title>
|
|
15626
|
+
<title>Crawl report for ${escapeHtml$1(report.config.url)}</title>
|
|
15619
15627
|
<style>
|
|
15620
15628
|
body { color: #17202a; font: 15px/1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; margin: 32px; max-width: 1120px; }
|
|
15621
15629
|
h1, h2 { line-height: 1.2; }
|
|
@@ -15631,10 +15639,10 @@ function renderCrawlHtml(report, fixes = topFixes(report)) {
|
|
|
15631
15639
|
</head>
|
|
15632
15640
|
<body>
|
|
15633
15641
|
<h1>Crawl report</h1>
|
|
15634
|
-
<p>${escapeHtml(report.config.url)}</p>
|
|
15635
|
-
<p>Strategy: ${escapeHtml(report.config.strategy === "health" ? "sitemap health pass" : "full audit")}<br>Crawler: <code>${escapeHtml(report.access.crawler.userAgent)}</code></p>
|
|
15642
|
+
<p>${escapeHtml$1(report.config.url)}</p>
|
|
15643
|
+
<p>Strategy: ${escapeHtml$1(report.config.strategy === "health" ? "sitemap health pass" : "full audit")}<br>Crawler: <code>${escapeHtml$1(report.access.crawler.userAgent)}</code></p>
|
|
15636
15644
|
<section class="summary">
|
|
15637
|
-
<div class="metric"><span>Status</span><strong>${escapeHtml(report.status)}</strong></div>
|
|
15645
|
+
<div class="metric"><span>Status</span><strong>${escapeHtml$1(report.status)}</strong></div>
|
|
15638
15646
|
<div class="metric"><span>Documents</span><strong>${report.summary.totalPages}</strong></div>
|
|
15639
15647
|
<div class="metric"><span>Requests</span><strong>${report.requestEvidenceStatus === "unavailable" ? "n/a" : report.requests.length}${report.requestEvidenceStatus === "partial" ? " (partial)" : ""}</strong></div>
|
|
15640
15648
|
<div class="metric"><span>Discovered</span><strong>${report.summary.discoveredUrls}</strong></div>
|
|
@@ -15688,11 +15696,11 @@ function renderCrawlMarkdownTickets(report, fixes = topFixes(report)) {
|
|
|
15688
15696
|
if (reviews.length) lines.push("## Review observations", "", "These are not implementation tickets. Check the evidence before deciding whether to work on them.", "", ...reviews.slice(0, 10).flatMap((observation) => [`- ${observation.title} (${observation.severity}, ${observation.count} URL${observation.count === 1 ? "" : "s"}): ${observation.howToFix}`, ` - First URL: ${observation.sampleUrls[0] ?? "No sample URL recorded"}`]), "");
|
|
15689
15697
|
return `${lines.join("\n")}\n`;
|
|
15690
15698
|
}
|
|
15691
|
-
function escapeHtml(value) {
|
|
15699
|
+
function escapeHtml$1(value) {
|
|
15692
15700
|
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll("\"", """).replaceAll("'", "'");
|
|
15693
15701
|
}
|
|
15694
15702
|
function escapeXml(value) {
|
|
15695
|
-
return escapeHtml(value);
|
|
15703
|
+
return escapeHtml$1(value);
|
|
15696
15704
|
}
|
|
15697
15705
|
//#endregion
|
|
15698
15706
|
//#region packages/core/src/analyze/crawler/history-provenance.ts
|
|
@@ -17549,7 +17557,7 @@ function compareText$10(left, right) {
|
|
|
17549
17557
|
}
|
|
17550
17558
|
return leftPoints.length - rightPoints.length;
|
|
17551
17559
|
}
|
|
17552
|
-
function boundedInteger$
|
|
17560
|
+
function boundedInteger$6(value, fallback, minimum, maximum) {
|
|
17553
17561
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
17554
17562
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
17555
17563
|
}
|
|
@@ -17626,8 +17634,8 @@ function initialSelection(sourceRows) {
|
|
|
17626
17634
|
};
|
|
17627
17635
|
}
|
|
17628
17636
|
function analyzeCtrUnderperformersFromRows(input) {
|
|
17629
|
-
const minImpressions = boundedInteger$
|
|
17630
|
-
const limit = boundedInteger$
|
|
17637
|
+
const minImpressions = boundedInteger$6(input.minImpressions, 200, 1, CTR_MAX_MIN_IMPRESSIONS);
|
|
17638
|
+
const limit = boundedInteger$6(input.limit, 25, 1, 100);
|
|
17631
17639
|
const selection = initialSelection(input.rows.length);
|
|
17632
17640
|
const benchmarkRows = aggregateCtrRows(input.rows, selection).filter((row) => {
|
|
17633
17641
|
const query = row.keys[0] ?? "";
|
|
@@ -18126,7 +18134,7 @@ function analyzeSegmentRows(input) {
|
|
|
18126
18134
|
}
|
|
18127
18135
|
//#endregion
|
|
18128
18136
|
//#region packages/core/src/analyze/segment-impact-input.ts
|
|
18129
|
-
const DAY_MS$
|
|
18137
|
+
const DAY_MS$2 = 864e5;
|
|
18130
18138
|
const MAX_WINDOW_DAYS = 240;
|
|
18131
18139
|
function subtractDays$1(value, days) {
|
|
18132
18140
|
const date = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -18134,7 +18142,7 @@ function subtractDays$1(value, days) {
|
|
|
18134
18142
|
return date.toISOString().slice(0, 10);
|
|
18135
18143
|
}
|
|
18136
18144
|
function segmentRangeDays(range) {
|
|
18137
|
-
return Math.floor((Date.parse(`${range.endDate}T00:00:00.000Z`) - Date.parse(`${range.startDate}T00:00:00.000Z`)) / DAY_MS$
|
|
18145
|
+
return Math.floor((Date.parse(`${range.endDate}T00:00:00.000Z`) - Date.parse(`${range.startDate}T00:00:00.000Z`)) / DAY_MS$2) + 1;
|
|
18138
18146
|
}
|
|
18139
18147
|
function segmentComparisonRange(after, compareDays) {
|
|
18140
18148
|
const endDate = subtractDays$1(after.startDate, 1);
|
|
@@ -18761,13 +18769,13 @@ const MINIMUM_PAGE_SHARE = .1;
|
|
|
18761
18769
|
const MINIMUM_PAGE_IMPRESSIONS = 10;
|
|
18762
18770
|
const MAXIMUM_DOMINANT_SHARE = .8;
|
|
18763
18771
|
const MAX_SUPPRESSIONS = 100;
|
|
18764
|
-
function boundedInteger$
|
|
18772
|
+
function boundedInteger$5(value, fallback, minimum, maximum) {
|
|
18765
18773
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
18766
18774
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
18767
18775
|
}
|
|
18768
18776
|
function analyzeCannibalRows(input) {
|
|
18769
|
-
const minImpressions = boundedInteger$
|
|
18770
|
-
const limit = boundedInteger$
|
|
18777
|
+
const minImpressions = boundedInteger$5(input.minImpressions, 50, 0, 1e9);
|
|
18778
|
+
const limit = boundedInteger$5(input.limit, 25, 1, 100);
|
|
18771
18779
|
const selection = {
|
|
18772
18780
|
sourceRows: input.rows.length,
|
|
18773
18781
|
invalidRows: 0,
|
|
@@ -19549,7 +19557,7 @@ function groupQuickWins(items) {
|
|
|
19549
19557
|
}
|
|
19550
19558
|
//#endregion
|
|
19551
19559
|
//#region packages/core/src/analyze/site-diagnostics/quick-wins-analysis-primitives.ts
|
|
19552
|
-
function boundedInteger$
|
|
19560
|
+
function boundedInteger$4(value, fallback, minimum, maximum) {
|
|
19553
19561
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
19554
19562
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
19555
19563
|
}
|
|
@@ -19670,12 +19678,12 @@ function roundedSum(values) {
|
|
|
19670
19678
|
//#region packages/core/src/analyze/site-diagnostics/quick-wins-analysis.ts
|
|
19671
19679
|
const DEFAULT_MIN_IMPRESSIONS$2 = 200;
|
|
19672
19680
|
const MAX_MIN_IMPRESSIONS$2 = 1e9;
|
|
19673
|
-
const DEFAULT_LIMIT$
|
|
19674
|
-
const MAX_LIMIT$
|
|
19681
|
+
const DEFAULT_LIMIT$3 = 25;
|
|
19682
|
+
const MAX_LIMIT$3 = 100;
|
|
19675
19683
|
const FALLBACK_SOURCE = "builtin_position_ctr_curve_v1";
|
|
19676
19684
|
function analyzeQuickWinsFromRows(input) {
|
|
19677
|
-
const minImpressions = boundedInteger$
|
|
19678
|
-
const limit = boundedInteger$
|
|
19685
|
+
const minImpressions = boundedInteger$4(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$2, 0, MAX_MIN_IMPRESSIONS$2);
|
|
19686
|
+
const limit = boundedInteger$4(input.limit, DEFAULT_LIMIT$3, 1, MAX_LIMIT$3);
|
|
19679
19687
|
const selected = selectBenchmarkRows(input);
|
|
19680
19688
|
const benchmarkContext = createCtrBenchmarkContext(selected.rows, {
|
|
19681
19689
|
samplePopulation: "all_qualified_url_samples",
|
|
@@ -20016,7 +20024,7 @@ function compareText$5(left, right) {
|
|
|
20016
20024
|
}
|
|
20017
20025
|
return leftPoints.length - rightPoints.length;
|
|
20018
20026
|
}
|
|
20019
|
-
function isHttpUrl(value) {
|
|
20027
|
+
function isHttpUrl$1(value) {
|
|
20020
20028
|
try {
|
|
20021
20029
|
const parsed = new URL(value);
|
|
20022
20030
|
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
@@ -20025,7 +20033,7 @@ function isHttpUrl(value) {
|
|
|
20025
20033
|
}
|
|
20026
20034
|
}
|
|
20027
20035
|
function isValidStrikingDistanceRow(row, query, url) {
|
|
20028
|
-
return Boolean(query) && isHttpUrl(url) && Number.isFinite(row.clicks) && Number.isFinite(row.impressions) && Number.isFinite(row.ctr) && Number.isFinite(row.position) && row.clicks >= 0 && row.impressions > 0 && row.clicks <= row.impressions && row.ctr >= 0 && row.ctr <= 1 && row.position > 0;
|
|
20036
|
+
return Boolean(query) && isHttpUrl$1(url) && Number.isFinite(row.clicks) && Number.isFinite(row.impressions) && Number.isFinite(row.ctr) && Number.isFinite(row.position) && row.clicks >= 0 && row.impressions > 0 && row.clicks <= row.impressions && row.ctr >= 0 && row.ctr <= 1 && row.position > 0;
|
|
20029
20037
|
}
|
|
20030
20038
|
function isStrikingDistanceBrandRow(input) {
|
|
20031
20039
|
return shouldExcludeBrandQuery({
|
|
@@ -20115,8 +20123,8 @@ function groupStrikingDistanceItems(items) {
|
|
|
20115
20123
|
//#region packages/core/src/analyze/striking-distance-analysis.ts
|
|
20116
20124
|
const DEFAULT_MIN_IMPRESSIONS$1 = 100;
|
|
20117
20125
|
const MAX_MIN_IMPRESSIONS$1 = 1e9;
|
|
20118
|
-
const DEFAULT_LIMIT$
|
|
20119
|
-
const MAX_LIMIT$
|
|
20126
|
+
const DEFAULT_LIMIT$2 = 25;
|
|
20127
|
+
const MAX_LIMIT$2 = 100;
|
|
20120
20128
|
function integerInRange$1(value, fallback, minimum, maximum) {
|
|
20121
20129
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
20122
20130
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
@@ -20158,7 +20166,7 @@ function selectRows(input) {
|
|
|
20158
20166
|
}
|
|
20159
20167
|
function analyzeStrikingDistanceRows(input) {
|
|
20160
20168
|
const minImpressions = integerInRange$1(input.minImpressions, DEFAULT_MIN_IMPRESSIONS$1, 0, MAX_MIN_IMPRESSIONS$1);
|
|
20161
|
-
const limit = integerInRange$1(input.limit, DEFAULT_LIMIT$
|
|
20169
|
+
const limit = integerInRange$1(input.limit, DEFAULT_LIMIT$2, 1, MAX_LIMIT$2);
|
|
20162
20170
|
const selected = selectRows({
|
|
20163
20171
|
...input,
|
|
20164
20172
|
minImpressions
|
|
@@ -20232,7 +20240,7 @@ const defaultDependencies$7 = {
|
|
|
20232
20240
|
verifyContent: verifyQueryContent,
|
|
20233
20241
|
now: () => /* @__PURE__ */ new Date()
|
|
20234
20242
|
};
|
|
20235
|
-
function boundedInteger$
|
|
20243
|
+
function boundedInteger$3(value, fallback, minimum, maximum) {
|
|
20236
20244
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
20237
20245
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
20238
20246
|
}
|
|
@@ -20296,7 +20304,7 @@ async function strikingDistance(input, dependencies = defaultDependencies$7) {
|
|
|
20296
20304
|
brandTerms: input.brandTerms,
|
|
20297
20305
|
includeBrand: input.includeBrand
|
|
20298
20306
|
});
|
|
20299
|
-
const verifyLimit = boundedInteger$
|
|
20307
|
+
const verifyLimit = boundedInteger$3(input.verifyLimit, DEFAULT_VERIFY_LIMIT, 0, analysis.limit);
|
|
20300
20308
|
const verificationRequested = input.verifyContent === true || input.verifyContent !== false && input.verifyLimit !== void 0;
|
|
20301
20309
|
const attempted = verificationRequested ? Math.min(verifyLimit, analysis.items.length) : 0;
|
|
20302
20310
|
const items = await Promise.all(analysis.items.map(async (item, index) => {
|
|
@@ -20485,7 +20493,7 @@ function standardDeviation(values) {
|
|
|
20485
20493
|
function toFixedNumber(value, digits = 3) {
|
|
20486
20494
|
return Number(value.toFixed(digits));
|
|
20487
20495
|
}
|
|
20488
|
-
function percentChange(baseline, comparison) {
|
|
20496
|
+
function percentChange$1(baseline, comparison) {
|
|
20489
20497
|
if (!baseline) return comparison ? null : 0;
|
|
20490
20498
|
return (comparison - baseline) / baseline * 100;
|
|
20491
20499
|
}
|
|
@@ -20671,7 +20679,7 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
20671
20679
|
const outsideFlatBaseline = zScore === null && comparisonMean !== baselineMean;
|
|
20672
20680
|
const significant = outsideFlatBaseline || zScore !== null && Math.abs(zScore) >= 2;
|
|
20673
20681
|
const direction = significant ? comparisonMean < baselineMean ? "drop" : "spike" : "normal";
|
|
20674
|
-
const relativeChange = percentChange(baselineMean, comparisonMean);
|
|
20682
|
+
const relativeChange = percentChange$1(baselineMean, comparisonMean);
|
|
20675
20683
|
return {
|
|
20676
20684
|
site: input.site,
|
|
20677
20685
|
metric,
|
|
@@ -21730,7 +21738,7 @@ function dateShift(date, days) {
|
|
|
21730
21738
|
}
|
|
21731
21739
|
//#endregion
|
|
21732
21740
|
//#region packages/core/src/analyze/experiments/measurement/window.ts
|
|
21733
|
-
const DAY_MS = 864e5;
|
|
21741
|
+
const DAY_MS$1 = 864e5;
|
|
21734
21742
|
function exactDate(value, label) {
|
|
21735
21743
|
const pattern = /^\d{4}-\d{2}-\d{2}$/;
|
|
21736
21744
|
const parsed = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -21745,7 +21753,7 @@ function boundedDays(value, fallback, label) {
|
|
|
21745
21753
|
function calendarDays(startDate, endDate) {
|
|
21746
21754
|
const start = (/* @__PURE__ */ new Date(`${startDate}T00:00:00.000Z`)).getTime();
|
|
21747
21755
|
const end = (/* @__PURE__ */ new Date(`${endDate}T00:00:00.000Z`)).getTime();
|
|
21748
|
-
return Math.floor((end - start) / DAY_MS) + 1;
|
|
21756
|
+
return Math.floor((end - start) / DAY_MS$1) + 1;
|
|
21749
21757
|
}
|
|
21750
21758
|
function subtractCalendarMonths(value, months) {
|
|
21751
21759
|
const source = /* @__PURE__ */ new Date(`${value}T00:00:00.000Z`);
|
|
@@ -22293,7 +22301,7 @@ function internalLinksVerdict(input) {
|
|
|
22293
22301
|
function uniqueInternalLinksWarnings(warnings) {
|
|
22294
22302
|
return [...new Map(warnings.map((warning) => [`${warning.stage}\u0000${warning.url}\u0000${warning.code}\u0000${warning.message}`, warning])).values()];
|
|
22295
22303
|
}
|
|
22296
|
-
function errorMessage(error) {
|
|
22304
|
+
function errorMessage$1(error) {
|
|
22297
22305
|
return error instanceof Error ? error.message : String(error);
|
|
22298
22306
|
}
|
|
22299
22307
|
function acceptedAlias(site, value) {
|
|
@@ -22343,7 +22351,7 @@ async function verifyInternalLinkTarget(input) {
|
|
|
22343
22351
|
stage: "target-fetch",
|
|
22344
22352
|
url: input.targetUrl,
|
|
22345
22353
|
code: "fetch-failed",
|
|
22346
|
-
message: errorMessage(error)
|
|
22354
|
+
message: errorMessage$1(error)
|
|
22347
22355
|
}]
|
|
22348
22356
|
};
|
|
22349
22357
|
}
|
|
@@ -22364,7 +22372,7 @@ async function verifyInternalLinkTarget(input) {
|
|
|
22364
22372
|
stage: "target-extract",
|
|
22365
22373
|
url: input.targetUrl,
|
|
22366
22374
|
code: "extract-failed",
|
|
22367
|
-
message: errorMessage(error)
|
|
22375
|
+
message: errorMessage$1(error)
|
|
22368
22376
|
}]
|
|
22369
22377
|
};
|
|
22370
22378
|
}
|
|
@@ -22427,7 +22435,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
22427
22435
|
stage: "source-fetch",
|
|
22428
22436
|
url: input.candidate.sourceUrl,
|
|
22429
22437
|
code: "fetch-failed",
|
|
22430
|
-
message: errorMessage(error)
|
|
22438
|
+
message: errorMessage$1(error)
|
|
22431
22439
|
}]
|
|
22432
22440
|
};
|
|
22433
22441
|
}
|
|
@@ -22445,7 +22453,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
22445
22453
|
stage: "source-fetch",
|
|
22446
22454
|
url: input.candidate.sourceUrl,
|
|
22447
22455
|
code: "fetch-failed",
|
|
22448
|
-
message: errorMessage(error)
|
|
22456
|
+
message: errorMessage$1(error)
|
|
22449
22457
|
}]
|
|
22450
22458
|
};
|
|
22451
22459
|
}
|
|
@@ -22459,7 +22467,7 @@ async function verifyInternalLinkCandidate(input) {
|
|
|
22459
22467
|
stage: "source-extract",
|
|
22460
22468
|
url: input.candidate.sourceUrl,
|
|
22461
22469
|
code: "extract-failed",
|
|
22462
|
-
message: errorMessage(error)
|
|
22470
|
+
message: errorMessage$1(error)
|
|
22463
22471
|
}]
|
|
22464
22472
|
};
|
|
22465
22473
|
}
|
|
@@ -23270,7 +23278,7 @@ function compareText$3(left, right) {
|
|
|
23270
23278
|
}
|
|
23271
23279
|
return leftPoints.length - rightPoints.length;
|
|
23272
23280
|
}
|
|
23273
|
-
function boundedInteger$
|
|
23281
|
+
function boundedInteger$2(value, fallback, maximum) {
|
|
23274
23282
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
23275
23283
|
return Math.min(maximum, Math.max(1, Math.trunc(value)));
|
|
23276
23284
|
}
|
|
@@ -23463,10 +23471,10 @@ function caveats(input) {
|
|
|
23463
23471
|
return items;
|
|
23464
23472
|
}
|
|
23465
23473
|
function analyzeIndexCoverageSignals(input) {
|
|
23466
|
-
const itemLimit = boundedInteger$
|
|
23467
|
-
const clusterLimit = boundedInteger$
|
|
23468
|
-
const clusterSamples = boundedInteger$
|
|
23469
|
-
const minUrls = boundedInteger$
|
|
23474
|
+
const itemLimit = boundedInteger$2(input.limits?.itemsPerSection, DEFAULT_ITEMS_PER_SECTION, MAX_ITEMS_PER_SECTION);
|
|
23475
|
+
const clusterLimit = boundedInteger$2(input.limits?.templateClusters, DEFAULT_TEMPLATE_LIMIT, MAX_TEMPLATE_LIMIT);
|
|
23476
|
+
const clusterSamples = boundedInteger$2(input.limits?.templateSamples, DEFAULT_TEMPLATE_SAMPLES, MAX_TEMPLATE_SAMPLES);
|
|
23477
|
+
const minUrls = boundedInteger$2(input.templateThresholds?.minUrls, DEFAULT_TEMPLATE_MIN_URLS, Number.MAX_SAFE_INTEGER);
|
|
23470
23478
|
const requestedMinShare = input.templateThresholds?.minShare;
|
|
23471
23479
|
const minShare = Number.isFinite(requestedMinShare) ? Math.min(1, Math.max(0, requestedMinShare ?? DEFAULT_TEMPLATE_MIN_SHARE)) : DEFAULT_TEMPLATE_MIN_SHARE;
|
|
23472
23480
|
const crawl = crawlInput(input.crawl);
|
|
@@ -27241,16 +27249,16 @@ function benchmarkDetails(benchmark) {
|
|
|
27241
27249
|
positiveUrlSamples: benchmark.positiveUrlSamples
|
|
27242
27250
|
};
|
|
27243
27251
|
}
|
|
27244
|
-
function boundedInteger(input) {
|
|
27252
|
+
function boundedInteger$1(input) {
|
|
27245
27253
|
const value = Number.isFinite(input.value) ? input.value ?? 0 : input.fallback;
|
|
27246
27254
|
return Math.min(input.maximum ?? Number.POSITIVE_INFINITY, Math.max(1, Math.floor(value)));
|
|
27247
27255
|
}
|
|
27248
27256
|
function analyzeQueryClustersFromRows(input) {
|
|
27249
|
-
const minImpressions = boundedInteger({
|
|
27257
|
+
const minImpressions = boundedInteger$1({
|
|
27250
27258
|
value: input.minImpressions,
|
|
27251
27259
|
fallback: 25
|
|
27252
27260
|
});
|
|
27253
|
-
const limit = boundedInteger({
|
|
27261
|
+
const limit = boundedInteger$1({
|
|
27254
27262
|
value: input.limit,
|
|
27255
27263
|
fallback: 25,
|
|
27256
27264
|
maximum: 100
|
|
@@ -27892,8 +27900,8 @@ function reportPresentation(report) {
|
|
|
27892
27900
|
//#region packages/core/src/analyze/second-page-analysis.ts
|
|
27893
27901
|
const DEFAULT_MIN_IMPRESSIONS = 50;
|
|
27894
27902
|
const MAX_MIN_IMPRESSIONS = 1e9;
|
|
27895
|
-
const DEFAULT_LIMIT = 10;
|
|
27896
|
-
const MAX_LIMIT = 100;
|
|
27903
|
+
const DEFAULT_LIMIT$1 = 10;
|
|
27904
|
+
const MAX_LIMIT$1 = 100;
|
|
27897
27905
|
function integerInRange(value, fallback, minimum, maximum) {
|
|
27898
27906
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
27899
27907
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
@@ -27980,7 +27988,7 @@ function compareItems(left, right) {
|
|
|
27980
27988
|
}
|
|
27981
27989
|
function analyzeSecondPageRows(input) {
|
|
27982
27990
|
const minImpressions = integerInRange(input.minImpressions, DEFAULT_MIN_IMPRESSIONS, 0, MAX_MIN_IMPRESSIONS);
|
|
27983
|
-
const limit = integerInRange(input.limit, DEFAULT_LIMIT, 1, MAX_LIMIT);
|
|
27991
|
+
const limit = integerInRange(input.limit, DEFAULT_LIMIT$1, 1, MAX_LIMIT$1);
|
|
27984
27992
|
const selection = {
|
|
27985
27993
|
sourceRows: input.rows.length,
|
|
27986
27994
|
invalidRows: 0,
|
|
@@ -29515,188 +29523,1243 @@ async function updatePostmortemWorkflow(input, dependencies = {}) {
|
|
|
29515
29523
|
});
|
|
29516
29524
|
}
|
|
29517
29525
|
//#endregion
|
|
29518
|
-
//#region packages/core/src/
|
|
29519
|
-
|
|
29520
|
-
|
|
29521
|
-
|
|
29522
|
-
|
|
29523
|
-
|
|
29524
|
-
|
|
29525
|
-
|
|
29526
|
-
|
|
29527
|
-
}
|
|
29528
|
-
|
|
29529
|
-
|
|
29530
|
-
if (
|
|
29531
|
-
return
|
|
29532
|
-
}
|
|
29533
|
-
function
|
|
29534
|
-
|
|
29535
|
-
|
|
29536
|
-
|
|
29537
|
-
|
|
29538
|
-
|
|
29539
|
-
|
|
29540
|
-
|
|
29541
|
-
|
|
29542
|
-
|
|
29543
|
-
|
|
29544
|
-
|
|
29545
|
-
|
|
29546
|
-
const now = Date.now();
|
|
29547
|
-
const client = {
|
|
29548
|
-
id,
|
|
29549
|
-
name: input.name ?? existing?.name ?? id,
|
|
29550
|
-
siteUrl: input.siteUrl,
|
|
29551
|
-
startUrl: input.startUrl ?? existing?.startUrl,
|
|
29552
|
-
watchUrls: uniqueStrings(input.watchUrls ?? existing?.watchUrls),
|
|
29553
|
-
brandTerms: uniqueStrings(input.brandTerms ?? existing?.brandTerms ?? deriveBrandTerms({
|
|
29554
|
-
id,
|
|
29555
|
-
name: input.name ?? existing?.name,
|
|
29556
|
-
siteUrl: input.siteUrl
|
|
29557
|
-
})),
|
|
29558
|
-
analytics: input.analytics ?? existing?.analytics ?? {},
|
|
29559
|
-
reportDay: input.reportDay ?? existing?.reportDay,
|
|
29560
|
-
technicalWeekday: input.technicalWeekday ?? existing?.technicalWeekday,
|
|
29561
|
-
isDefault: input.isDefault ?? existing?.isDefault,
|
|
29562
|
-
createdAt: existing?.createdAt ?? now,
|
|
29563
|
-
updatedAt: now
|
|
29564
|
-
};
|
|
29565
|
-
const clients = config.clients.filter((item) => item.id !== id);
|
|
29566
|
-
if (client.isDefault) {
|
|
29567
|
-
for (const item of clients) item.isDefault = false;
|
|
29568
|
-
config.defaultSite = client.siteUrl;
|
|
29526
|
+
//#region packages/core/src/bing/analysis.ts
|
|
29527
|
+
const DAY_MS = 864e5;
|
|
29528
|
+
const DEFAULT_COMPARISON_DAYS = 28;
|
|
29529
|
+
const WEEKLY_PERIODS = 4;
|
|
29530
|
+
const OPPORTUNITY_MIN_IMPRESSIONS = 20;
|
|
29531
|
+
const OPPORTUNITY_MIN_POSITION = 4;
|
|
29532
|
+
const OPPORTUNITY_MAX_POSITION = 20;
|
|
29533
|
+
const RESULT_LIMIT = 10;
|
|
29534
|
+
function dateOffset(date, days) {
|
|
29535
|
+
return new Date(Date.parse(`${date}T00:00:00Z`) + days * DAY_MS).toISOString().slice(0, 10);
|
|
29536
|
+
}
|
|
29537
|
+
function percentChange(previous, current) {
|
|
29538
|
+
if (previous === 0) return null;
|
|
29539
|
+
return round((current - previous) / previous * 100);
|
|
29540
|
+
}
|
|
29541
|
+
function round(value, digits = 3) {
|
|
29542
|
+
const factor = 10 ** digits;
|
|
29543
|
+
return Math.round(value * factor) / factor;
|
|
29544
|
+
}
|
|
29545
|
+
function trafficPeriod(byDate, startDate, endDate, expectedDays) {
|
|
29546
|
+
let clicks = 0;
|
|
29547
|
+
let impressions = 0;
|
|
29548
|
+
let observedDays = 0;
|
|
29549
|
+
for (const [date, row] of byDate) {
|
|
29550
|
+
if (date < startDate || date > endDate) continue;
|
|
29551
|
+
observedDays += 1;
|
|
29552
|
+
clicks += row.clicks;
|
|
29553
|
+
impressions += row.impressions;
|
|
29569
29554
|
}
|
|
29570
|
-
|
|
29571
|
-
|
|
29572
|
-
|
|
29555
|
+
return {
|
|
29556
|
+
startDate,
|
|
29557
|
+
endDate,
|
|
29558
|
+
expectedDays,
|
|
29559
|
+
observedDays,
|
|
29560
|
+
missingDays: Math.max(0, expectedDays - observedDays),
|
|
29561
|
+
clicks,
|
|
29562
|
+
impressions,
|
|
29563
|
+
ctr: impressions ? round(clicks / impressions * 100) : null
|
|
29564
|
+
};
|
|
29573
29565
|
}
|
|
29574
|
-
function
|
|
29575
|
-
|
|
29576
|
-
const
|
|
29577
|
-
|
|
29578
|
-
|
|
29579
|
-
|
|
29580
|
-
|
|
29581
|
-
|
|
29566
|
+
function analyzeBingTraffic(rows, comparisonDays = DEFAULT_COMPARISON_DAYS) {
|
|
29567
|
+
if (!rows.length) return void 0;
|
|
29568
|
+
const byDate = /* @__PURE__ */ new Map();
|
|
29569
|
+
for (const row of rows) {
|
|
29570
|
+
const existing = byDate.get(row.date) ?? {
|
|
29571
|
+
clicks: 0,
|
|
29572
|
+
impressions: 0
|
|
29573
|
+
};
|
|
29574
|
+
existing.clicks += row.clicks;
|
|
29575
|
+
existing.impressions += row.impressions;
|
|
29576
|
+
byDate.set(row.date, existing);
|
|
29577
|
+
}
|
|
29578
|
+
const latestDate = [...byDate.keys()].sort().at(-1);
|
|
29579
|
+
if (!latestDate) return void 0;
|
|
29580
|
+
const currentStart = dateOffset(latestDate, -(comparisonDays - 1));
|
|
29581
|
+
const previousEnd = dateOffset(currentStart, -1);
|
|
29582
|
+
const previousStart = dateOffset(previousEnd, -(comparisonDays - 1));
|
|
29583
|
+
const current = trafficPeriod(byDate, currentStart, latestDate, comparisonDays);
|
|
29584
|
+
const previous = trafficPeriod(byDate, previousStart, previousEnd, comparisonDays);
|
|
29585
|
+
return {
|
|
29586
|
+
comparisonDays,
|
|
29587
|
+
current,
|
|
29588
|
+
previous,
|
|
29589
|
+
changes: {
|
|
29590
|
+
clicks: current.clicks - previous.clicks,
|
|
29591
|
+
clicksPercent: percentChange(previous.clicks, current.clicks),
|
|
29592
|
+
impressions: current.impressions - previous.impressions,
|
|
29593
|
+
impressionsPercent: percentChange(previous.impressions, current.impressions),
|
|
29594
|
+
ctrPercentagePoints: current.ctr === null || previous.ctr === null ? null : round(current.ctr - previous.ctr)
|
|
29595
|
+
}
|
|
29596
|
+
};
|
|
29597
|
+
}
|
|
29598
|
+
const CRAWL_FIELDS = [
|
|
29599
|
+
"crawledPages",
|
|
29600
|
+
"inIndex",
|
|
29601
|
+
"inLinks",
|
|
29602
|
+
"crawlErrors",
|
|
29603
|
+
"code4xx",
|
|
29604
|
+
"code5xx",
|
|
29605
|
+
"blockedByRobotsTxt",
|
|
29606
|
+
"connectionTimeout",
|
|
29607
|
+
"dnsFailures",
|
|
29608
|
+
"containsMalware"
|
|
29609
|
+
];
|
|
29610
|
+
function analyzeBingCrawl(rows, comparisonDays = DEFAULT_COMPARISON_DAYS) {
|
|
29611
|
+
const ordered = [...rows].sort((a, b) => a.date.localeCompare(b.date, "en"));
|
|
29612
|
+
const current = ordered.at(-1);
|
|
29613
|
+
if (!current) return void 0;
|
|
29614
|
+
const targetDate = dateOffset(current.date, -comparisonDays);
|
|
29615
|
+
const previous = ordered.filter((row) => row.date <= targetDate).at(-1);
|
|
29616
|
+
const changes = {};
|
|
29617
|
+
if (!previous) return {
|
|
29618
|
+
comparisonDays,
|
|
29619
|
+
targetDate,
|
|
29620
|
+
current,
|
|
29621
|
+
previous: void 0,
|
|
29622
|
+
changes
|
|
29623
|
+
};
|
|
29624
|
+
for (const field of CRAWL_FIELDS) {
|
|
29625
|
+
const currentValue = current[field];
|
|
29626
|
+
const previousValue = previous[field];
|
|
29627
|
+
if (typeof currentValue !== "number" || typeof previousValue !== "number") continue;
|
|
29628
|
+
changes[field] = {
|
|
29629
|
+
previous: previousValue,
|
|
29630
|
+
current: currentValue,
|
|
29631
|
+
absolute: currentValue - previousValue,
|
|
29632
|
+
percent: percentChange(previousValue, currentValue)
|
|
29633
|
+
};
|
|
29634
|
+
}
|
|
29635
|
+
return {
|
|
29636
|
+
comparisonDays,
|
|
29637
|
+
targetDate,
|
|
29638
|
+
current,
|
|
29639
|
+
previous,
|
|
29640
|
+
changes
|
|
29641
|
+
};
|
|
29582
29642
|
}
|
|
29583
|
-
function
|
|
29584
|
-
const
|
|
29585
|
-
const
|
|
29586
|
-
|
|
29587
|
-
|
|
29588
|
-
|
|
29589
|
-
|
|
29590
|
-
|
|
29591
|
-
|
|
29592
|
-
|
|
29643
|
+
function aggregateDimensions(rows, dates) {
|
|
29644
|
+
const values = /* @__PURE__ */ new Map();
|
|
29645
|
+
for (const row of rows) {
|
|
29646
|
+
if (!dates.has(row.date)) continue;
|
|
29647
|
+
const item = values.get(row.value) ?? {
|
|
29648
|
+
clicks: 0,
|
|
29649
|
+
impressions: 0,
|
|
29650
|
+
clickPositionTotal: 0,
|
|
29651
|
+
clickPositionWeight: 0,
|
|
29652
|
+
impressionPositionTotal: 0,
|
|
29653
|
+
impressionPositionWeight: 0,
|
|
29654
|
+
observedDates: /* @__PURE__ */ new Set()
|
|
29655
|
+
};
|
|
29656
|
+
item.clicks += row.clicks;
|
|
29657
|
+
item.impressions += row.impressions;
|
|
29658
|
+
item.observedDates.add(row.date);
|
|
29659
|
+
if (row.avgClickPosition !== void 0 && row.clicks > 0) {
|
|
29660
|
+
const weight = row.clicks;
|
|
29661
|
+
item.clickPositionTotal += row.avgClickPosition * weight;
|
|
29662
|
+
item.clickPositionWeight += weight;
|
|
29663
|
+
}
|
|
29664
|
+
if (row.avgImpressionPosition !== void 0 && row.impressions > 0) {
|
|
29665
|
+
const weight = row.impressions;
|
|
29666
|
+
item.impressionPositionTotal += row.avgImpressionPosition * weight;
|
|
29667
|
+
item.impressionPositionWeight += weight;
|
|
29668
|
+
}
|
|
29669
|
+
values.set(row.value, item);
|
|
29670
|
+
}
|
|
29671
|
+
return new Map([...values].map(([value, item]) => [value, {
|
|
29672
|
+
value,
|
|
29673
|
+
observedWeeks: item.observedDates.size,
|
|
29674
|
+
clicks: item.clicks,
|
|
29675
|
+
impressions: item.impressions,
|
|
29676
|
+
avgClickPosition: item.clickPositionWeight ? round(item.clickPositionTotal / item.clickPositionWeight) : null,
|
|
29677
|
+
avgImpressionPosition: item.impressionPositionWeight ? round(item.impressionPositionTotal / item.impressionPositionWeight) : null
|
|
29678
|
+
}]));
|
|
29679
|
+
}
|
|
29680
|
+
function compareValue(a, b) {
|
|
29681
|
+
return a.value.localeCompare(b.value, "en");
|
|
29682
|
+
}
|
|
29683
|
+
function analyzeBingDimensions(rows, kind) {
|
|
29684
|
+
const dates = [...new Set(rows.map((row) => row.date))].sort().reverse();
|
|
29685
|
+
const currentDates = dates.slice(0, WEEKLY_PERIODS);
|
|
29686
|
+
const previousDates = dates.slice(WEEKLY_PERIODS, WEEKLY_PERIODS * 2);
|
|
29687
|
+
const current = aggregateDimensions(rows, new Set(currentDates));
|
|
29688
|
+
const previous = aggregateDimensions(rows, new Set(previousDates));
|
|
29689
|
+
const matchedValues = [...current.keys()].filter((value) => previous.has(value));
|
|
29690
|
+
const comparableValues = currentDates.length === WEEKLY_PERIODS && previousDates.length === WEEKLY_PERIODS ? matchedValues.filter((value) => {
|
|
29691
|
+
const currentValue = current.get(value);
|
|
29692
|
+
const previousValue = previous.get(value);
|
|
29693
|
+
return currentValue?.observedWeeks === WEEKLY_PERIODS && previousValue?.observedWeeks === WEEKLY_PERIODS;
|
|
29694
|
+
}) : [];
|
|
29695
|
+
const movements = comparableValues.map((value) => {
|
|
29696
|
+
const currentValue = current.get(value);
|
|
29697
|
+
const previousValue = previous.get(value);
|
|
29698
|
+
return {
|
|
29699
|
+
value,
|
|
29700
|
+
current: currentValue,
|
|
29701
|
+
previous: previousValue,
|
|
29702
|
+
changes: {
|
|
29703
|
+
clicks: currentValue.clicks - previousValue.clicks,
|
|
29704
|
+
clicksPercent: percentChange(previousValue.clicks, currentValue.clicks),
|
|
29705
|
+
impressions: currentValue.impressions - previousValue.impressions,
|
|
29706
|
+
impressionsPercent: percentChange(previousValue.impressions, currentValue.impressions)
|
|
29707
|
+
}
|
|
29708
|
+
};
|
|
29709
|
+
}).sort((a, b) => Math.abs(b.changes.clicks) - Math.abs(a.changes.clicks) || Math.abs(b.changes.impressions) - Math.abs(a.changes.impressions) || compareValue(a, b)).slice(0, RESULT_LIMIT);
|
|
29710
|
+
const opportunities = [...current.values()].filter((row) => row.impressions >= OPPORTUNITY_MIN_IMPRESSIONS && row.avgImpressionPosition !== null && row.avgImpressionPosition >= OPPORTUNITY_MIN_POSITION && row.avgImpressionPosition <= OPPORTUNITY_MAX_POSITION).sort((a, b) => b.impressions - a.impressions || b.clicks - a.clicks || compareValue(a, b)).slice(0, RESULT_LIMIT);
|
|
29593
29711
|
return {
|
|
29594
|
-
|
|
29595
|
-
|
|
29712
|
+
kind,
|
|
29713
|
+
sourceSemantics: "Bing weekly top-list evidence. Missing dimensions are unknown, not zero. Movements require observation in every week of both periods.",
|
|
29714
|
+
periods: {
|
|
29715
|
+
weeksPerPeriod: WEEKLY_PERIODS,
|
|
29716
|
+
currentDates,
|
|
29717
|
+
previousDates
|
|
29718
|
+
},
|
|
29719
|
+
coverage: {
|
|
29720
|
+
sourceRows: rows.length,
|
|
29721
|
+
currentDimensions: current.size,
|
|
29722
|
+
previousDimensions: previous.size,
|
|
29723
|
+
matchedDimensions: matchedValues.length,
|
|
29724
|
+
comparableDimensions: comparableValues.length,
|
|
29725
|
+
incompleteMatchedDimensions: matchedValues.length - comparableValues.length,
|
|
29726
|
+
currentOnlyDimensions: [...current.keys()].filter((value) => !previous.has(value)).length,
|
|
29727
|
+
previousOnlyDimensions: [...previous.keys()].filter((value) => !current.has(value)).length
|
|
29728
|
+
},
|
|
29729
|
+
thresholds: {
|
|
29730
|
+
heuristic: true,
|
|
29731
|
+
opportunityMinImpressions: OPPORTUNITY_MIN_IMPRESSIONS,
|
|
29732
|
+
opportunityPositionRange: [OPPORTUNITY_MIN_POSITION, OPPORTUNITY_MAX_POSITION],
|
|
29733
|
+
resultLimit: RESULT_LIMIT
|
|
29734
|
+
},
|
|
29735
|
+
movements,
|
|
29736
|
+
opportunities
|
|
29596
29737
|
};
|
|
29597
29738
|
}
|
|
29739
|
+
const bingAnalysisDefaults = {
|
|
29740
|
+
comparisonDays: DEFAULT_COMPARISON_DAYS,
|
|
29741
|
+
weeklyPeriods: WEEKLY_PERIODS,
|
|
29742
|
+
resultLimit: RESULT_LIMIT
|
|
29743
|
+
};
|
|
29598
29744
|
//#endregion
|
|
29599
|
-
//#region packages/core/src/
|
|
29600
|
-
const
|
|
29601
|
-
|
|
29602
|
-
|
|
29603
|
-
|
|
29604
|
-
|
|
29605
|
-
|
|
29606
|
-
|
|
29607
|
-
const
|
|
29608
|
-
const
|
|
29609
|
-
|
|
29745
|
+
//#region packages/core/src/bing/client.ts
|
|
29746
|
+
const DEFAULT_BASE_URL = "https://ssl.bing.com/webmaster/api.svc/json";
|
|
29747
|
+
const MAX_RESPONSE_BYTES$1 = 2e6;
|
|
29748
|
+
const MAX_STAT_ROWS = 400;
|
|
29749
|
+
const MAX_DIMENSION_ROWS = 8e3;
|
|
29750
|
+
const MAX_LINK_ROWS = 200;
|
|
29751
|
+
function bingDate(value) {
|
|
29752
|
+
if (typeof value !== "string") return void 0;
|
|
29753
|
+
const legacy = /^\/Date\((-?\d+)(?:[+-]\d{4})?\)\/$/.exec(value);
|
|
29754
|
+
const date = legacy ? new Date(Number(legacy[1])) : new Date(value);
|
|
29755
|
+
return Number.isNaN(date.getTime()) ? void 0 : date.toISOString().slice(0, 10);
|
|
29756
|
+
}
|
|
29757
|
+
function nonnegativeNumber(value) {
|
|
29758
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
|
|
29759
|
+
}
|
|
29760
|
+
function positionNumber(value) {
|
|
29761
|
+
if (value === void 0 || value === null || value === -1) return { valid: true };
|
|
29762
|
+
const number = nonnegativeNumber(value);
|
|
29763
|
+
return number === void 0 ? { valid: false } : {
|
|
29764
|
+
valid: true,
|
|
29765
|
+
value: number
|
|
29766
|
+
};
|
|
29767
|
+
}
|
|
29768
|
+
function isHttpUrl(value) {
|
|
29769
|
+
if (typeof value !== "string" || value.length > 2e3) return false;
|
|
29610
29770
|
try {
|
|
29611
|
-
const
|
|
29612
|
-
|
|
29613
|
-
|
|
29614
|
-
|
|
29615
|
-
status: "pass",
|
|
29616
|
-
detail: database.dbPath
|
|
29617
|
-
});
|
|
29618
|
-
} catch (error) {
|
|
29619
|
-
checks.push({
|
|
29620
|
-
id: "local-database",
|
|
29621
|
-
label: "Local database",
|
|
29622
|
-
status: "fail",
|
|
29623
|
-
detail: error instanceof Error ? error.message : "The local database could not be opened.",
|
|
29624
|
-
fix: "Upgrade or reinstall `seo`. If this still fails, report it at https://github.com/iannuttall/seo/issues."
|
|
29625
|
-
});
|
|
29771
|
+
const url = new URL(value);
|
|
29772
|
+
return url.protocol === "http:" || url.protocol === "https:";
|
|
29773
|
+
} catch {
|
|
29774
|
+
return false;
|
|
29626
29775
|
}
|
|
29627
|
-
|
|
29628
|
-
|
|
29629
|
-
|
|
29630
|
-
status: existsSync(paths.configDir) ? "pass" : "warn",
|
|
29631
|
-
detail: paths.configDir,
|
|
29632
|
-
fix: existsSync(paths.configDir) ? void 0 : "Run `seo start` or `seo auth setup-client`."
|
|
29633
|
-
});
|
|
29634
|
-
checks.push({
|
|
29635
|
-
id: "oauth-client",
|
|
29636
|
-
label: "Google credentials",
|
|
29637
|
-
status: serviceAccount.configured ? "pass" : tokenClientConfigured === false ? "fail" : serviceAccount.error ? "fail" : authMode.sharedConfigured || authMode.byoConfigured ? "pass" : "fail",
|
|
29638
|
-
detail: serviceAccount.configured ? `Service account configured for ${serviceAccount.identity}.` : tokenClientConfigured === false ? `Stored Google login uses the ${tokens?.client_source === "shared" ? "shared seo app" : "BYO client"}, but that client is not configured.` : serviceAccount.error ? serviceAccount.error : authMode.sharedConfigured ? "Shared client configured." : authMode.byoConfigured ? `BYO client configured at ${paths.oauthClientFile}.` : "No shared or BYO Google OAuth client configured.",
|
|
29639
|
-
fix: serviceAccount.configured ? void 0 : tokenClientConfigured === false ? tokens?.client_source === "shared" ? "Reinstall `seo`. If the shared client is still missing, report it at https://github.com/iannuttall/seo/issues." : "Restore the same client with `seo auth setup-client`, or run `seo auth logout` before signing in with a different client." : serviceAccount.error ? "Set one valid service account credential source, then run `seo doctor` again." : authMode.sharedConfigured || authMode.byoConfigured ? void 0 : "Run `seo auth setup-client` or set SEO_GOOGLE_CLIENT_ID and SEO_GOOGLE_CLIENT_SECRET."
|
|
29640
|
-
});
|
|
29641
|
-
checks.push({
|
|
29642
|
-
id: "oauth-client-file",
|
|
29643
|
-
label: "BYO OAuth file",
|
|
29644
|
-
status: oauthClient ? "pass" : "warn",
|
|
29645
|
-
detail: oauthClient ? paths.oauthClientFile : "No BYO OAuth client file. This is fine if using a shared client."
|
|
29646
|
-
});
|
|
29647
|
-
checks.push({
|
|
29648
|
-
id: "google-login",
|
|
29649
|
-
label: "Google login",
|
|
29650
|
-
status: serviceAccount.configured || tokens ? "pass" : "fail",
|
|
29651
|
-
detail: serviceAccount.configured ? `Using service account ${serviceAccount.identity}.` : tokens ? `Signed in as ${tokens.account_email}.` : "No Google token found.",
|
|
29652
|
-
fix: serviceAccount.configured || tokens ? void 0 : "Run `seo auth login`, or configure a service account for CI."
|
|
29653
|
-
});
|
|
29654
|
-
const grantedScopes = new Set(tokens?.scope.split(/\s+/).filter(Boolean) ?? []);
|
|
29655
|
-
for (const scope of REQUIRED_SCOPES) checks.push({
|
|
29656
|
-
id: `scope:${scope}`,
|
|
29657
|
-
label: scope.endsWith("webmasters.readonly") ? "Search Console scope" : "Google Analytics scope",
|
|
29658
|
-
status: serviceAccount.configured || tokens && grantedScopes.has(scope) ? "pass" : tokens ? "fail" : "warn",
|
|
29659
|
-
detail: serviceAccount.configured ? `${scope} requested by the service account.` : tokens ? grantedScopes.has(scope) ? scope : `Missing ${scope}.` : "Cannot inspect scopes until login is complete.",
|
|
29660
|
-
fix: !serviceAccount.configured && tokens && !grantedScopes.has(scope) ? "Run `seo auth logout` then `seo auth login`." : void 0
|
|
29661
|
-
});
|
|
29662
|
-
checks.push({
|
|
29663
|
-
id: "default-site",
|
|
29664
|
-
label: "Saved GSC property",
|
|
29665
|
-
status: "pass",
|
|
29666
|
-
detail: config.defaultSite ?? "No saved default. Human CLI commands will prompt; agents can pass --site."
|
|
29667
|
-
});
|
|
29668
|
-
checks.push({
|
|
29669
|
-
id: "default-google-analytics",
|
|
29670
|
-
label: "Saved Google Analytics property",
|
|
29671
|
-
status: "pass",
|
|
29672
|
-
detail: config.analytics.google.defaultPropertyId ?? "No saved default. Human CLI commands will prompt; agents can pass --property."
|
|
29673
|
-
});
|
|
29776
|
+
}
|
|
29777
|
+
function boundedRows(rows, invalidRows, limit = MAX_STAT_ROWS) {
|
|
29778
|
+
const capped = rows.length > limit;
|
|
29674
29779
|
return {
|
|
29675
|
-
|
|
29676
|
-
|
|
29677
|
-
|
|
29780
|
+
rows: capped ? rows.slice(-limit) : rows,
|
|
29781
|
+
invalidRows,
|
|
29782
|
+
capped,
|
|
29783
|
+
returnedRows: rows.length
|
|
29678
29784
|
};
|
|
29679
29785
|
}
|
|
29680
|
-
|
|
29681
|
-
|
|
29682
|
-
const
|
|
29683
|
-
|
|
29684
|
-
|
|
29685
|
-
|
|
29686
|
-
|
|
29687
|
-
|
|
29688
|
-
|
|
29689
|
-
|
|
29690
|
-
"
|
|
29691
|
-
|
|
29692
|
-
|
|
29693
|
-
|
|
29694
|
-
|
|
29695
|
-
|
|
29696
|
-
|
|
29697
|
-
|
|
29698
|
-
|
|
29699
|
-
|
|
29786
|
+
function dimensionValue(value, kind) {
|
|
29787
|
+
if (typeof value !== "string") return void 0;
|
|
29788
|
+
const normalized = value.trim();
|
|
29789
|
+
if (!normalized || normalized.length > 2e3) return void 0;
|
|
29790
|
+
if (kind === "page" && !isHttpUrl(normalized)) return void 0;
|
|
29791
|
+
return normalized;
|
|
29792
|
+
}
|
|
29793
|
+
function errorMessage(body) {
|
|
29794
|
+
try {
|
|
29795
|
+
const parsed = JSON.parse(body);
|
|
29796
|
+
return typeof parsed.Message === "string" ? parsed.Message : void 0;
|
|
29797
|
+
} catch {
|
|
29798
|
+
return;
|
|
29799
|
+
}
|
|
29800
|
+
}
|
|
29801
|
+
var BingWebmasterClient = class {
|
|
29802
|
+
credentials;
|
|
29803
|
+
authentication;
|
|
29804
|
+
baseUrl;
|
|
29805
|
+
fetchImpl;
|
|
29806
|
+
timeoutMs;
|
|
29807
|
+
constructor(credentials, options = {}) {
|
|
29808
|
+
this.credentials = credentials;
|
|
29809
|
+
if (!(credentials.apiKey?.trim() || credentials.accessToken?.trim())) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster credentials are empty.");
|
|
29810
|
+
this.authentication = credentials.apiKey ? "api-key" : "oauth";
|
|
29811
|
+
this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, "");
|
|
29812
|
+
this.fetchImpl = options.fetchImpl ?? publicHttpFetch;
|
|
29813
|
+
this.timeoutMs = options.timeoutMs ?? 3e4;
|
|
29814
|
+
}
|
|
29815
|
+
async request(method, params = {}) {
|
|
29816
|
+
const url = new URL(`${this.baseUrl}/${method}`);
|
|
29817
|
+
for (const [key, value] of Object.entries(params)) url.searchParams.set(key, value);
|
|
29818
|
+
if (this.credentials.apiKey) url.searchParams.set("apikey", this.credentials.apiKey);
|
|
29819
|
+
let response;
|
|
29820
|
+
try {
|
|
29821
|
+
response = await this.fetchImpl(url.toString(), {
|
|
29822
|
+
headers: {
|
|
29823
|
+
accept: "application/json",
|
|
29824
|
+
...this.credentials.accessToken ? { authorization: `Bearer ${this.credentials.accessToken}` } : {}
|
|
29825
|
+
},
|
|
29826
|
+
signal: AbortSignal.timeout(this.timeoutMs)
|
|
29827
|
+
});
|
|
29828
|
+
} catch (error) {
|
|
29829
|
+
if (error instanceof SeoError) throw error;
|
|
29830
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster did not respond: ${error instanceof Error ? error.message : String(error)}`);
|
|
29831
|
+
}
|
|
29832
|
+
const body = await readBoundedResponseText(response, MAX_RESPONSE_BYTES$1, "Bing Webmaster response");
|
|
29833
|
+
if (!response.ok) {
|
|
29834
|
+
const message = errorMessage(body);
|
|
29835
|
+
if (response.status === 401 || /invalidapikey|unauthorized/i.test(message ?? "")) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster rejected the saved credential. Run `seo providers bing connect` again.");
|
|
29836
|
+
if (response.status === 403) throw new SeoError("ACCESS_DENIED", "Bing Webmaster denied access to this site. Check that it is verified for the connected account.");
|
|
29837
|
+
if (response.status === 429) throw new SeoError("RATE_LIMITED", "Bing Webmaster rate limited this request. Try again later.");
|
|
29838
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster request failed with HTTP ${response.status}${message ? ` (${message})` : ""}.`);
|
|
29839
|
+
}
|
|
29840
|
+
try {
|
|
29841
|
+
return JSON.parse(body).d;
|
|
29842
|
+
} catch {
|
|
29843
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid JSON.");
|
|
29844
|
+
}
|
|
29845
|
+
}
|
|
29846
|
+
async listSites() {
|
|
29847
|
+
const data = await this.request("GetUserSites");
|
|
29848
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned an invalid site list.");
|
|
29849
|
+
const sites = [];
|
|
29850
|
+
let invalidRows = 0;
|
|
29851
|
+
for (const item of data) {
|
|
29852
|
+
if (!item || typeof item !== "object") {
|
|
29853
|
+
invalidRows += 1;
|
|
29854
|
+
continue;
|
|
29855
|
+
}
|
|
29856
|
+
const row = item;
|
|
29857
|
+
if (typeof row.Url !== "string" || typeof row.IsVerified !== "boolean") {
|
|
29858
|
+
invalidRows += 1;
|
|
29859
|
+
continue;
|
|
29860
|
+
}
|
|
29861
|
+
sites.push({
|
|
29862
|
+
url: row.Url,
|
|
29863
|
+
isVerified: row.IsVerified
|
|
29864
|
+
});
|
|
29865
|
+
}
|
|
29866
|
+
sites.sort((a, b) => a.url.localeCompare(b.url, "en"));
|
|
29867
|
+
return {
|
|
29868
|
+
sites,
|
|
29869
|
+
invalidRows
|
|
29870
|
+
};
|
|
29871
|
+
}
|
|
29872
|
+
async getTraffic(siteUrl) {
|
|
29873
|
+
const data = await this.request("GetRankAndTrafficStats", { siteUrl });
|
|
29874
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid traffic statistics.");
|
|
29875
|
+
const rows = [];
|
|
29876
|
+
let invalidRows = 0;
|
|
29877
|
+
for (const item of data) {
|
|
29878
|
+
if (!item || typeof item !== "object") {
|
|
29879
|
+
invalidRows += 1;
|
|
29880
|
+
continue;
|
|
29881
|
+
}
|
|
29882
|
+
const row = item;
|
|
29883
|
+
const date = bingDate(row.Date);
|
|
29884
|
+
const clicks = nonnegativeNumber(row.Clicks);
|
|
29885
|
+
const impressions = nonnegativeNumber(row.Impressions);
|
|
29886
|
+
if (!date || clicks === void 0 || impressions === void 0) {
|
|
29887
|
+
invalidRows += 1;
|
|
29888
|
+
continue;
|
|
29889
|
+
}
|
|
29890
|
+
rows.push({
|
|
29891
|
+
date,
|
|
29892
|
+
clicks,
|
|
29893
|
+
impressions
|
|
29894
|
+
});
|
|
29895
|
+
}
|
|
29896
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en") || a.clicks - b.clicks || a.impressions - b.impressions);
|
|
29897
|
+
return boundedRows(rows, invalidRows);
|
|
29898
|
+
}
|
|
29899
|
+
async getCrawlStats(siteUrl) {
|
|
29900
|
+
const data = await this.request("GetCrawlStats", { siteUrl });
|
|
29901
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid crawl statistics.");
|
|
29902
|
+
const fieldMap = {
|
|
29903
|
+
AllOtherCodes: "allOtherCodes",
|
|
29904
|
+
BlockedByRobotsTxt: "blockedByRobotsTxt",
|
|
29905
|
+
Code2xx: "code2xx",
|
|
29906
|
+
Code301: "code301",
|
|
29907
|
+
Code302: "code302",
|
|
29908
|
+
Code4xx: "code4xx",
|
|
29909
|
+
Code5xx: "code5xx",
|
|
29910
|
+
ConnectionTimeout: "connectionTimeout",
|
|
29911
|
+
ContainsMalware: "containsMalware",
|
|
29912
|
+
CrawledPages: "crawledPages",
|
|
29913
|
+
CrawlErrors: "crawlErrors",
|
|
29914
|
+
DnsFailures: "dnsFailures",
|
|
29915
|
+
InIndex: "inIndex",
|
|
29916
|
+
InLinks: "inLinks"
|
|
29917
|
+
};
|
|
29918
|
+
const rows = [];
|
|
29919
|
+
let invalidRows = 0;
|
|
29920
|
+
for (const item of data) {
|
|
29921
|
+
if (!item || typeof item !== "object") {
|
|
29922
|
+
invalidRows += 1;
|
|
29923
|
+
continue;
|
|
29924
|
+
}
|
|
29925
|
+
const source = item;
|
|
29926
|
+
const date = bingDate(source.Date);
|
|
29927
|
+
if (!date) {
|
|
29928
|
+
invalidRows += 1;
|
|
29929
|
+
continue;
|
|
29930
|
+
}
|
|
29931
|
+
const row = { date };
|
|
29932
|
+
let invalid = false;
|
|
29933
|
+
for (const [providerField, field] of Object.entries(fieldMap)) {
|
|
29934
|
+
const value = source[providerField];
|
|
29935
|
+
if (value === void 0 || value === null) continue;
|
|
29936
|
+
const number = nonnegativeNumber(value);
|
|
29937
|
+
if (number === void 0) {
|
|
29938
|
+
invalid = true;
|
|
29939
|
+
break;
|
|
29940
|
+
}
|
|
29941
|
+
row[field] = number;
|
|
29942
|
+
}
|
|
29943
|
+
if (invalid) {
|
|
29944
|
+
invalidRows += 1;
|
|
29945
|
+
continue;
|
|
29946
|
+
}
|
|
29947
|
+
rows.push(row);
|
|
29948
|
+
}
|
|
29949
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en"));
|
|
29950
|
+
return boundedRows(rows, invalidRows);
|
|
29951
|
+
}
|
|
29952
|
+
async getQueryStats(siteUrl) {
|
|
29953
|
+
return this.getDimensionStats("GetQueryStats", siteUrl, "query");
|
|
29954
|
+
}
|
|
29955
|
+
async getPageStats(siteUrl) {
|
|
29956
|
+
return this.getDimensionStats("GetPageStats", siteUrl, "page");
|
|
29957
|
+
}
|
|
29958
|
+
async getDimensionStats(method, siteUrl, kind) {
|
|
29959
|
+
const data = await this.request(method, { siteUrl });
|
|
29960
|
+
if (!Array.isArray(data)) throw new SeoError("PROVIDER_UNAVAILABLE", `Bing Webmaster returned invalid ${kind} statistics.`);
|
|
29961
|
+
const rows = [];
|
|
29962
|
+
let invalidRows = 0;
|
|
29963
|
+
for (const item of data) {
|
|
29964
|
+
if (!item || typeof item !== "object") {
|
|
29965
|
+
invalidRows += 1;
|
|
29966
|
+
continue;
|
|
29967
|
+
}
|
|
29968
|
+
const source = item;
|
|
29969
|
+
const date = bingDate(source.Date);
|
|
29970
|
+
const value = dimensionValue(source.Query, kind);
|
|
29971
|
+
const clicks = nonnegativeNumber(source.Clicks);
|
|
29972
|
+
const impressions = nonnegativeNumber(source.Impressions);
|
|
29973
|
+
const clickPosition = positionNumber(source.AvgClickPosition);
|
|
29974
|
+
const impressionPosition = positionNumber(source.AvgImpressionPosition);
|
|
29975
|
+
if (!date || !value || clicks === void 0 || impressions === void 0 || !clickPosition.valid || !impressionPosition.valid) {
|
|
29976
|
+
invalidRows += 1;
|
|
29977
|
+
continue;
|
|
29978
|
+
}
|
|
29979
|
+
rows.push({
|
|
29980
|
+
date,
|
|
29981
|
+
value,
|
|
29982
|
+
clicks,
|
|
29983
|
+
impressions,
|
|
29984
|
+
avgClickPosition: clickPosition.value,
|
|
29985
|
+
avgImpressionPosition: impressionPosition.value
|
|
29986
|
+
});
|
|
29987
|
+
}
|
|
29988
|
+
rows.sort((a, b) => a.date.localeCompare(b.date, "en") || a.value.localeCompare(b.value, "en"));
|
|
29989
|
+
return boundedRows(rows, invalidRows, MAX_DIMENSION_ROWS);
|
|
29990
|
+
}
|
|
29991
|
+
async getLinkCounts(siteUrl, page) {
|
|
29992
|
+
const data = await this.request("GetLinkCounts", {
|
|
29993
|
+
siteUrl,
|
|
29994
|
+
page: String(page)
|
|
29995
|
+
});
|
|
29996
|
+
if (!data || typeof data !== "object") throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link counts.");
|
|
29997
|
+
const response = data;
|
|
29998
|
+
if (!Array.isArray(response.Links)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link counts.");
|
|
29999
|
+
const rows = [];
|
|
30000
|
+
let invalidRows = 0;
|
|
30001
|
+
for (const item of response.Links) {
|
|
30002
|
+
if (!item || typeof item !== "object") {
|
|
30003
|
+
invalidRows += 1;
|
|
30004
|
+
continue;
|
|
30005
|
+
}
|
|
30006
|
+
const row = item;
|
|
30007
|
+
const count = nonnegativeNumber(row.Count);
|
|
30008
|
+
if (!isHttpUrl(row.Url) || count === void 0) {
|
|
30009
|
+
invalidRows += 1;
|
|
30010
|
+
continue;
|
|
30011
|
+
}
|
|
30012
|
+
rows.push({
|
|
30013
|
+
url: row.Url,
|
|
30014
|
+
count
|
|
30015
|
+
});
|
|
30016
|
+
}
|
|
30017
|
+
rows.sort((a, b) => b.count - a.count || a.url.localeCompare(b.url, "en"));
|
|
30018
|
+
return linkPage(rows, response.TotalPages, invalidRows);
|
|
30019
|
+
}
|
|
30020
|
+
async getUrlLinks(siteUrl, link, page) {
|
|
30021
|
+
const data = await this.request("GetUrlLinks", {
|
|
30022
|
+
siteUrl,
|
|
30023
|
+
link,
|
|
30024
|
+
page: String(page)
|
|
30025
|
+
});
|
|
30026
|
+
if (!data || typeof data !== "object") throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid referring links.");
|
|
30027
|
+
const response = data;
|
|
30028
|
+
if (!Array.isArray(response.Details)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid referring links.");
|
|
30029
|
+
const rows = [];
|
|
30030
|
+
let invalidRows = 0;
|
|
30031
|
+
for (const item of response.Details) {
|
|
30032
|
+
if (!item || typeof item !== "object") {
|
|
30033
|
+
invalidRows += 1;
|
|
30034
|
+
continue;
|
|
30035
|
+
}
|
|
30036
|
+
const row = item;
|
|
30037
|
+
if (!isHttpUrl(row.Url)) {
|
|
30038
|
+
invalidRows += 1;
|
|
30039
|
+
continue;
|
|
30040
|
+
}
|
|
30041
|
+
rows.push({
|
|
30042
|
+
url: row.Url,
|
|
30043
|
+
anchorText: typeof row.AnchorText === "string" && row.AnchorText.trim() ? row.AnchorText.trim() : void 0
|
|
30044
|
+
});
|
|
30045
|
+
}
|
|
30046
|
+
rows.sort((a, b) => a.url.localeCompare(b.url, "en") || (a.anchorText ?? "").localeCompare(b.anchorText ?? "", "en"));
|
|
30047
|
+
return linkPage(rows, response.TotalPages, invalidRows);
|
|
30048
|
+
}
|
|
30049
|
+
};
|
|
30050
|
+
function linkPage(rows, totalPagesValue, invalidRows) {
|
|
30051
|
+
const totalPages = nonnegativeNumber(totalPagesValue);
|
|
30052
|
+
if (totalPages === void 0 || !Number.isInteger(totalPages)) throw new SeoError("PROVIDER_UNAVAILABLE", "Bing Webmaster returned invalid link pagination.");
|
|
30053
|
+
const capped = rows.length > MAX_LINK_ROWS;
|
|
30054
|
+
return {
|
|
30055
|
+
rows: capped ? rows.slice(0, MAX_LINK_ROWS) : rows,
|
|
30056
|
+
totalPages,
|
|
30057
|
+
invalidRows,
|
|
30058
|
+
capped,
|
|
30059
|
+
returnedRows: rows.length
|
|
30060
|
+
};
|
|
30061
|
+
}
|
|
30062
|
+
//#endregion
|
|
30063
|
+
//#region packages/core/src/storage/provider-secrets.ts
|
|
30064
|
+
const KEYRING_SERVICE = "seo";
|
|
30065
|
+
const PRIVATE_FILE_MODE = 384;
|
|
30066
|
+
function account(name) {
|
|
30067
|
+
return `provider:${name}`;
|
|
30068
|
+
}
|
|
30069
|
+
function readFile$1() {
|
|
30070
|
+
return readJsonFile(getSeoCliPaths().providerSecretsFile) ?? {
|
|
30071
|
+
version: 1,
|
|
30072
|
+
secrets: {}
|
|
30073
|
+
};
|
|
30074
|
+
}
|
|
30075
|
+
function writeFile$1(file) {
|
|
30076
|
+
const path = getSeoCliPaths().providerSecretsFile;
|
|
30077
|
+
if (Object.keys(file.secrets).length === 0) {
|
|
30078
|
+
safeRemove(path);
|
|
30079
|
+
return;
|
|
30080
|
+
}
|
|
30081
|
+
writeJsonAtomic(path, file, PRIVATE_FILE_MODE);
|
|
30082
|
+
}
|
|
30083
|
+
function deleteFileSecret(name) {
|
|
30084
|
+
const file = readFile$1();
|
|
30085
|
+
if (!(name in file.secrets)) return;
|
|
30086
|
+
delete file.secrets[name];
|
|
30087
|
+
writeFile$1(file);
|
|
30088
|
+
}
|
|
30089
|
+
async function readProviderSecret(input) {
|
|
30090
|
+
const environment = input.env ?? process.env;
|
|
30091
|
+
const environmentValue = input.envVar ? environment[input.envVar]?.trim() : void 0;
|
|
30092
|
+
if (environmentValue) return {
|
|
30093
|
+
value: environmentValue,
|
|
30094
|
+
source: "environment"
|
|
30095
|
+
};
|
|
30096
|
+
if (readConfig().security.useKeychain) try {
|
|
30097
|
+
const value = await getKeyringPassword(KEYRING_SERVICE, account(input.name));
|
|
30098
|
+
if (value) return {
|
|
30099
|
+
value,
|
|
30100
|
+
source: "keychain"
|
|
30101
|
+
};
|
|
30102
|
+
} catch {}
|
|
30103
|
+
const value = readFile$1().secrets[input.name];
|
|
30104
|
+
return value ? {
|
|
30105
|
+
value,
|
|
30106
|
+
source: "file"
|
|
30107
|
+
} : void 0;
|
|
30108
|
+
}
|
|
30109
|
+
async function writeProviderSecret(name, value) {
|
|
30110
|
+
const normalized = value.trim();
|
|
30111
|
+
if (!normalized) throw new Error("Provider secret cannot be empty.");
|
|
30112
|
+
if (readConfig().security.useKeychain) try {
|
|
30113
|
+
await setKeyringPassword(KEYRING_SERVICE, account(name), normalized);
|
|
30114
|
+
deleteFileSecret(name);
|
|
30115
|
+
return "keychain";
|
|
30116
|
+
} catch {}
|
|
30117
|
+
const file = readFile$1();
|
|
30118
|
+
file.secrets[name] = normalized;
|
|
30119
|
+
writeFile$1(file);
|
|
30120
|
+
return "file";
|
|
30121
|
+
}
|
|
30122
|
+
async function deleteProviderSecret(name) {
|
|
30123
|
+
await deleteKeyringPassword(KEYRING_SERVICE, account(name)).catch(() => false);
|
|
30124
|
+
deleteFileSecret(name);
|
|
30125
|
+
}
|
|
30126
|
+
//#endregion
|
|
30127
|
+
//#region packages/core/src/bing/credentials.ts
|
|
30128
|
+
const BING_API_KEY_SECRET = "bing-api-key";
|
|
30129
|
+
const BING_API_KEY_ENV = "SEO_BING_API_KEY";
|
|
30130
|
+
async function createBingWebmasterClient(input = {}) {
|
|
30131
|
+
const secret = await readProviderSecret({
|
|
30132
|
+
name: BING_API_KEY_SECRET,
|
|
30133
|
+
envVar: BING_API_KEY_ENV,
|
|
30134
|
+
env: input.env
|
|
30135
|
+
});
|
|
30136
|
+
if (!secret) throw new SeoError("AUTH_REQUIRED", "Bing Webmaster is not connected. Run `seo providers bing connect`, or set SEO_BING_API_KEY for this process.");
|
|
30137
|
+
return {
|
|
30138
|
+
client: new BingWebmasterClient({ apiKey: secret.value }),
|
|
30139
|
+
credentialSource: secret.source
|
|
30140
|
+
};
|
|
30141
|
+
}
|
|
30142
|
+
//#endregion
|
|
30143
|
+
//#region packages/core/src/bing/overview.ts
|
|
30144
|
+
const DAILY_OUTPUT_ROWS = 14;
|
|
30145
|
+
function message(error) {
|
|
30146
|
+
return error instanceof Error ? error.message : String(error);
|
|
30147
|
+
}
|
|
30148
|
+
function range(rows) {
|
|
30149
|
+
return rows.length ? {
|
|
30150
|
+
startDate: rows[0]?.date ?? "",
|
|
30151
|
+
endDate: rows.at(-1)?.date ?? ""
|
|
30152
|
+
} : void 0;
|
|
30153
|
+
}
|
|
30154
|
+
function dailyOutput(data) {
|
|
30155
|
+
const rows = data.rows.slice(-14);
|
|
30156
|
+
return {
|
|
30157
|
+
...data,
|
|
30158
|
+
rows,
|
|
30159
|
+
retainedRows: data.rows.length,
|
|
30160
|
+
outputSelection: {
|
|
30161
|
+
strategy: "most-recent",
|
|
30162
|
+
availableRows: data.rows.length,
|
|
30163
|
+
returnedRows: rows.length,
|
|
30164
|
+
omittedRows: Math.max(0, data.rows.length - rows.length)
|
|
30165
|
+
}
|
|
30166
|
+
};
|
|
30167
|
+
}
|
|
30168
|
+
async function section(work, options) {
|
|
30169
|
+
try {
|
|
30170
|
+
const data = await work();
|
|
30171
|
+
const reasons = [
|
|
30172
|
+
...data.invalidRows ? [`${data.invalidRows} invalid provider rows were excluded.`] : [],
|
|
30173
|
+
...data.capped ? [`The provider response exceeded the ${options.rowLimit}-row limit.`] : [],
|
|
30174
|
+
...options.sourceWarning ? [options.sourceWarning] : []
|
|
30175
|
+
];
|
|
30176
|
+
return reasons.length ? {
|
|
30177
|
+
status: "partial",
|
|
30178
|
+
data,
|
|
30179
|
+
warning: reasons.join(" ")
|
|
30180
|
+
} : {
|
|
30181
|
+
status: "complete",
|
|
30182
|
+
data
|
|
30183
|
+
};
|
|
30184
|
+
} catch (error) {
|
|
30185
|
+
if (error instanceof SeoError && (error.code === "AUTH_REQUIRED" || error.code === "ACCESS_DENIED")) throw error;
|
|
30186
|
+
return {
|
|
30187
|
+
status: "unavailable",
|
|
30188
|
+
warning: message(error)
|
|
30189
|
+
};
|
|
30190
|
+
}
|
|
30191
|
+
}
|
|
30192
|
+
function changeFinding(input) {
|
|
30193
|
+
return {
|
|
30194
|
+
code: input.code,
|
|
30195
|
+
severity: "warning",
|
|
30196
|
+
title: input.title,
|
|
30197
|
+
evidence: {
|
|
30198
|
+
current: input.current,
|
|
30199
|
+
previous: input.previous,
|
|
30200
|
+
absoluteChange: input.absolute,
|
|
30201
|
+
percentChange: input.percent,
|
|
30202
|
+
currentDate: input.dates.current,
|
|
30203
|
+
previousDate: input.dates.previous
|
|
30204
|
+
},
|
|
30205
|
+
interpretation: input.interpretation,
|
|
30206
|
+
verification: input.verification
|
|
30207
|
+
};
|
|
30208
|
+
}
|
|
30209
|
+
function buildFindings(input) {
|
|
30210
|
+
const findings = [];
|
|
30211
|
+
const traffic = input.traffic;
|
|
30212
|
+
if (traffic && traffic.current.observedDays === traffic.current.expectedDays && traffic.previous.observedDays === traffic.previous.expectedDays && (traffic.changes.clicksPercent !== null && Math.abs(traffic.changes.clicksPercent) >= 10 || traffic.changes.impressionsPercent !== null && Math.abs(traffic.changes.impressionsPercent) >= 10)) {
|
|
30213
|
+
const metric = traffic.changes.clicksPercent !== null && Math.abs(traffic.changes.clicksPercent) >= 10 ? "clicks" : "impressions";
|
|
30214
|
+
const declined = traffic.changes[metric] < 0;
|
|
30215
|
+
findings.push({
|
|
30216
|
+
code: `bing_${metric}_${declined ? "declined" : "increased"}`,
|
|
30217
|
+
severity: declined ? "warning" : "info",
|
|
30218
|
+
title: `Bing ${metric} ${declined ? "declined" : "increased"} in the latest complete period`,
|
|
30219
|
+
evidence: {
|
|
30220
|
+
currentClicks: traffic.current.clicks,
|
|
30221
|
+
previousClicks: traffic.previous.clicks,
|
|
30222
|
+
clicksChange: traffic.changes.clicks,
|
|
30223
|
+
clicksPercentChange: traffic.changes.clicksPercent,
|
|
30224
|
+
currentImpressions: traffic.current.impressions,
|
|
30225
|
+
previousImpressions: traffic.previous.impressions,
|
|
30226
|
+
impressionsChange: traffic.changes.impressions,
|
|
30227
|
+
impressionsPercentChange: traffic.changes.impressionsPercent,
|
|
30228
|
+
ctrPercentagePointChange: traffic.changes.ctrPercentagePoints,
|
|
30229
|
+
currentRange: {
|
|
30230
|
+
startDate: traffic.current.startDate,
|
|
30231
|
+
endDate: traffic.current.endDate
|
|
30232
|
+
},
|
|
30233
|
+
previousRange: {
|
|
30234
|
+
startDate: traffic.previous.startDate,
|
|
30235
|
+
endDate: traffic.previous.endDate
|
|
30236
|
+
}
|
|
30237
|
+
},
|
|
30238
|
+
interpretation: "This is a Bing-observed traffic change. It does not establish the cause or describe Google traffic.",
|
|
30239
|
+
verification: "Review the query and page evidence for the same dates, then compare current pages and crawl health before changing content."
|
|
30240
|
+
});
|
|
30241
|
+
}
|
|
30242
|
+
const crawl = input.crawl;
|
|
30243
|
+
if (crawl?.previous) {
|
|
30244
|
+
const dates = {
|
|
30245
|
+
current: crawl.current.date,
|
|
30246
|
+
previous: crawl.previous.date
|
|
30247
|
+
};
|
|
30248
|
+
const errors = crawl.changes.crawlErrors;
|
|
30249
|
+
if (errors && errors.absolute >= Math.max(10, Math.ceil(errors.previous * .5))) findings.push(changeFinding({
|
|
30250
|
+
code: "bing_crawl_errors_increased",
|
|
30251
|
+
title: "Bing crawl errors increased",
|
|
30252
|
+
...errors,
|
|
30253
|
+
dates,
|
|
30254
|
+
interpretation: "Bing reported more crawl errors in its latest daily snapshot. The count may include URLs outside the current sitemap.",
|
|
30255
|
+
verification: "Run the fast sitemap health check, inspect Bing crawl issue samples, then audit any affected URLs directly."
|
|
30256
|
+
}));
|
|
30257
|
+
const code4xx = crawl.changes.code4xx;
|
|
30258
|
+
if (code4xx && code4xx.absolute >= Math.max(10, Math.ceil(code4xx.previous * .5))) findings.push(changeFinding({
|
|
30259
|
+
code: "bing_4xx_increased",
|
|
30260
|
+
title: "Bing-observed 4xx responses increased",
|
|
30261
|
+
...code4xx,
|
|
30262
|
+
dates,
|
|
30263
|
+
interpretation: "Bing encountered more 4xx responses. Intentional removals and obsolete discovered URLs can contribute to this count.",
|
|
30264
|
+
verification: "Inspect Bing crawl issue URLs and recover or redirect only URLs that should still exist or retain search value."
|
|
30265
|
+
}));
|
|
30266
|
+
const code5xx = crawl.changes.code5xx;
|
|
30267
|
+
if (code5xx && code5xx.current > 0 && code5xx.absolute > 0) findings.push(changeFinding({
|
|
30268
|
+
code: "bing_5xx_increased",
|
|
30269
|
+
title: "Bing-observed server errors increased",
|
|
30270
|
+
...code5xx,
|
|
30271
|
+
dates,
|
|
30272
|
+
interpretation: "Bing encountered more server errors in the latest snapshot. Even a small increase deserves a live check.",
|
|
30273
|
+
verification: "Check current server logs and run targeted URL audits before treating the provider count as an active outage."
|
|
30274
|
+
}));
|
|
30275
|
+
}
|
|
30276
|
+
for (const [kind, analysis] of [["query", input.queries], ["page", input.pages]]) {
|
|
30277
|
+
if (!analysis?.opportunities.length) continue;
|
|
30278
|
+
findings.push({
|
|
30279
|
+
code: `bing_${kind}_opportunities`,
|
|
30280
|
+
severity: "review",
|
|
30281
|
+
title: `${analysis.opportunities.length} Bing ${kind} opportunities need review`,
|
|
30282
|
+
evidence: {
|
|
30283
|
+
retainedOpportunities: analysis.opportunities.length,
|
|
30284
|
+
currentDimensions: analysis.coverage.currentDimensions,
|
|
30285
|
+
matchedDimensions: analysis.coverage.matchedDimensions,
|
|
30286
|
+
comparableDimensions: analysis.coverage.comparableDimensions,
|
|
30287
|
+
thresholds: analysis.thresholds
|
|
30288
|
+
},
|
|
30289
|
+
interpretation: "These are high-impression top-list entries currently observed around positions 4 to 20. The threshold is a prioritization heuristic, not a search-engine rule.",
|
|
30290
|
+
verification: kind === "query" ? "Check intent and the ranking page before changing copy or creating content." : "Audit the selected page and its matching queries before changing internal links, titles, or content."
|
|
30291
|
+
});
|
|
30292
|
+
}
|
|
30293
|
+
return findings;
|
|
30294
|
+
}
|
|
30295
|
+
async function bingWebmasterOverview(input) {
|
|
30296
|
+
if (!input.site.trim()) throw new SeoError("INVALID_INPUT", "Pass a Bing Webmaster site URL.");
|
|
30297
|
+
const resolved = input.client ? {
|
|
30298
|
+
client: input.client,
|
|
30299
|
+
credentialSource: input.credentialSource ?? "environment"
|
|
30300
|
+
} : await createBingWebmasterClient();
|
|
30301
|
+
const observedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
30302
|
+
const topListWarning = "Bing returns weekly top-list rows, so dimensions missing from a period remain unknown.";
|
|
30303
|
+
const [traffic, crawl, queries, pages] = await Promise.all([
|
|
30304
|
+
section(() => resolved.client.getTraffic(input.site), { rowLimit: 400 }),
|
|
30305
|
+
section(() => resolved.client.getCrawlStats(input.site), { rowLimit: 400 }),
|
|
30306
|
+
section(() => resolved.client.getQueryStats(input.site), {
|
|
30307
|
+
rowLimit: 8e3,
|
|
30308
|
+
sourceWarning: topListWarning
|
|
30309
|
+
}),
|
|
30310
|
+
section(() => resolved.client.getPageStats(input.site), {
|
|
30311
|
+
rowLimit: 8e3,
|
|
30312
|
+
sourceWarning: topListWarning
|
|
30313
|
+
})
|
|
30314
|
+
]);
|
|
30315
|
+
const statuses = [
|
|
30316
|
+
traffic.status,
|
|
30317
|
+
crawl.status,
|
|
30318
|
+
queries.status,
|
|
30319
|
+
pages.status
|
|
30320
|
+
];
|
|
30321
|
+
const dataStatus = statuses.every((status) => status === "complete") ? "complete" : statuses.every((status) => status === "unavailable") ? "unavailable" : "partial";
|
|
30322
|
+
const trafficAnalysis = traffic.status === "unavailable" ? void 0 : analyzeBingTraffic(traffic.data.rows);
|
|
30323
|
+
const crawlAnalysis = crawl.status === "unavailable" ? void 0 : analyzeBingCrawl(crawl.data.rows);
|
|
30324
|
+
const queryAnalysis = queries.status === "unavailable" ? void 0 : analyzeBingDimensions(queries.data.rows, "query");
|
|
30325
|
+
const pageAnalysis = pages.status === "unavailable" ? void 0 : analyzeBingDimensions(pages.data.rows, "page");
|
|
30326
|
+
const findings = buildFindings({
|
|
30327
|
+
traffic: trafficAnalysis,
|
|
30328
|
+
crawl: crawlAnalysis,
|
|
30329
|
+
queries: queryAnalysis,
|
|
30330
|
+
pages: pageAnalysis
|
|
30331
|
+
});
|
|
30332
|
+
return {
|
|
30333
|
+
schemaVersion: 2,
|
|
30334
|
+
site: input.site,
|
|
30335
|
+
generatedAt: observedAt,
|
|
30336
|
+
dataStatus,
|
|
30337
|
+
summary: {
|
|
30338
|
+
findings: findings.length,
|
|
30339
|
+
warnings: findings.filter((finding) => finding.severity === "warning").length,
|
|
30340
|
+
reviewItems: findings.filter((finding) => finding.severity === "review").length
|
|
30341
|
+
},
|
|
30342
|
+
findings,
|
|
30343
|
+
outputBudget: {
|
|
30344
|
+
maxFindings: 6,
|
|
30345
|
+
maxDailyRowsPerSection: DAILY_OUTPUT_ROWS,
|
|
30346
|
+
maxDailyRowsTotal: DAILY_OUTPUT_ROWS * 2,
|
|
30347
|
+
maxDimensionItemsPerList: 10,
|
|
30348
|
+
maxDimensionItemsTotal: 40
|
|
30349
|
+
},
|
|
30350
|
+
provenance: {
|
|
30351
|
+
provider: "bing-webmaster",
|
|
30352
|
+
authentication: resolved.client.authentication,
|
|
30353
|
+
credentialSource: resolved.credentialSource,
|
|
30354
|
+
observedAt,
|
|
30355
|
+
cached: false,
|
|
30356
|
+
rowLimits: {
|
|
30357
|
+
traffic: 400,
|
|
30358
|
+
crawl: 400,
|
|
30359
|
+
queries: 8e3,
|
|
30360
|
+
pages: 8e3
|
|
30361
|
+
},
|
|
30362
|
+
methods: [
|
|
30363
|
+
"GetRankAndTrafficStats",
|
|
30364
|
+
"GetCrawlStats",
|
|
30365
|
+
"GetQueryStats",
|
|
30366
|
+
"GetPageStats"
|
|
30367
|
+
]
|
|
30368
|
+
},
|
|
30369
|
+
traffic: traffic.status === "unavailable" ? traffic : {
|
|
30370
|
+
...traffic,
|
|
30371
|
+
data: {
|
|
30372
|
+
...dailyOutput(traffic.data),
|
|
30373
|
+
range: range(traffic.data.rows),
|
|
30374
|
+
clicks: traffic.data.rows.reduce((total, row) => total + row.clicks, 0),
|
|
30375
|
+
impressions: traffic.data.rows.reduce((total, row) => total + row.impressions, 0),
|
|
30376
|
+
analysis: trafficAnalysis
|
|
30377
|
+
}
|
|
30378
|
+
},
|
|
30379
|
+
crawl: crawl.status === "unavailable" ? crawl : {
|
|
30380
|
+
...crawl,
|
|
30381
|
+
data: {
|
|
30382
|
+
...dailyOutput(crawl.data),
|
|
30383
|
+
range: range(crawl.data.rows),
|
|
30384
|
+
latest: crawl.data.rows.at(-1),
|
|
30385
|
+
analysis: crawlAnalysis
|
|
30386
|
+
}
|
|
30387
|
+
},
|
|
30388
|
+
queries: queries.status === "unavailable" ? queries : {
|
|
30389
|
+
...queries,
|
|
30390
|
+
data: {
|
|
30391
|
+
returnedRows: queries.data.returnedRows,
|
|
30392
|
+
retainedRows: queries.data.rows.length,
|
|
30393
|
+
invalidRows: queries.data.invalidRows,
|
|
30394
|
+
capped: queries.data.capped,
|
|
30395
|
+
range: range(queries.data.rows),
|
|
30396
|
+
analysis: queryAnalysis
|
|
30397
|
+
}
|
|
30398
|
+
},
|
|
30399
|
+
pages: pages.status === "unavailable" ? pages : {
|
|
30400
|
+
...pages,
|
|
30401
|
+
data: {
|
|
30402
|
+
returnedRows: pages.data.returnedRows,
|
|
30403
|
+
retainedRows: pages.data.rows.length,
|
|
30404
|
+
invalidRows: pages.data.invalidRows,
|
|
30405
|
+
capped: pages.data.capped,
|
|
30406
|
+
range: range(pages.data.rows),
|
|
30407
|
+
analysis: pageAnalysis
|
|
30408
|
+
}
|
|
30409
|
+
},
|
|
30410
|
+
caveats: [
|
|
30411
|
+
"Bing reports its own observed search and crawl evidence. It is not a complete view of every search engine.",
|
|
30412
|
+
"The inIndex field is provider-reported crawl statistics, not independent proof that a URL is indexed.",
|
|
30413
|
+
"Query and page movements include only dimensions observed in every weekly top list in both periods. Missing entries are not treated as zero.",
|
|
30414
|
+
"Opportunity thresholds prioritize review. They are heuristics, not search-engine requirements."
|
|
30415
|
+
]
|
|
30416
|
+
};
|
|
30417
|
+
}
|
|
30418
|
+
//#endregion
|
|
30419
|
+
//#region packages/core/src/clients.ts
|
|
30420
|
+
function googleAnalyticsPropertyId(client) {
|
|
30421
|
+
return client?.analytics.google?.propertyId;
|
|
30422
|
+
}
|
|
30423
|
+
function bingWebmasterSiteUrl(client) {
|
|
30424
|
+
return client?.searchEngines?.bing?.siteUrl;
|
|
30425
|
+
}
|
|
30426
|
+
function slug(value) {
|
|
30427
|
+
return value.toLowerCase().replace(/^https?:\/\//, "").replace(/^sc-domain:/, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60);
|
|
30428
|
+
}
|
|
30429
|
+
function uniqueStrings(values = []) {
|
|
30430
|
+
return [...new Set(values.map((value) => value.trim()).filter(Boolean))];
|
|
30431
|
+
}
|
|
30432
|
+
function deriveId(input) {
|
|
30433
|
+
const id = slug(input.id ?? input.name ?? input.siteUrl);
|
|
30434
|
+
if (!id) throw new Error("Client id could not be derived.");
|
|
30435
|
+
return id;
|
|
30436
|
+
}
|
|
30437
|
+
function listClients() {
|
|
30438
|
+
return [...readConfig().clients].sort((a, b) => a.name.localeCompare(b.name));
|
|
30439
|
+
}
|
|
30440
|
+
function getClient(idOrName) {
|
|
30441
|
+
const clients = readConfig().clients;
|
|
30442
|
+
if (!idOrName) return clients.find((client) => client.isDefault);
|
|
30443
|
+
const normalized = idOrName.toLowerCase();
|
|
30444
|
+
return clients.find((client) => client.id.toLowerCase() === normalized || client.name.toLowerCase() === normalized);
|
|
30445
|
+
}
|
|
30446
|
+
function saveClient(input) {
|
|
30447
|
+
const config = readConfig();
|
|
30448
|
+
const id = deriveId(input);
|
|
30449
|
+
const existing = config.clients.find((client) => client.id === id);
|
|
30450
|
+
const now = Date.now();
|
|
30451
|
+
const client = {
|
|
30452
|
+
id,
|
|
30453
|
+
name: input.name ?? existing?.name ?? id,
|
|
30454
|
+
siteUrl: input.siteUrl,
|
|
30455
|
+
startUrl: input.startUrl ?? existing?.startUrl,
|
|
30456
|
+
watchUrls: uniqueStrings(input.watchUrls ?? existing?.watchUrls),
|
|
30457
|
+
brandTerms: uniqueStrings(input.brandTerms ?? existing?.brandTerms ?? deriveBrandTerms({
|
|
30458
|
+
id,
|
|
30459
|
+
name: input.name ?? existing?.name,
|
|
30460
|
+
siteUrl: input.siteUrl
|
|
30461
|
+
})),
|
|
30462
|
+
analytics: input.analytics ?? existing?.analytics ?? {},
|
|
30463
|
+
searchEngines: input.searchEngines ?? existing?.searchEngines,
|
|
30464
|
+
reportDay: input.reportDay ?? existing?.reportDay,
|
|
30465
|
+
technicalWeekday: input.technicalWeekday ?? existing?.technicalWeekday,
|
|
30466
|
+
isDefault: input.isDefault ?? existing?.isDefault,
|
|
30467
|
+
createdAt: existing?.createdAt ?? now,
|
|
30468
|
+
updatedAt: now
|
|
30469
|
+
};
|
|
30470
|
+
const clients = config.clients.filter((item) => item.id !== id);
|
|
30471
|
+
if (client.isDefault) {
|
|
30472
|
+
for (const item of clients) item.isDefault = false;
|
|
30473
|
+
config.defaultSite = client.siteUrl;
|
|
30474
|
+
}
|
|
30475
|
+
config.clients = [...clients, client];
|
|
30476
|
+
writeConfig(config);
|
|
30477
|
+
return client;
|
|
30478
|
+
}
|
|
30479
|
+
function setClientBingSite(idOrName, siteUrl) {
|
|
30480
|
+
const client = getClient(idOrName);
|
|
30481
|
+
if (!client) throw new Error(`Client not found: ${idOrName}`);
|
|
30482
|
+
return saveClient({
|
|
30483
|
+
id: client.id,
|
|
30484
|
+
name: client.name,
|
|
30485
|
+
siteUrl: client.siteUrl,
|
|
30486
|
+
searchEngines: {
|
|
30487
|
+
...client.searchEngines,
|
|
30488
|
+
bing: { siteUrl }
|
|
30489
|
+
}
|
|
30490
|
+
});
|
|
30491
|
+
}
|
|
30492
|
+
function deleteClient(idOrName) {
|
|
30493
|
+
const config = readConfig();
|
|
30494
|
+
const client = getClient(idOrName);
|
|
30495
|
+
if (!client) return false;
|
|
30496
|
+
config.clients = config.clients.filter((item) => item.id !== client.id);
|
|
30497
|
+
if (config.defaultSite === client.siteUrl && !config.clients.some((item) => item.siteUrl === client.siteUrl) && !config.sites.some((site) => site.siteUrl === client.siteUrl)) config.defaultSite = void 0;
|
|
30498
|
+
writeConfig(config);
|
|
30499
|
+
return true;
|
|
30500
|
+
}
|
|
30501
|
+
function setDefaultClient(idOrName) {
|
|
30502
|
+
const config = readConfig();
|
|
30503
|
+
const client = getClient(idOrName);
|
|
30504
|
+
if (!client) throw new Error(`Client not found: ${idOrName}`);
|
|
30505
|
+
config.clients = config.clients.map((item) => ({
|
|
30506
|
+
...item,
|
|
30507
|
+
isDefault: item.id === client.id
|
|
30508
|
+
}));
|
|
30509
|
+
config.defaultSite = client.siteUrl;
|
|
30510
|
+
writeConfig(config);
|
|
30511
|
+
return {
|
|
30512
|
+
...client,
|
|
30513
|
+
isDefault: true
|
|
30514
|
+
};
|
|
30515
|
+
}
|
|
30516
|
+
//#endregion
|
|
30517
|
+
//#region packages/core/src/doctor.ts
|
|
30518
|
+
const REQUIRED_SCOPES = ["https://www.googleapis.com/auth/webmasters.readonly", "https://www.googleapis.com/auth/analytics.readonly"];
|
|
30519
|
+
async function runDoctor(dependencies = {}) {
|
|
30520
|
+
const paths = getSeoCliPaths();
|
|
30521
|
+
const config = readConfig();
|
|
30522
|
+
const authMode = getAuthModeStatus();
|
|
30523
|
+
const serviceAccount = getServiceAccountStatus();
|
|
30524
|
+
const tokens = await readTokens();
|
|
30525
|
+
const oauthClient = readOauthClient();
|
|
30526
|
+
const tokenClientConfigured = tokens ? Boolean(getClientConfig(tokens.client_source)) : void 0;
|
|
30527
|
+
const checks = [];
|
|
30528
|
+
try {
|
|
30529
|
+
const database = (dependencies.checkDatabase ?? checkDatabaseReadiness)();
|
|
30530
|
+
checks.push({
|
|
30531
|
+
id: "local-database",
|
|
30532
|
+
label: "Local database",
|
|
30533
|
+
status: "pass",
|
|
30534
|
+
detail: database.dbPath
|
|
30535
|
+
});
|
|
30536
|
+
} catch (error) {
|
|
30537
|
+
checks.push({
|
|
30538
|
+
id: "local-database",
|
|
30539
|
+
label: "Local database",
|
|
30540
|
+
status: "fail",
|
|
30541
|
+
detail: error instanceof Error ? error.message : "The local database could not be opened.",
|
|
30542
|
+
fix: "Upgrade or reinstall `seo`. If this still fails, report it at https://github.com/iannuttall/seo/issues."
|
|
30543
|
+
});
|
|
30544
|
+
}
|
|
30545
|
+
checks.push({
|
|
30546
|
+
id: "config-dir",
|
|
30547
|
+
label: "Config directory",
|
|
30548
|
+
status: existsSync(paths.configDir) ? "pass" : "warn",
|
|
30549
|
+
detail: paths.configDir,
|
|
30550
|
+
fix: existsSync(paths.configDir) ? void 0 : "Run `seo start` or `seo auth setup-client`."
|
|
30551
|
+
});
|
|
30552
|
+
checks.push({
|
|
30553
|
+
id: "oauth-client",
|
|
30554
|
+
label: "Google credentials",
|
|
30555
|
+
status: serviceAccount.configured ? "pass" : tokenClientConfigured === false ? "fail" : serviceAccount.error ? "fail" : authMode.sharedConfigured || authMode.byoConfigured ? "pass" : "fail",
|
|
30556
|
+
detail: serviceAccount.configured ? `Service account configured for ${serviceAccount.identity}.` : tokenClientConfigured === false ? `Stored Google login uses the ${tokens?.client_source === "shared" ? "shared seo app" : "BYO client"}, but that client is not configured.` : serviceAccount.error ? serviceAccount.error : authMode.sharedConfigured ? "Shared client configured." : authMode.byoConfigured ? `BYO client configured at ${paths.oauthClientFile}.` : "No shared or BYO Google OAuth client configured.",
|
|
30557
|
+
fix: serviceAccount.configured ? void 0 : tokenClientConfigured === false ? tokens?.client_source === "shared" ? "Reinstall `seo`. If the shared client is still missing, report it at https://github.com/iannuttall/seo/issues." : "Restore the same client with `seo auth setup-client`, or run `seo auth logout` before signing in with a different client." : serviceAccount.error ? "Set one valid service account credential source, then run `seo doctor` again." : authMode.sharedConfigured || authMode.byoConfigured ? void 0 : "Run `seo auth setup-client` or set SEO_GOOGLE_CLIENT_ID and SEO_GOOGLE_CLIENT_SECRET."
|
|
30558
|
+
});
|
|
30559
|
+
checks.push({
|
|
30560
|
+
id: "oauth-client-file",
|
|
30561
|
+
label: "BYO OAuth file",
|
|
30562
|
+
status: oauthClient ? "pass" : "warn",
|
|
30563
|
+
detail: oauthClient ? paths.oauthClientFile : "No BYO OAuth client file. This is fine if using a shared client."
|
|
30564
|
+
});
|
|
30565
|
+
checks.push({
|
|
30566
|
+
id: "google-login",
|
|
30567
|
+
label: "Google login",
|
|
30568
|
+
status: serviceAccount.configured || tokens ? "pass" : "fail",
|
|
30569
|
+
detail: serviceAccount.configured ? `Using service account ${serviceAccount.identity}.` : tokens ? `Signed in as ${tokens.account_email}.` : "No Google token found.",
|
|
30570
|
+
fix: serviceAccount.configured || tokens ? void 0 : "Run `seo auth login`, or configure a service account for CI."
|
|
30571
|
+
});
|
|
30572
|
+
const grantedScopes = new Set(tokens?.scope.split(/\s+/).filter(Boolean) ?? []);
|
|
30573
|
+
for (const scope of REQUIRED_SCOPES) checks.push({
|
|
30574
|
+
id: `scope:${scope}`,
|
|
30575
|
+
label: scope.endsWith("webmasters.readonly") ? "Search Console scope" : "Google Analytics scope",
|
|
30576
|
+
status: serviceAccount.configured || tokens && grantedScopes.has(scope) ? "pass" : tokens ? "fail" : "warn",
|
|
30577
|
+
detail: serviceAccount.configured ? `${scope} requested by the service account.` : tokens ? grantedScopes.has(scope) ? scope : `Missing ${scope}.` : "Cannot inspect scopes until login is complete.",
|
|
30578
|
+
fix: !serviceAccount.configured && tokens && !grantedScopes.has(scope) ? "Run `seo auth logout` then `seo auth login`." : void 0
|
|
30579
|
+
});
|
|
30580
|
+
checks.push({
|
|
30581
|
+
id: "default-site",
|
|
30582
|
+
label: "Saved GSC property",
|
|
30583
|
+
status: "pass",
|
|
30584
|
+
detail: config.defaultSite ?? "No saved default. Human CLI commands will prompt; agents can pass --site."
|
|
30585
|
+
});
|
|
30586
|
+
checks.push({
|
|
30587
|
+
id: "default-google-analytics",
|
|
30588
|
+
label: "Saved Google Analytics property",
|
|
30589
|
+
status: "pass",
|
|
30590
|
+
detail: config.analytics.google.defaultPropertyId ?? "No saved default. Human CLI commands will prompt; agents can pass --property."
|
|
30591
|
+
});
|
|
30592
|
+
return {
|
|
30593
|
+
ok: checks.every((check) => check.status !== "fail"),
|
|
30594
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
30595
|
+
checks
|
|
30596
|
+
};
|
|
30597
|
+
}
|
|
30598
|
+
//#endregion
|
|
30599
|
+
//#region packages/core/src/export/report-html.ts
|
|
30600
|
+
function escapeHtml(value) {
|
|
30601
|
+
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll("\"", """).replaceAll("'", "'");
|
|
30602
|
+
}
|
|
30603
|
+
function safeHttpUrl(value) {
|
|
30604
|
+
try {
|
|
30605
|
+
const url = new URL(value);
|
|
30606
|
+
return url.protocol === "http:" || url.protocol === "https:" ? url.toString() : void 0;
|
|
30607
|
+
} catch {
|
|
30608
|
+
return;
|
|
30609
|
+
}
|
|
30610
|
+
}
|
|
30611
|
+
function siteMarkup(site) {
|
|
30612
|
+
const safeUrl = safeHttpUrl(site);
|
|
30613
|
+
const label = escapeHtml(site);
|
|
30614
|
+
return safeUrl ? `<a href="${escapeHtml(safeUrl)}" rel="noreferrer">${label}</a>` : label;
|
|
30615
|
+
}
|
|
30616
|
+
function list(items) {
|
|
30617
|
+
if (items.length === 0) return "<p class=\"empty\">No items to report.</p>";
|
|
30618
|
+
return `<ul>${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul>`;
|
|
30619
|
+
}
|
|
30620
|
+
function reportSections(report) {
|
|
30621
|
+
return report.sections.map((section) => `<section class="panel">
|
|
30622
|
+
<h2>${escapeHtml(section.title)}</h2>
|
|
30623
|
+
${list(section.bullets)}
|
|
30624
|
+
</section>`).join("");
|
|
30625
|
+
}
|
|
30626
|
+
function additionalSections(sections) {
|
|
30627
|
+
return sections.map((section) => `<section class="panel">
|
|
30628
|
+
<h2>${escapeHtml(section.title)}</h2>
|
|
30629
|
+
${section.summary ? `<p>${escapeHtml(section.summary)}</p>` : ""}
|
|
30630
|
+
${list(section.items)}
|
|
30631
|
+
</section>`).join("");
|
|
30632
|
+
}
|
|
30633
|
+
function priorities(report) {
|
|
30634
|
+
if (report.priorities.length === 0) return "<p class=\"empty\">No evidence-backed priorities were found.</p>";
|
|
30635
|
+
return `<ol class="priorities">${report.priorities.map((priority) => `<li>
|
|
30636
|
+
<div class="priority-heading">
|
|
30637
|
+
<strong>${escapeHtml(priority.title)}</strong>
|
|
30638
|
+
<span class="confidence">${escapeHtml(priority.confidence)} confidence</span>
|
|
30639
|
+
</div>
|
|
30640
|
+
<p>${escapeHtml(priority.action)}</p>
|
|
30641
|
+
</li>`).join("")}</ol>`;
|
|
30642
|
+
}
|
|
30643
|
+
function analystEvidence(report) {
|
|
30644
|
+
const skipped = report.diagnosis.skippedSections ?? [];
|
|
30645
|
+
const attempts = report.changeMeasurementAttempts;
|
|
30646
|
+
return `<section class="panel analyst-only">
|
|
30647
|
+
<h2>Evidence coverage</h2>
|
|
30648
|
+
<dl class="facts">
|
|
30649
|
+
<div><dt>Data status</dt><dd>${escapeHtml(report.dataStatus)}</dd></div>
|
|
30650
|
+
<div><dt>Generated</dt><dd>${escapeHtml(report.generatedAt)}</dd></div>
|
|
30651
|
+
<div><dt>Saved changes checked</dt><dd>${attempts.length}</dd></div>
|
|
30652
|
+
<div><dt>Unavailable sections</dt><dd>${skipped.length}</dd></div>
|
|
30653
|
+
</dl>
|
|
30654
|
+
${skipped.length ? `<h3>Unavailable sections</h3>${list(skipped.map((section) => "reason" in section && typeof section.reason === "string" ? `${section.section}: ${section.reason}` : section.section))}` : ""}
|
|
30655
|
+
</section>`;
|
|
30656
|
+
}
|
|
30657
|
+
const styles = `
|
|
30658
|
+
:root { color-scheme: light; font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; color: #17211b; background: #f2f4ef; }
|
|
30659
|
+
* { box-sizing: border-box; }
|
|
30660
|
+
body { margin: 0; line-height: 1.55; }
|
|
30661
|
+
a { color: inherit; text-decoration-color: #ff5b35; text-underline-offset: 0.18em; }
|
|
30662
|
+
main { width: min(980px, calc(100% - 32px)); margin: 32px auto 64px; }
|
|
30663
|
+
header { padding: clamp(28px, 6vw, 64px); color: #f9fbf7; background: #17211b; border-radius: 20px; }
|
|
30664
|
+
.eyebrow { margin: 0 0 12px; color: #ff7958; font-size: 0.78rem; font-weight: 800; letter-spacing: 0.12em; text-transform: uppercase; }
|
|
30665
|
+
h1 { max-width: 760px; margin: 0; font-size: clamp(2rem, 5vw, 4rem); line-height: 1.02; letter-spacing: -0.04em; }
|
|
30666
|
+
header .headline { max-width: 760px; margin: 24px 0 0; font-size: clamp(1.05rem, 2vw, 1.3rem); color: #d9e1d9; }
|
|
30667
|
+
.meta { display: flex; flex-wrap: wrap; gap: 10px 22px; margin-top: 28px; color: #b9c5ba; font-size: 0.9rem; }
|
|
30668
|
+
.status { display: inline-flex; width: fit-content; margin-top: 22px; padding: 6px 10px; border: 1px solid #4f6153; border-radius: 999px; font-size: 0.8rem; font-weight: 700; text-transform: capitalize; }
|
|
30669
|
+
.panel { margin-top: 20px; padding: clamp(22px, 4vw, 38px); background: #fff; border: 1px solid #dfe4dc; border-radius: 16px; box-shadow: 0 8px 30px rgba(23, 33, 27, 0.05); }
|
|
30670
|
+
h2 { margin: 0 0 16px; font-size: clamp(1.25rem, 3vw, 1.65rem); letter-spacing: -0.02em; }
|
|
30671
|
+
h3 { margin: 26px 0 10px; font-size: 1rem; }
|
|
30672
|
+
p { margin: 0 0 14px; }
|
|
30673
|
+
ul, ol { margin: 0; padding-left: 1.25rem; }
|
|
30674
|
+
li + li { margin-top: 10px; }
|
|
30675
|
+
.priorities { list-style: none; padding: 0; counter-reset: priority; }
|
|
30676
|
+
.priorities > li { counter-increment: priority; padding: 18px 0 18px 52px; position: relative; border-top: 1px solid #e6eae4; }
|
|
30677
|
+
.priorities > li:first-child { border-top: 0; padding-top: 4px; }
|
|
30678
|
+
.priorities > li::before { content: counter(priority); position: absolute; left: 0; top: 16px; display: grid; width: 34px; height: 34px; place-items: center; border-radius: 50%; color: #17211b; background: #ff7958; font-weight: 800; }
|
|
30679
|
+
.priorities > li:first-child::before { top: 2px; }
|
|
30680
|
+
.priority-heading { display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 8px 20px; }
|
|
30681
|
+
.confidence { color: #647067; font-size: 0.82rem; text-transform: capitalize; }
|
|
30682
|
+
.priorities p { margin: 8px 0 0; color: #3e4941; }
|
|
30683
|
+
.facts { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 12px; margin: 0; }
|
|
30684
|
+
.facts div { padding: 14px; background: #f5f7f3; border-radius: 10px; }
|
|
30685
|
+
dt { color: #647067; font-size: 0.78rem; font-weight: 700; text-transform: uppercase; }
|
|
30686
|
+
dd { margin: 4px 0 0; font-weight: 700; }
|
|
30687
|
+
details summary { cursor: pointer; font-weight: 750; }
|
|
30688
|
+
details ul { margin-top: 14px; }
|
|
30689
|
+
.empty { color: #647067; font-style: italic; }
|
|
30690
|
+
footer { margin-top: 26px; color: #647067; font-size: 0.8rem; text-align: center; }
|
|
30691
|
+
@media print { :root { background: #fff; } main { width: 100%; margin: 0; } header, .panel { break-inside: avoid; box-shadow: none; } header { border-radius: 0; } }
|
|
30692
|
+
`;
|
|
30693
|
+
function renderReportHtml(input) {
|
|
30694
|
+
const report = input.report;
|
|
30695
|
+
const view = input.view ?? "client";
|
|
30696
|
+
const title = input.title ?? "SEO report";
|
|
30697
|
+
const evidenceNotesOpen = report.dataStatus !== "complete" ? " open" : "";
|
|
30698
|
+
const analyst = view === "analyst" ? analystEvidence(report) : "";
|
|
30699
|
+
return `<!doctype html>
|
|
30700
|
+
<html lang="en">
|
|
30701
|
+
<head>
|
|
30702
|
+
<meta charset="utf-8">
|
|
30703
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
30704
|
+
<meta name="robots" content="noindex,nofollow">
|
|
30705
|
+
<title>${escapeHtml(title)}: ${escapeHtml(report.site)}</title>
|
|
30706
|
+
<style>${styles}</style>
|
|
30707
|
+
</head>
|
|
30708
|
+
<body>
|
|
30709
|
+
<main>
|
|
30710
|
+
<header>
|
|
30711
|
+
<p class="eyebrow">${escapeHtml(title)}</p>
|
|
30712
|
+
<h1>${siteMarkup(report.site)}</h1>
|
|
30713
|
+
<p class="headline">${escapeHtml(report.headline)}</p>
|
|
30714
|
+
<div class="meta">
|
|
30715
|
+
<span>${escapeHtml(report.period.startDate)} to ${escapeHtml(report.period.endDate)}</span>
|
|
30716
|
+
<span>Generated ${escapeHtml(report.generatedAt.slice(0, 10))}</span>
|
|
30717
|
+
<span>${escapeHtml(view)} view</span>
|
|
30718
|
+
</div>
|
|
30719
|
+
<span class="status">${escapeHtml(report.dataStatus)} data</span>
|
|
30720
|
+
</header>
|
|
30721
|
+
|
|
30722
|
+
<section class="panel">
|
|
30723
|
+
<h2>Priorities</h2>
|
|
30724
|
+
${priorities(report)}
|
|
30725
|
+
</section>
|
|
30726
|
+
|
|
30727
|
+
${reportSections(report)}
|
|
30728
|
+
${additionalSections(input.additionalSections ?? [])}
|
|
30729
|
+
${analyst}
|
|
30730
|
+
|
|
30731
|
+
<section class="panel">
|
|
30732
|
+
<details${evidenceNotesOpen}>
|
|
30733
|
+
<summary>Evidence notes and limitations</summary>
|
|
30734
|
+
${list(report.caveats)}
|
|
30735
|
+
</details>
|
|
30736
|
+
</section>
|
|
30737
|
+
|
|
30738
|
+
<footer>Generated locally from observed report evidence.</footer>
|
|
30739
|
+
</main>
|
|
30740
|
+
</body>
|
|
30741
|
+
</html>`;
|
|
30742
|
+
}
|
|
30743
|
+
//#endregion
|
|
30744
|
+
//#region packages/core/src/export/update-postmortem.ts
|
|
30745
|
+
const HEADERS = {
|
|
30746
|
+
summary: [
|
|
30747
|
+
"workflow",
|
|
30748
|
+
"site",
|
|
30749
|
+
"generated_at",
|
|
30750
|
+
"summary",
|
|
30751
|
+
"attribution",
|
|
30752
|
+
"confidence",
|
|
30753
|
+
"classification",
|
|
30754
|
+
"updates_matched",
|
|
30755
|
+
"known_confounders",
|
|
30756
|
+
"page_segment_status",
|
|
30757
|
+
"page_segment_eligible_rows",
|
|
30758
|
+
"page_segment_returned_rows",
|
|
30759
|
+
"page_segment_limited_rows",
|
|
30760
|
+
"page_segment_result_limit",
|
|
30761
|
+
"page_segment_source_row_limit",
|
|
30762
|
+
"page_segment_source_possibly_truncated",
|
|
29700
30763
|
"page_segment_warnings",
|
|
29701
30764
|
"query_segment_status",
|
|
29702
30765
|
"query_segment_eligible_rows",
|
|
@@ -29988,60 +31051,779 @@ function updatePostmortemCsvFiles(report) {
|
|
|
29988
31051
|
headers: [...HEADERS.segment],
|
|
29989
31052
|
rows: postmortemSegmentRows(report, dimension)
|
|
29990
31053
|
})),
|
|
29991
|
-
...[
|
|
29992
|
-
"page",
|
|
29993
|
-
"query",
|
|
29994
|
-
"device",
|
|
29995
|
-
"country"
|
|
29996
|
-
].map((dimension) => ({
|
|
29997
|
-
filename: `postmortem-segment-${dimension}-unmatched.csv`,
|
|
29998
|
-
headers: [...HEADERS.unmatchedSegment],
|
|
29999
|
-
rows: postmortemUnmatchedSegmentRows(report, dimension)
|
|
30000
|
-
}))
|
|
30001
|
-
];
|
|
31054
|
+
...[
|
|
31055
|
+
"page",
|
|
31056
|
+
"query",
|
|
31057
|
+
"device",
|
|
31058
|
+
"country"
|
|
31059
|
+
].map((dimension) => ({
|
|
31060
|
+
filename: `postmortem-segment-${dimension}-unmatched.csv`,
|
|
31061
|
+
headers: [...HEADERS.unmatchedSegment],
|
|
31062
|
+
rows: postmortemUnmatchedSegmentRows(report, dimension)
|
|
31063
|
+
}))
|
|
31064
|
+
];
|
|
31065
|
+
}
|
|
31066
|
+
//#endregion
|
|
31067
|
+
//#region packages/core/src/ga4/matching.ts
|
|
31068
|
+
function hostnameFromUrl(value) {
|
|
31069
|
+
try {
|
|
31070
|
+
const url = new URL(value);
|
|
31071
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
31072
|
+
return url.hostname.toLowerCase().replace(/\.$/, "") || void 0;
|
|
31073
|
+
} catch {
|
|
31074
|
+
return;
|
|
31075
|
+
}
|
|
31076
|
+
}
|
|
31077
|
+
function domainFromProperty(value) {
|
|
31078
|
+
if (!value.startsWith("sc-domain:")) return void 0;
|
|
31079
|
+
const domain = value.slice(10).trim().toLowerCase();
|
|
31080
|
+
if (!domain || /[/:]/.test(domain)) return void 0;
|
|
31081
|
+
return domain.replace(/\.$/, "") || void 0;
|
|
31082
|
+
}
|
|
31083
|
+
function matchForSite(site, streamHostname) {
|
|
31084
|
+
const domain = domainFromProperty(site);
|
|
31085
|
+
if (domain) return streamHostname === domain || streamHostname.endsWith(`.${domain}`) ? "domain" : void 0;
|
|
31086
|
+
return hostnameFromUrl(site) === streamHostname ? "hostname" : void 0;
|
|
31087
|
+
}
|
|
31088
|
+
function matchGa4WebStreams(site, candidates) {
|
|
31089
|
+
return candidates.flatMap((candidate) => {
|
|
31090
|
+
const defaultUri = candidate.stream.webStreamData?.defaultUri;
|
|
31091
|
+
if (!defaultUri) return [];
|
|
31092
|
+
const hostname = hostnameFromUrl(defaultUri);
|
|
31093
|
+
if (!hostname) return [];
|
|
31094
|
+
const match = matchForSite(site, hostname);
|
|
31095
|
+
if (!match) return [];
|
|
31096
|
+
return [{
|
|
31097
|
+
...candidate,
|
|
31098
|
+
defaultUri,
|
|
31099
|
+
hostname,
|
|
31100
|
+
match
|
|
31101
|
+
}];
|
|
31102
|
+
});
|
|
31103
|
+
}
|
|
31104
|
+
function ga4MatchReason(match, site) {
|
|
31105
|
+
const stream = match.stream.displayName ?? match.defaultUri;
|
|
31106
|
+
const location = match.match === "domain" ? "is inside" : "matches";
|
|
31107
|
+
return `Matched Google Analytics web stream ${stream} (${match.defaultUri}): its hostname ${location} ${site}.`;
|
|
31108
|
+
}
|
|
31109
|
+
//#endregion
|
|
31110
|
+
//#region packages/core/src/indexnow/types.ts
|
|
31111
|
+
const INDEXNOW_ENDPOINT = "https://api.indexnow.org/indexnow";
|
|
31112
|
+
const INDEXNOW_MAX_URLS = 1e3;
|
|
31113
|
+
const INDEXNOW_KEY_ENV = "SEO_INDEXNOW_KEY";
|
|
31114
|
+
const INDEXNOW_KEYS_SECRET = "indexnow-keys";
|
|
31115
|
+
//#endregion
|
|
31116
|
+
//#region packages/core/src/indexnow/keys.ts
|
|
31117
|
+
const KEY_PATTERN = /^[A-Za-z0-9-]{8,128}$/;
|
|
31118
|
+
function isPrivateHost(hostname) {
|
|
31119
|
+
const host = hostname.toLowerCase().replace(/^\[|\]$/g, "");
|
|
31120
|
+
if (host === "localhost" || host.endsWith(".localhost") || host.includes(":")) return true;
|
|
31121
|
+
const octets = host.split(".").map(Number);
|
|
31122
|
+
if (octets.length !== 4 || octets.some((value) => !Number.isInteger(value))) return false;
|
|
31123
|
+
const [first, second] = octets;
|
|
31124
|
+
return first === 0 || first === 10 || first === 127 || first === 100 && second !== void 0 && second >= 64 && second <= 127 || first === 169 && second === 254 || first === 172 && second !== void 0 && second >= 16 && second <= 31 || first === 192 && second === 0 || first === 192 && second === 168 || first === 198 && (second === 18 || second === 19) || first !== void 0 && first >= 224;
|
|
31125
|
+
}
|
|
31126
|
+
function parseIndexNowSite(value) {
|
|
31127
|
+
let url;
|
|
31128
|
+
try {
|
|
31129
|
+
url = new URL(value);
|
|
31130
|
+
} catch {
|
|
31131
|
+
throw new SeoError("INVALID_INPUT", "IndexNow site must be a valid URL.");
|
|
31132
|
+
}
|
|
31133
|
+
if (!["http:", "https:"].includes(url.protocol) || !url.hostname) throw new SeoError("INVALID_INPUT", "IndexNow site must use HTTP or HTTPS.");
|
|
31134
|
+
if (url.username || url.password) throw new SeoError("INVALID_INPUT", "IndexNow site URLs cannot contain credentials.");
|
|
31135
|
+
if (isPrivateHost(url.hostname)) throw new SeoError("INVALID_INPUT", "IndexNow requires a public site host.");
|
|
31136
|
+
return url;
|
|
31137
|
+
}
|
|
31138
|
+
function validateIndexNowKey(value) {
|
|
31139
|
+
const key = value.trim();
|
|
31140
|
+
if (!KEY_PATTERN.test(key)) throw new SeoError("INVALID_INPUT", "IndexNow keys must contain 8 to 128 letters, numbers, or hyphens.");
|
|
31141
|
+
return key;
|
|
31142
|
+
}
|
|
31143
|
+
function createIndexNowKey() {
|
|
31144
|
+
return randomBytes(16).toString("hex");
|
|
31145
|
+
}
|
|
31146
|
+
function createIndexNowKeyRecord(input) {
|
|
31147
|
+
const site = parseIndexNowSite(input.site);
|
|
31148
|
+
const key = validateIndexNowKey(input.key ?? createIndexNowKey());
|
|
31149
|
+
const keyLocation = input.keyLocation ? parseIndexNowSite(input.keyLocation) : new URL(`/${key}.txt`, site.origin);
|
|
31150
|
+
if (keyLocation.hostname.toLowerCase() !== site.hostname.toLowerCase()) throw new SeoError("INVALID_INPUT", "IndexNow key location must use the same host as the site.");
|
|
31151
|
+
return {
|
|
31152
|
+
host: site.hostname.toLowerCase(),
|
|
31153
|
+
key,
|
|
31154
|
+
keyLocation: keyLocation.toString(),
|
|
31155
|
+
createdAt: (input.now ?? /* @__PURE__ */ new Date()).toISOString()
|
|
31156
|
+
};
|
|
31157
|
+
}
|
|
31158
|
+
function validateIndexNowKeyRecord(record) {
|
|
31159
|
+
const createdAt = new Date(record.createdAt);
|
|
31160
|
+
if (Number.isNaN(createdAt.getTime())) throw new SeoError("INVALID_INPUT", "IndexNow key record has an invalid creation date.");
|
|
31161
|
+
const normalized = createIndexNowKeyRecord({
|
|
31162
|
+
site: record.keyLocation,
|
|
31163
|
+
key: record.key,
|
|
31164
|
+
keyLocation: record.keyLocation,
|
|
31165
|
+
now: createdAt
|
|
31166
|
+
});
|
|
31167
|
+
if (normalized.host !== record.host.toLowerCase()) throw new SeoError("INVALID_INPUT", "IndexNow key record host does not match its key location.");
|
|
31168
|
+
return normalized;
|
|
31169
|
+
}
|
|
31170
|
+
function emptyKeys() {
|
|
31171
|
+
return {
|
|
31172
|
+
version: 1,
|
|
31173
|
+
sites: {}
|
|
31174
|
+
};
|
|
31175
|
+
}
|
|
31176
|
+
function parseSavedKeys(value) {
|
|
31177
|
+
try {
|
|
31178
|
+
const parsed = JSON.parse(value);
|
|
31179
|
+
if (parsed.version !== 1 || !parsed.sites || typeof parsed.sites !== "object" || Array.isArray(parsed.sites)) throw new Error("invalid shape");
|
|
31180
|
+
const sites = {};
|
|
31181
|
+
for (const [host, value] of Object.entries(parsed.sites)) {
|
|
31182
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("invalid key record");
|
|
31183
|
+
const record = validateIndexNowKeyRecord(value);
|
|
31184
|
+
if (record.host !== host) throw new Error("invalid host mapping");
|
|
31185
|
+
sites[host] = record;
|
|
31186
|
+
}
|
|
31187
|
+
return {
|
|
31188
|
+
version: 1,
|
|
31189
|
+
sites
|
|
31190
|
+
};
|
|
31191
|
+
} catch {
|
|
31192
|
+
throw new SeoError("AUTH_REQUIRED", "Saved IndexNow keys are invalid. Run `seo indexnow setup` again.");
|
|
31193
|
+
}
|
|
31194
|
+
}
|
|
31195
|
+
async function readSavedKeys() {
|
|
31196
|
+
const saved = await readProviderSecret({ name: INDEXNOW_KEYS_SECRET });
|
|
31197
|
+
if (!saved) return { keys: emptyKeys() };
|
|
31198
|
+
return {
|
|
31199
|
+
keys: parseSavedKeys(saved.value),
|
|
31200
|
+
source: saved.source
|
|
31201
|
+
};
|
|
31202
|
+
}
|
|
31203
|
+
async function saveIndexNowKey(record) {
|
|
31204
|
+
const normalized = validateIndexNowKeyRecord(record);
|
|
31205
|
+
const { keys } = await readSavedKeys();
|
|
31206
|
+
keys.sites[normalized.host] = normalized;
|
|
31207
|
+
return writeProviderSecret(INDEXNOW_KEYS_SECRET, JSON.stringify(keys));
|
|
31208
|
+
}
|
|
31209
|
+
async function resolveIndexNowKey(input) {
|
|
31210
|
+
const site = parseIndexNowSite(input.site);
|
|
31211
|
+
const environmentKey = (input.env ?? process.env)[INDEXNOW_KEY_ENV]?.trim();
|
|
31212
|
+
if (environmentKey) return {
|
|
31213
|
+
record: createIndexNowKeyRecord({
|
|
31214
|
+
site: site.origin,
|
|
31215
|
+
key: environmentKey
|
|
31216
|
+
}),
|
|
31217
|
+
source: "environment"
|
|
31218
|
+
};
|
|
31219
|
+
const { keys, source } = await readSavedKeys();
|
|
31220
|
+
const record = keys.sites[site.hostname.toLowerCase()];
|
|
31221
|
+
if (!record || !source) throw new SeoError("AUTH_REQUIRED", `IndexNow is not set up for ${site.hostname}. Run \`seo indexnow setup --site ${site.origin} --output <public-directory>\`, or set SEO_INDEXNOW_KEY for this process.`);
|
|
31222
|
+
return {
|
|
31223
|
+
record: validateIndexNowKeyRecord(record),
|
|
31224
|
+
source
|
|
31225
|
+
};
|
|
31226
|
+
}
|
|
31227
|
+
async function listIndexNowKeys() {
|
|
31228
|
+
const { keys } = await readSavedKeys();
|
|
31229
|
+
return Object.values(keys.sites).map(({ key: _key, ...record }) => record).sort((a, b) => a.host < b.host ? -1 : a.host > b.host ? 1 : 0);
|
|
31230
|
+
}
|
|
31231
|
+
async function removeIndexNowKey(siteValue) {
|
|
31232
|
+
const site = parseIndexNowSite(siteValue);
|
|
31233
|
+
const { keys } = await readSavedKeys();
|
|
31234
|
+
const host = site.hostname.toLowerCase();
|
|
31235
|
+
if (!keys.sites[host]) return false;
|
|
31236
|
+
delete keys.sites[host];
|
|
31237
|
+
if (Object.keys(keys.sites).length === 0) await deleteProviderSecret(INDEXNOW_KEYS_SECRET);
|
|
31238
|
+
else await writeProviderSecret(INDEXNOW_KEYS_SECRET, JSON.stringify(keys));
|
|
31239
|
+
return true;
|
|
31240
|
+
}
|
|
31241
|
+
//#endregion
|
|
31242
|
+
//#region packages/core/src/indexnow/client.ts
|
|
31243
|
+
function uniqueUrls(values, host) {
|
|
31244
|
+
if (values.length === 0) throw new SeoError("INVALID_INPUT", "Pass at least one URL to IndexNow.");
|
|
31245
|
+
if (values.length > 1e3) throw new SeoError("INVALID_INPUT", `IndexNow submissions are limited to ${INDEXNOW_MAX_URLS} URLs per run.`);
|
|
31246
|
+
const urls = /* @__PURE__ */ new Set();
|
|
31247
|
+
for (const value of values) {
|
|
31248
|
+
const url = parseIndexNowSite(value);
|
|
31249
|
+
if (url.toString().length > 2e3) throw new SeoError("INVALID_INPUT", "IndexNow URLs cannot exceed 2,000 characters.");
|
|
31250
|
+
if (url.hostname.toLowerCase() !== host) throw new SeoError("INVALID_INPUT", `Every IndexNow URL must belong to ${host}.`);
|
|
31251
|
+
url.hash = "";
|
|
31252
|
+
urls.add(url.toString());
|
|
31253
|
+
}
|
|
31254
|
+
return [...urls].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
31255
|
+
}
|
|
31256
|
+
async function verifyIndexNowKey(input) {
|
|
31257
|
+
const record = validateIndexNowKeyRecord(input.record);
|
|
31258
|
+
const keyLocation = parseIndexNowSite(record.keyLocation);
|
|
31259
|
+
let response;
|
|
31260
|
+
try {
|
|
31261
|
+
response = await (input.fetchImpl ?? publicHttpFetch)(keyLocation.toString(), {
|
|
31262
|
+
headers: { accept: "text/plain" },
|
|
31263
|
+
redirect: "manual",
|
|
31264
|
+
signal: AbortSignal.timeout(input.timeoutMs ?? 15e3)
|
|
31265
|
+
});
|
|
31266
|
+
} catch (error) {
|
|
31267
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `IndexNow key verification failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
31268
|
+
}
|
|
31269
|
+
const body = await readBoundedResponseText(response, 512, "IndexNow key file");
|
|
31270
|
+
return {
|
|
31271
|
+
verified: response.ok && body.trim() === record.key,
|
|
31272
|
+
status: response.status,
|
|
31273
|
+
keyLocation: keyLocation.toString()
|
|
31274
|
+
};
|
|
31275
|
+
}
|
|
31276
|
+
function indexNowFailure(status) {
|
|
31277
|
+
if (status === 403) return new SeoError("ACCESS_DENIED", "IndexNow could not validate the key. Confirm the public key file contains the saved key.");
|
|
31278
|
+
if (status === 429) return new SeoError("RATE_LIMITED", "IndexNow rate limited this submission. Wait before trying again.");
|
|
31279
|
+
if (status === 400 || status === 422) return new SeoError("INVALID_INPUT", `IndexNow rejected the submission with HTTP ${status}. Check the URL host and key configuration.`);
|
|
31280
|
+
return new SeoError("PROVIDER_UNAVAILABLE", `IndexNow submission failed with HTTP ${status}.`);
|
|
31281
|
+
}
|
|
31282
|
+
async function submitIndexNow(input) {
|
|
31283
|
+
const record = validateIndexNowKeyRecord(input.record);
|
|
31284
|
+
const urls = uniqueUrls(input.urls, record.host);
|
|
31285
|
+
const keyLocation = parseIndexNowSite(record.keyLocation);
|
|
31286
|
+
const base = {
|
|
31287
|
+
schemaVersion: 1,
|
|
31288
|
+
generatedAt: (input.now ?? /* @__PURE__ */ new Date()).toISOString(),
|
|
31289
|
+
endpoint: INDEXNOW_ENDPOINT,
|
|
31290
|
+
host: record.host,
|
|
31291
|
+
keyLocation: keyLocation.toString(),
|
|
31292
|
+
submittedUrls: urls.length,
|
|
31293
|
+
caveats: ["IndexNow confirms receipt, not crawling, indexing, ranking, or traffic.", "Submit URLs only after they are added, updated, or deleted."]
|
|
31294
|
+
};
|
|
31295
|
+
if (input.dryRun) return {
|
|
31296
|
+
...base,
|
|
31297
|
+
dryRun: true,
|
|
31298
|
+
status: "validated"
|
|
31299
|
+
};
|
|
31300
|
+
const verification = await verifyIndexNowKey({
|
|
31301
|
+
record,
|
|
31302
|
+
fetchImpl: input.fetchImpl
|
|
31303
|
+
});
|
|
31304
|
+
if (!verification.verified) throw new SeoError("ACCESS_DENIED", `IndexNow key verification failed at ${verification.keyLocation} with HTTP ${verification.status}. Deploy the generated key file before submitting URLs.`);
|
|
31305
|
+
let response;
|
|
31306
|
+
try {
|
|
31307
|
+
response = await (input.fetchImpl ?? publicHttpFetch)(INDEXNOW_ENDPOINT, {
|
|
31308
|
+
method: "POST",
|
|
31309
|
+
headers: {
|
|
31310
|
+
accept: "application/json",
|
|
31311
|
+
"content-type": "application/json; charset=utf-8"
|
|
31312
|
+
},
|
|
31313
|
+
body: JSON.stringify({
|
|
31314
|
+
host: record.host,
|
|
31315
|
+
key: record.key,
|
|
31316
|
+
keyLocation: keyLocation.toString(),
|
|
31317
|
+
urlList: urls
|
|
31318
|
+
}),
|
|
31319
|
+
signal: AbortSignal.timeout(3e4)
|
|
31320
|
+
});
|
|
31321
|
+
} catch (error) {
|
|
31322
|
+
throw new SeoError("PROVIDER_UNAVAILABLE", `IndexNow did not respond: ${error instanceof Error ? error.message : String(error)}`);
|
|
31323
|
+
}
|
|
31324
|
+
await readBoundedResponseText(response, 64e3, "IndexNow response");
|
|
31325
|
+
if (response.status !== 200 && response.status !== 202) throw indexNowFailure(response.status);
|
|
31326
|
+
return {
|
|
31327
|
+
...base,
|
|
31328
|
+
dryRun: false,
|
|
31329
|
+
status: response.status === 200 ? "submitted" : "pending-validation",
|
|
31330
|
+
responseStatus: response.status
|
|
31331
|
+
};
|
|
31332
|
+
}
|
|
31333
|
+
//#endregion
|
|
31334
|
+
//#region packages/core/src/links/normalize.ts
|
|
31335
|
+
const FIELD_ALIASES = {
|
|
31336
|
+
sourceUrl: [
|
|
31337
|
+
"sourceurl",
|
|
31338
|
+
"source",
|
|
31339
|
+
"referringurl",
|
|
31340
|
+
"referringpage",
|
|
31341
|
+
"fromurl"
|
|
31342
|
+
],
|
|
31343
|
+
targetUrl: [
|
|
31344
|
+
"targeturl",
|
|
31345
|
+
"target",
|
|
31346
|
+
"destinationurl",
|
|
31347
|
+
"destination",
|
|
31348
|
+
"tourl"
|
|
31349
|
+
],
|
|
31350
|
+
anchorText: ["anchortext", "anchor"],
|
|
31351
|
+
firstSeenAt: ["firstseenat", "firstseen"],
|
|
31352
|
+
lastSeenAt: ["lastseenat", "lastseen"],
|
|
31353
|
+
nofollow: ["nofollow", "isnofollow"]
|
|
31354
|
+
};
|
|
31355
|
+
function key(value) {
|
|
31356
|
+
return value.toLowerCase().replaceAll(/[^a-z0-9]/g, "");
|
|
31357
|
+
}
|
|
31358
|
+
function valueFor(row, field) {
|
|
31359
|
+
const aliases = new Set(FIELD_ALIASES[field]);
|
|
31360
|
+
for (const [name, value] of Object.entries(row)) if (aliases.has(key(name))) return value;
|
|
31361
|
+
}
|
|
31362
|
+
function httpUrl(value) {
|
|
31363
|
+
if (typeof value !== "string" || !value.trim() || value.trim().length > 2e3) return;
|
|
31364
|
+
try {
|
|
31365
|
+
const url = new URL(value.trim());
|
|
31366
|
+
return url.protocol === "http:" || url.protocol === "https:" ? url : void 0;
|
|
31367
|
+
} catch {
|
|
31368
|
+
return;
|
|
31369
|
+
}
|
|
31370
|
+
}
|
|
31371
|
+
function optionalString(value) {
|
|
31372
|
+
return typeof value === "string" && value.trim() ? value.trim().slice(0, 1e3) : void 0;
|
|
31373
|
+
}
|
|
31374
|
+
function optionalBoolean(value) {
|
|
31375
|
+
if (typeof value === "boolean") return value;
|
|
31376
|
+
if (typeof value !== "string") return void 0;
|
|
31377
|
+
if (/^(true|yes|1)$/i.test(value.trim())) return true;
|
|
31378
|
+
if (/^(false|no|0)$/i.test(value.trim())) return false;
|
|
31379
|
+
}
|
|
31380
|
+
function normalizeLinkEvidenceRow(raw) {
|
|
31381
|
+
const source = httpUrl(valueFor(raw, "sourceUrl"));
|
|
31382
|
+
const target = httpUrl(valueFor(raw, "targetUrl"));
|
|
31383
|
+
if (!source || !target) return void 0;
|
|
31384
|
+
return {
|
|
31385
|
+
sourceUrl: source.toString(),
|
|
31386
|
+
sourceDomain: source.hostname.toLowerCase(),
|
|
31387
|
+
targetUrl: target.toString(),
|
|
31388
|
+
anchorText: optionalString(valueFor(raw, "anchorText")),
|
|
31389
|
+
firstSeenAt: optionalString(valueFor(raw, "firstSeenAt")),
|
|
31390
|
+
lastSeenAt: optionalString(valueFor(raw, "lastSeenAt")),
|
|
31391
|
+
nofollow: optionalBoolean(valueFor(raw, "nofollow"))
|
|
31392
|
+
};
|
|
31393
|
+
}
|
|
31394
|
+
function linkEvidenceKey(row) {
|
|
31395
|
+
return `${row.sourceUrl}\u0000${row.targetUrl}\u0000${row.anchorText ?? ""}`;
|
|
31396
|
+
}
|
|
31397
|
+
function compareLinkEvidence(a, b) {
|
|
31398
|
+
return a.targetUrl.localeCompare(b.targetUrl, "en") || a.sourceUrl.localeCompare(b.sourceUrl, "en") || (a.anchorText ?? "").localeCompare(b.anchorText ?? "", "en");
|
|
31399
|
+
}
|
|
31400
|
+
//#endregion
|
|
31401
|
+
//#region packages/core/src/links/bing.ts
|
|
31402
|
+
const MAX_COUNT_PAGES = 5;
|
|
31403
|
+
const DEFAULT_TARGET_LIMIT = 20;
|
|
31404
|
+
const MAX_TARGET_LIMIT = 50;
|
|
31405
|
+
const DEFAULT_DETAIL_PAGES = 1;
|
|
31406
|
+
const MAX_DETAIL_PAGES = 3;
|
|
31407
|
+
const DEFAULT_ROW_LIMIT$2 = 500;
|
|
31408
|
+
const MAX_ROW_LIMIT$2 = 1e3;
|
|
31409
|
+
const MAX_CONCURRENCY = 3;
|
|
31410
|
+
function integer(value, fallback, max, name) {
|
|
31411
|
+
const resolved = value ?? fallback;
|
|
31412
|
+
if (!Number.isInteger(resolved) || resolved < 1 || resolved > max) throw new SeoError("INVALID_INPUT", `${name} must be between 1 and ${max}.`);
|
|
31413
|
+
return resolved;
|
|
31414
|
+
}
|
|
31415
|
+
async function mapConcurrent(values, concurrency, work) {
|
|
31416
|
+
const output = new Array(values.length);
|
|
31417
|
+
let next = 0;
|
|
31418
|
+
async function worker() {
|
|
31419
|
+
while (next < values.length) {
|
|
31420
|
+
const index = next;
|
|
31421
|
+
next += 1;
|
|
31422
|
+
const value = values[index];
|
|
31423
|
+
if (value !== void 0) output[index] = await work(value);
|
|
31424
|
+
}
|
|
31425
|
+
}
|
|
31426
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, () => worker()));
|
|
31427
|
+
return output;
|
|
31428
|
+
}
|
|
31429
|
+
async function targetDetails(input) {
|
|
31430
|
+
const rows = [];
|
|
31431
|
+
let requestedPages = 0;
|
|
31432
|
+
let invalidRows = 0;
|
|
31433
|
+
let returnedRows = 0;
|
|
31434
|
+
let partial = false;
|
|
31435
|
+
let totalPages = 0;
|
|
31436
|
+
for (let page = 0; page < input.pages; page += 1) {
|
|
31437
|
+
requestedPages += 1;
|
|
31438
|
+
let response;
|
|
31439
|
+
try {
|
|
31440
|
+
response = await input.client.getUrlLinks(input.site, input.target.url, page);
|
|
31441
|
+
} catch (error) {
|
|
31442
|
+
if (error instanceof SeoError && (error.code === "AUTH_REQUIRED" || error.code === "ACCESS_DENIED")) throw error;
|
|
31443
|
+
return {
|
|
31444
|
+
rows,
|
|
31445
|
+
requestedPages,
|
|
31446
|
+
invalidRows,
|
|
31447
|
+
returnedRows,
|
|
31448
|
+
partial: true,
|
|
31449
|
+
failed: true
|
|
31450
|
+
};
|
|
31451
|
+
}
|
|
31452
|
+
invalidRows += response.invalidRows;
|
|
31453
|
+
returnedRows += response.returnedRows;
|
|
31454
|
+
rows.push(...response.rows);
|
|
31455
|
+
totalPages = response.totalPages;
|
|
31456
|
+
partial ||= response.capped;
|
|
31457
|
+
if (page + 1 >= response.totalPages || response.rows.length === 0) break;
|
|
31458
|
+
}
|
|
31459
|
+
partial ||= requestedPages < totalPages;
|
|
31460
|
+
return {
|
|
31461
|
+
rows,
|
|
31462
|
+
requestedPages,
|
|
31463
|
+
invalidRows,
|
|
31464
|
+
returnedRows,
|
|
31465
|
+
partial,
|
|
31466
|
+
failed: false
|
|
31467
|
+
};
|
|
31468
|
+
}
|
|
31469
|
+
async function collectBingLinkEvidence(input) {
|
|
31470
|
+
if (!input.site.trim()) throw new SeoError("INVALID_INPUT", "Pass a Bing Webmaster site URL.");
|
|
31471
|
+
const targetLimit = integer(input.targetLimit, DEFAULT_TARGET_LIMIT, MAX_TARGET_LIMIT, "Bing link target limit");
|
|
31472
|
+
const detailPages = integer(input.detailPagesPerTarget, DEFAULT_DETAIL_PAGES, MAX_DETAIL_PAGES, "Bing detail page limit");
|
|
31473
|
+
const rowLimit = integer(input.rowLimit, DEFAULT_ROW_LIMIT$2, MAX_ROW_LIMIT$2, "Bing link row limit");
|
|
31474
|
+
const client = input.client ?? (await createBingWebmasterClient()).client;
|
|
31475
|
+
const countRows = [];
|
|
31476
|
+
let countPagesRequested = 0;
|
|
31477
|
+
let invalidRows = 0;
|
|
31478
|
+
let countInvalidRows = 0;
|
|
31479
|
+
let countRowsReturned = 0;
|
|
31480
|
+
let partial = false;
|
|
31481
|
+
let totalCountPages = 0;
|
|
31482
|
+
for (let page = 0; page < MAX_COUNT_PAGES; page += 1) {
|
|
31483
|
+
const response = await client.getLinkCounts(input.site, page);
|
|
31484
|
+
countPagesRequested += 1;
|
|
31485
|
+
invalidRows += response.invalidRows;
|
|
31486
|
+
countInvalidRows += response.invalidRows;
|
|
31487
|
+
countRowsReturned += response.returnedRows;
|
|
31488
|
+
countRows.push(...response.rows);
|
|
31489
|
+
totalCountPages = response.totalPages;
|
|
31490
|
+
partial ||= response.capped;
|
|
31491
|
+
if (countRows.length >= targetLimit || page + 1 >= response.totalPages || response.rows.length === 0) break;
|
|
31492
|
+
}
|
|
31493
|
+
partial ||= countPagesRequested < totalCountPages;
|
|
31494
|
+
const countsByUrl = /* @__PURE__ */ new Map();
|
|
31495
|
+
for (const row of countRows) countsByUrl.set(row.url, Math.max(row.count, countsByUrl.get(row.url) ?? 0));
|
|
31496
|
+
const counts = [...countsByUrl].map(([url, count]) => ({
|
|
31497
|
+
url,
|
|
31498
|
+
count
|
|
31499
|
+
}));
|
|
31500
|
+
counts.sort((a, b) => b.count - a.count || a.url.localeCompare(b.url, "en"));
|
|
31501
|
+
const selectedTargets = counts.slice(0, targetLimit);
|
|
31502
|
+
partial ||= counts.length > targetLimit;
|
|
31503
|
+
const details = await mapConcurrent(selectedTargets, MAX_CONCURRENCY, async (target) => targetDetails({
|
|
31504
|
+
client,
|
|
31505
|
+
site: input.site,
|
|
31506
|
+
target,
|
|
31507
|
+
pages: detailPages
|
|
31508
|
+
}));
|
|
31509
|
+
const rows = [];
|
|
31510
|
+
const keys = /* @__PURE__ */ new Set();
|
|
31511
|
+
let duplicateRows = 0;
|
|
31512
|
+
let detailPagesRequested = 0;
|
|
31513
|
+
let detailInvalidRows = 0;
|
|
31514
|
+
let detailRowsReturned = 0;
|
|
31515
|
+
let failedTargets = 0;
|
|
31516
|
+
for (const [index, detail] of details.entries()) {
|
|
31517
|
+
const target = selectedTargets[index];
|
|
31518
|
+
if (!target) continue;
|
|
31519
|
+
detailPagesRequested += detail.requestedPages;
|
|
31520
|
+
if (detail.failed) failedTargets += 1;
|
|
31521
|
+
invalidRows += detail.invalidRows;
|
|
31522
|
+
detailInvalidRows += detail.invalidRows;
|
|
31523
|
+
detailRowsReturned += detail.returnedRows;
|
|
31524
|
+
partial ||= detail.partial;
|
|
31525
|
+
for (const source of detail.rows) {
|
|
31526
|
+
if (rows.length >= rowLimit) {
|
|
31527
|
+
partial = true;
|
|
31528
|
+
break;
|
|
31529
|
+
}
|
|
31530
|
+
const row = normalizeLinkEvidenceRow({
|
|
31531
|
+
sourceUrl: source.url,
|
|
31532
|
+
targetUrl: target.url,
|
|
31533
|
+
anchorText: source.anchorText
|
|
31534
|
+
});
|
|
31535
|
+
if (!row) {
|
|
31536
|
+
invalidRows += 1;
|
|
31537
|
+
continue;
|
|
31538
|
+
}
|
|
31539
|
+
const key = linkEvidenceKey(row);
|
|
31540
|
+
if (keys.has(key)) {
|
|
31541
|
+
duplicateRows += 1;
|
|
31542
|
+
continue;
|
|
31543
|
+
}
|
|
31544
|
+
keys.add(key);
|
|
31545
|
+
rows.push(row);
|
|
31546
|
+
}
|
|
31547
|
+
}
|
|
31548
|
+
rows.sort(compareLinkEvidence);
|
|
31549
|
+
return {
|
|
31550
|
+
rows,
|
|
31551
|
+
targetCounts: selectedTargets.map((target) => ({
|
|
31552
|
+
targetUrl: target.url,
|
|
31553
|
+
providerReportedLinks: target.count,
|
|
31554
|
+
observedLinks: rows.filter((row) => row.targetUrl === target.url).length
|
|
31555
|
+
})),
|
|
31556
|
+
provenance: {
|
|
31557
|
+
provider: "bing-webmaster",
|
|
31558
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
31559
|
+
cached: false,
|
|
31560
|
+
suppliedRows: detailRowsReturned + detailInvalidRows,
|
|
31561
|
+
validRows: rows.length,
|
|
31562
|
+
invalidRows,
|
|
31563
|
+
duplicateRows,
|
|
31564
|
+
capped: partial,
|
|
31565
|
+
rowLimit,
|
|
31566
|
+
completeness: partial ? "partial" : "unknown",
|
|
31567
|
+
providerRequests: {
|
|
31568
|
+
methods: ["GetLinkCounts", "GetUrlLinks"],
|
|
31569
|
+
targetPagesRequested: countPagesRequested,
|
|
31570
|
+
detailPagesRequested,
|
|
31571
|
+
maxConcurrentRequests: MAX_CONCURRENCY
|
|
31572
|
+
},
|
|
31573
|
+
providerCoverage: {
|
|
31574
|
+
targetCountRows: {
|
|
31575
|
+
returnedRows: countRowsReturned,
|
|
31576
|
+
retainedRows: counts.length,
|
|
31577
|
+
invalidRows: countInvalidRows
|
|
31578
|
+
},
|
|
31579
|
+
detailRows: {
|
|
31580
|
+
returnedRows: detailRowsReturned,
|
|
31581
|
+
retainedRows: rows.length,
|
|
31582
|
+
invalidRows: detailInvalidRows
|
|
31583
|
+
}
|
|
31584
|
+
}
|
|
31585
|
+
},
|
|
31586
|
+
warnings: [
|
|
31587
|
+
...countRowsReturned === 0 ? ["Bing returned no target-link rows. This does not prove that the site has no links."] : [],
|
|
31588
|
+
...partial ? ["Bing link evidence was bounded by provider pagination, failed requests, or report limits."] : [],
|
|
31589
|
+
...failedTargets ? [`${failedTargets} target detail request${failedTargets === 1 ? "" : "s"} failed while other evidence was retained.`] : []
|
|
31590
|
+
]
|
|
31591
|
+
};
|
|
30002
31592
|
}
|
|
30003
31593
|
//#endregion
|
|
30004
|
-
//#region packages/core/src/
|
|
30005
|
-
|
|
31594
|
+
//#region packages/core/src/links/import.ts
|
|
31595
|
+
const DEFAULT_ROW_LIMIT$1 = 1e4;
|
|
31596
|
+
const MAX_ROW_LIMIT$1 = 1e5;
|
|
31597
|
+
const MAX_STREAM_BYTES = 5e7;
|
|
31598
|
+
const MAX_JSON_BYTES = 1e7;
|
|
31599
|
+
const MAX_CELL_BYTES = 1e6;
|
|
31600
|
+
function formatFor$1(path, explicit) {
|
|
31601
|
+
if (explicit) return explicit;
|
|
31602
|
+
const extension = extname(path).toLowerCase();
|
|
31603
|
+
if (extension === ".csv") return "csv";
|
|
31604
|
+
if (extension === ".jsonl" || extension === ".ndjson") return "jsonl";
|
|
31605
|
+
if (extension === ".json") return "json";
|
|
31606
|
+
throw new SeoError("INVALID_INPUT", "Link imports must be CSV, JSON, JSONL, or NDJSON.");
|
|
31607
|
+
}
|
|
31608
|
+
function boundedRowLimit(value) {
|
|
31609
|
+
const limit = value ?? DEFAULT_ROW_LIMIT$1;
|
|
31610
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_ROW_LIMIT$1) throw new SeoError("INVALID_INPUT", `Link import row limit must be between 1 and ${MAX_ROW_LIMIT$1}.`);
|
|
31611
|
+
return limit;
|
|
31612
|
+
}
|
|
31613
|
+
async function* csvRecords(path, byteLimit, progress) {
|
|
31614
|
+
const stream = createReadStream(path, { encoding: "utf8" });
|
|
31615
|
+
let record = [];
|
|
31616
|
+
let field = "";
|
|
31617
|
+
let quoted = false;
|
|
31618
|
+
let pendingQuote = false;
|
|
31619
|
+
for await (const chunk of stream) {
|
|
31620
|
+
progress.bytesRead += Buffer.byteLength(chunk);
|
|
31621
|
+
if (progress.bytesRead > byteLimit) {
|
|
31622
|
+
stream.destroy();
|
|
31623
|
+
throw new SeoError("INVALID_INPUT", `Link import exceeds the ${byteLimit}-byte streaming limit.`);
|
|
31624
|
+
}
|
|
31625
|
+
for (const character of chunk) {
|
|
31626
|
+
if (pendingQuote) {
|
|
31627
|
+
if (character === "\"") {
|
|
31628
|
+
field += "\"";
|
|
31629
|
+
pendingQuote = false;
|
|
31630
|
+
continue;
|
|
31631
|
+
}
|
|
31632
|
+
quoted = false;
|
|
31633
|
+
pendingQuote = false;
|
|
31634
|
+
}
|
|
31635
|
+
if (quoted) if (character === "\"") pendingQuote = true;
|
|
31636
|
+
else field += character;
|
|
31637
|
+
else if (character === "\"" && field.length === 0) quoted = true;
|
|
31638
|
+
else if (character === ",") {
|
|
31639
|
+
record.push(field);
|
|
31640
|
+
field = "";
|
|
31641
|
+
} else if (character === "\n") {
|
|
31642
|
+
record.push(field.replace(/\r$/, ""));
|
|
31643
|
+
field = "";
|
|
31644
|
+
if (record.some((value) => value.length > 0)) yield record;
|
|
31645
|
+
record = [];
|
|
31646
|
+
} else field += character;
|
|
31647
|
+
if (Buffer.byteLength(field) > MAX_CELL_BYTES) {
|
|
31648
|
+
stream.destroy();
|
|
31649
|
+
throw new SeoError("INVALID_INPUT", `Link import contains a cell larger than ${MAX_CELL_BYTES} bytes.`);
|
|
31650
|
+
}
|
|
31651
|
+
}
|
|
31652
|
+
}
|
|
31653
|
+
if (quoted && !pendingQuote) throw new SeoError("INVALID_INPUT", "Link CSV contains an unclosed quote.");
|
|
31654
|
+
if (field || record.length) {
|
|
31655
|
+
record.push(field.replace(/\r$/, ""));
|
|
31656
|
+
if (record.some((value) => value.length > 0)) yield record;
|
|
31657
|
+
}
|
|
31658
|
+
}
|
|
31659
|
+
async function* jsonlRecords(path, byteLimit, progress) {
|
|
31660
|
+
const stream = createReadStream(path, { encoding: "utf8" });
|
|
31661
|
+
let remainder = "";
|
|
31662
|
+
for await (const chunk of stream) {
|
|
31663
|
+
progress.bytesRead += Buffer.byteLength(chunk);
|
|
31664
|
+
if (progress.bytesRead > byteLimit) {
|
|
31665
|
+
stream.destroy();
|
|
31666
|
+
throw new SeoError("INVALID_INPUT", `Link import exceeds the ${byteLimit}-byte streaming limit.`);
|
|
31667
|
+
}
|
|
31668
|
+
remainder += chunk;
|
|
31669
|
+
let newline = remainder.indexOf("\n");
|
|
31670
|
+
while (newline >= 0) {
|
|
31671
|
+
const line = remainder.slice(0, newline).trim();
|
|
31672
|
+
remainder = remainder.slice(newline + 1);
|
|
31673
|
+
if (line) yield parseObject(line);
|
|
31674
|
+
newline = remainder.indexOf("\n");
|
|
31675
|
+
}
|
|
31676
|
+
if (Buffer.byteLength(remainder) > MAX_CELL_BYTES) {
|
|
31677
|
+
stream.destroy();
|
|
31678
|
+
throw new SeoError("INVALID_INPUT", `Link import contains a line larger than ${MAX_CELL_BYTES} bytes.`);
|
|
31679
|
+
}
|
|
31680
|
+
}
|
|
31681
|
+
if (remainder.trim()) yield parseObject(remainder);
|
|
31682
|
+
}
|
|
31683
|
+
function parseObject(value) {
|
|
30006
31684
|
try {
|
|
30007
|
-
const
|
|
30008
|
-
if (
|
|
30009
|
-
return
|
|
31685
|
+
const parsed = JSON.parse(value);
|
|
31686
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("not an object");
|
|
31687
|
+
return parsed;
|
|
30010
31688
|
} catch {
|
|
30011
|
-
|
|
31689
|
+
throw new SeoError("INVALID_INPUT", "Link JSONL must contain one JSON object per line.");
|
|
31690
|
+
}
|
|
31691
|
+
}
|
|
31692
|
+
function rowFromCsv(headers, values) {
|
|
31693
|
+
return Object.fromEntries(headers.map((header, index) => [header, values[index]]));
|
|
31694
|
+
}
|
|
31695
|
+
async function importLinkEvidence(input) {
|
|
31696
|
+
const path = resolve(input.file);
|
|
31697
|
+
const file = await stat(path).catch(() => void 0);
|
|
31698
|
+
if (!file?.isFile()) throw new SeoError("INVALID_INPUT", `Link import file was not found: ${path}`);
|
|
31699
|
+
const format = formatFor$1(path, input.format);
|
|
31700
|
+
const rowLimit = boundedRowLimit(input.rowLimit);
|
|
31701
|
+
if (format === "json" && file.size > MAX_JSON_BYTES) throw new SeoError("INVALID_INPUT", `JSON link imports are limited to ${MAX_JSON_BYTES} bytes. Use CSV or JSONL for larger files.`);
|
|
31702
|
+
const progress = { bytesRead: 0 };
|
|
31703
|
+
let records;
|
|
31704
|
+
if (format === "csv") {
|
|
31705
|
+
const csv = csvRecords(path, MAX_STREAM_BYTES, progress);
|
|
31706
|
+
const first = await csv.next();
|
|
31707
|
+
const headers = (Array.isArray(first.value) ? first.value : []).map((value, index) => index === 0 ? value.replace(/^\uFEFF/, "") : value);
|
|
31708
|
+
if (!headers.length) throw new SeoError("INVALID_INPUT", "Link CSV has no header row.");
|
|
31709
|
+
records = (async function* () {
|
|
31710
|
+
for await (const values of csv) yield rowFromCsv(headers, values);
|
|
31711
|
+
})();
|
|
31712
|
+
} else if (format === "jsonl") records = jsonlRecords(path, MAX_STREAM_BYTES, progress);
|
|
31713
|
+
else {
|
|
31714
|
+
const body = await readFile(path, "utf8");
|
|
31715
|
+
progress.bytesRead = Buffer.byteLength(body);
|
|
31716
|
+
let parsed;
|
|
31717
|
+
try {
|
|
31718
|
+
parsed = JSON.parse(body);
|
|
31719
|
+
} catch {
|
|
31720
|
+
throw new SeoError("INVALID_INPUT", "Link JSON is not valid JSON.");
|
|
31721
|
+
}
|
|
31722
|
+
if (!Array.isArray(parsed)) throw new SeoError("INVALID_INPUT", "Link JSON must be an array of objects.");
|
|
31723
|
+
records = (async function* () {
|
|
31724
|
+
for (const row of parsed) yield row && typeof row === "object" && !Array.isArray(row) ? row : {};
|
|
31725
|
+
})();
|
|
30012
31726
|
}
|
|
31727
|
+
const rows = [];
|
|
31728
|
+
const keys = /* @__PURE__ */ new Set();
|
|
31729
|
+
let suppliedRows = 0;
|
|
31730
|
+
let invalidRows = 0;
|
|
31731
|
+
let duplicateRows = 0;
|
|
31732
|
+
let capped = false;
|
|
31733
|
+
for await (const raw of records) {
|
|
31734
|
+
if (suppliedRows >= rowLimit) {
|
|
31735
|
+
capped = true;
|
|
31736
|
+
break;
|
|
31737
|
+
}
|
|
31738
|
+
suppliedRows += 1;
|
|
31739
|
+
const row = normalizeLinkEvidenceRow(raw);
|
|
31740
|
+
if (!row) {
|
|
31741
|
+
invalidRows += 1;
|
|
31742
|
+
continue;
|
|
31743
|
+
}
|
|
31744
|
+
const key = linkEvidenceKey(row);
|
|
31745
|
+
if (keys.has(key)) {
|
|
31746
|
+
duplicateRows += 1;
|
|
31747
|
+
continue;
|
|
31748
|
+
}
|
|
31749
|
+
keys.add(key);
|
|
31750
|
+
rows.push(row);
|
|
31751
|
+
}
|
|
31752
|
+
rows.sort(compareLinkEvidence);
|
|
31753
|
+
const observedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
31754
|
+
return {
|
|
31755
|
+
rows,
|
|
31756
|
+
targetCounts: [],
|
|
31757
|
+
provenance: {
|
|
31758
|
+
provider: `${format}-import`,
|
|
31759
|
+
observedAt,
|
|
31760
|
+
cached: false,
|
|
31761
|
+
suppliedRows,
|
|
31762
|
+
validRows: rows.length,
|
|
31763
|
+
invalidRows,
|
|
31764
|
+
duplicateRows,
|
|
31765
|
+
capped,
|
|
31766
|
+
rowLimit,
|
|
31767
|
+
completeness: capped ? "partial" : "unknown",
|
|
31768
|
+
file: {
|
|
31769
|
+
path,
|
|
31770
|
+
format,
|
|
31771
|
+
bytesRead: progress.bytesRead,
|
|
31772
|
+
fileBytes: file.size
|
|
31773
|
+
}
|
|
31774
|
+
},
|
|
31775
|
+
warnings: capped ? [`Import stopped after ${rowLimit} rows. The file contains more data.`] : []
|
|
31776
|
+
};
|
|
30013
31777
|
}
|
|
30014
|
-
|
|
30015
|
-
|
|
30016
|
-
|
|
30017
|
-
|
|
30018
|
-
|
|
30019
|
-
|
|
30020
|
-
|
|
30021
|
-
|
|
30022
|
-
if (domain) return streamHostname === domain || streamHostname.endsWith(`.${domain}`) ? "domain" : void 0;
|
|
30023
|
-
return hostnameFromUrl(site) === streamHostname ? "hostname" : void 0;
|
|
31778
|
+
//#endregion
|
|
31779
|
+
//#region packages/core/src/links/report.ts
|
|
31780
|
+
const DEFAULT_OUTPUT_LIMIT = 100;
|
|
31781
|
+
const MAX_OUTPUT_LIMIT = 500;
|
|
31782
|
+
function boundedLimit(value) {
|
|
31783
|
+
const limit = value ?? DEFAULT_OUTPUT_LIMIT;
|
|
31784
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_OUTPUT_LIMIT) throw new SeoError("INVALID_INPUT", `Link report limit must be between 1 and ${MAX_OUTPUT_LIMIT}.`);
|
|
31785
|
+
return limit;
|
|
30024
31786
|
}
|
|
30025
|
-
function
|
|
30026
|
-
|
|
30027
|
-
|
|
30028
|
-
|
|
30029
|
-
|
|
30030
|
-
|
|
30031
|
-
|
|
30032
|
-
if (!match) return [];
|
|
30033
|
-
return [{
|
|
30034
|
-
...candidate,
|
|
30035
|
-
defaultUri,
|
|
30036
|
-
hostname,
|
|
30037
|
-
match
|
|
30038
|
-
}];
|
|
30039
|
-
});
|
|
31787
|
+
function targetCounts(rows) {
|
|
31788
|
+
const counts = /* @__PURE__ */ new Map();
|
|
31789
|
+
for (const row of rows) counts.set(row.targetUrl, (counts.get(row.targetUrl) ?? 0) + 1);
|
|
31790
|
+
return [...counts].map(([targetUrl, observedLinks]) => ({
|
|
31791
|
+
targetUrl,
|
|
31792
|
+
observedLinks
|
|
31793
|
+
})).sort((a, b) => b.observedLinks - a.observedLinks || a.targetUrl.localeCompare(b.targetUrl, "en"));
|
|
30040
31794
|
}
|
|
30041
|
-
function
|
|
30042
|
-
const
|
|
30043
|
-
const
|
|
30044
|
-
|
|
31795
|
+
function linkEvidenceReport(input) {
|
|
31796
|
+
const limit = boundedLimit(input.limit);
|
|
31797
|
+
const rows = input.evidence.rows.slice(0, limit);
|
|
31798
|
+
const domains = new Set(input.evidence.rows.map((row) => row.sourceDomain));
|
|
31799
|
+
const observedTargets = targetCounts(input.evidence.rows);
|
|
31800
|
+
const counts = input.evidence.targetCounts.length ? input.evidence.targetCounts : observedTargets;
|
|
31801
|
+
return {
|
|
31802
|
+
schemaVersion: 1,
|
|
31803
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
31804
|
+
dataStatus: input.evidence.provenance.completeness === "complete" ? "complete" : "partial",
|
|
31805
|
+
summary: {
|
|
31806
|
+
observedLinks: input.evidence.rows.length,
|
|
31807
|
+
referringDomains: domains.size,
|
|
31808
|
+
targetPages: new Set(input.evidence.rows.map((row) => row.targetUrl)).size,
|
|
31809
|
+
providerTargetPages: counts.length
|
|
31810
|
+
},
|
|
31811
|
+
provenance: input.evidence.provenance,
|
|
31812
|
+
selection: {
|
|
31813
|
+
availableRows: input.evidence.rows.length,
|
|
31814
|
+
returnedRows: rows.length,
|
|
31815
|
+
omittedRows: Math.max(0, input.evidence.rows.length - rows.length),
|
|
31816
|
+
limit
|
|
31817
|
+
},
|
|
31818
|
+
targetCounts: counts.slice(0, 100),
|
|
31819
|
+
links: rows,
|
|
31820
|
+
warnings: input.evidence.warnings,
|
|
31821
|
+
caveats: [
|
|
31822
|
+
"This is bounded provider or imported evidence, not a complete backlink index.",
|
|
31823
|
+
"A missing link is not proof that no link exists, and an observed link is not a quality or ranking verdict.",
|
|
31824
|
+
"Referring-domain counts describe the retained rows only."
|
|
31825
|
+
]
|
|
31826
|
+
};
|
|
30045
31827
|
}
|
|
30046
31828
|
//#endregion
|
|
30047
31829
|
//#region packages/core/src/storage/log-retention.ts
|
|
@@ -30418,6 +32200,472 @@ var JsonFileStorageAdapter = class {
|
|
|
30418
32200
|
};
|
|
30419
32201
|
const localCredentialsProvider = new LocalCredentialsProvider();
|
|
30420
32202
|
//#endregion
|
|
32203
|
+
//#region packages/core/src/server-logs/parse.ts
|
|
32204
|
+
const MONTHS = {
|
|
32205
|
+
Jan: 0,
|
|
32206
|
+
Feb: 1,
|
|
32207
|
+
Mar: 2,
|
|
32208
|
+
Apr: 3,
|
|
32209
|
+
May: 4,
|
|
32210
|
+
Jun: 5,
|
|
32211
|
+
Jul: 6,
|
|
32212
|
+
Aug: 7,
|
|
32213
|
+
Sep: 8,
|
|
32214
|
+
Oct: 9,
|
|
32215
|
+
Nov: 10,
|
|
32216
|
+
Dec: 11
|
|
32217
|
+
};
|
|
32218
|
+
const CRAWLERS = [
|
|
32219
|
+
{
|
|
32220
|
+
family: "Googlebot",
|
|
32221
|
+
category: "search",
|
|
32222
|
+
patterns: ["googlebot", "google-inspectiontool"]
|
|
32223
|
+
},
|
|
32224
|
+
{
|
|
32225
|
+
family: "Bingbot",
|
|
32226
|
+
category: "search",
|
|
32227
|
+
patterns: ["bingbot", "bingpreview"]
|
|
32228
|
+
},
|
|
32229
|
+
{
|
|
32230
|
+
family: "Applebot",
|
|
32231
|
+
category: "search",
|
|
32232
|
+
patterns: ["applebot"]
|
|
32233
|
+
},
|
|
32234
|
+
{
|
|
32235
|
+
family: "DuckDuckBot",
|
|
32236
|
+
category: "search",
|
|
32237
|
+
patterns: ["duckduckbot"]
|
|
32238
|
+
},
|
|
32239
|
+
{
|
|
32240
|
+
family: "YandexBot",
|
|
32241
|
+
category: "search",
|
|
32242
|
+
patterns: ["yandexbot"]
|
|
32243
|
+
},
|
|
32244
|
+
{
|
|
32245
|
+
family: "Baiduspider",
|
|
32246
|
+
category: "search",
|
|
32247
|
+
patterns: ["baiduspider"]
|
|
32248
|
+
},
|
|
32249
|
+
{
|
|
32250
|
+
family: "PetalBot",
|
|
32251
|
+
category: "search",
|
|
32252
|
+
patterns: ["petalbot"]
|
|
32253
|
+
},
|
|
32254
|
+
{
|
|
32255
|
+
family: "OpenAI",
|
|
32256
|
+
category: "ai",
|
|
32257
|
+
patterns: [
|
|
32258
|
+
"gptbot",
|
|
32259
|
+
"chatgpt-user",
|
|
32260
|
+
"oai-searchbot"
|
|
32261
|
+
]
|
|
32262
|
+
},
|
|
32263
|
+
{
|
|
32264
|
+
family: "Anthropic",
|
|
32265
|
+
category: "ai",
|
|
32266
|
+
patterns: [
|
|
32267
|
+
"claudebot",
|
|
32268
|
+
"claude-searchbot",
|
|
32269
|
+
"claude-user"
|
|
32270
|
+
]
|
|
32271
|
+
},
|
|
32272
|
+
{
|
|
32273
|
+
family: "Perplexity",
|
|
32274
|
+
category: "ai",
|
|
32275
|
+
patterns: ["perplexitybot", "perplexity-user"]
|
|
32276
|
+
},
|
|
32277
|
+
{
|
|
32278
|
+
family: "Amazonbot",
|
|
32279
|
+
category: "ai",
|
|
32280
|
+
patterns: ["amazonbot"]
|
|
32281
|
+
},
|
|
32282
|
+
{
|
|
32283
|
+
family: "Bytespider",
|
|
32284
|
+
category: "ai",
|
|
32285
|
+
patterns: ["bytespider"]
|
|
32286
|
+
},
|
|
32287
|
+
{
|
|
32288
|
+
family: "CCBot",
|
|
32289
|
+
category: "ai",
|
|
32290
|
+
patterns: ["ccbot"]
|
|
32291
|
+
},
|
|
32292
|
+
{
|
|
32293
|
+
family: "Cohere",
|
|
32294
|
+
category: "ai",
|
|
32295
|
+
patterns: ["cohere-ai"]
|
|
32296
|
+
}
|
|
32297
|
+
];
|
|
32298
|
+
const COMBINED_LOG = /^\S+\s+\S+\s+\S+\s+\[([^\]]+)]\s+"([^"\s]+)\s+([^"\s]+)(?:\s+[^"\s]+)?"\s+(\d{3})\s+(\d+|-)\s*(?:"[^"]*"\s+"([^"]*)")?\s*$/;
|
|
32299
|
+
function classifyCrawler(userAgent) {
|
|
32300
|
+
if (!userAgent) return void 0;
|
|
32301
|
+
const value = userAgent.toLowerCase();
|
|
32302
|
+
const match = CRAWLERS.find((crawler) => crawler.patterns.some((pattern) => value.includes(pattern)));
|
|
32303
|
+
return match ? {
|
|
32304
|
+
family: match.family,
|
|
32305
|
+
category: match.category
|
|
32306
|
+
} : void 0;
|
|
32307
|
+
}
|
|
32308
|
+
function logTimestamp(value) {
|
|
32309
|
+
const match = /^(\d{2})\/([A-Z][a-z]{2})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})$/.exec(value);
|
|
32310
|
+
if (!match) return void 0;
|
|
32311
|
+
const month = MONTHS[match[2] ?? ""];
|
|
32312
|
+
if (month === void 0) return void 0;
|
|
32313
|
+
const utc = Date.UTC(Number(match[3]), month, Number(match[1]), Number(match[4]), Number(match[5]), Number(match[6]));
|
|
32314
|
+
const offsetMinutes = Number(match[8]) * 60 + Number(match[9]);
|
|
32315
|
+
const adjusted = utc + (match[7] === "+" ? -1 : 1) * offsetMinutes * 6e4;
|
|
32316
|
+
const date = new Date(adjusted);
|
|
32317
|
+
return Number.isNaN(date.getTime()) ? void 0 : date.toISOString();
|
|
32318
|
+
}
|
|
32319
|
+
function normalizedPath(value) {
|
|
32320
|
+
if (typeof value !== "string" || value.length < 1 || value.length > 2e3) return;
|
|
32321
|
+
try {
|
|
32322
|
+
if (value.startsWith("/")) return new URL(value, "https://server-log.invalid").pathname;
|
|
32323
|
+
const url = new URL(value);
|
|
32324
|
+
return ["http:", "https:"].includes(url.protocol) ? url.pathname : void 0;
|
|
32325
|
+
} catch {
|
|
32326
|
+
return;
|
|
32327
|
+
}
|
|
32328
|
+
}
|
|
32329
|
+
function normalizedMethod(value) {
|
|
32330
|
+
if (typeof value !== "string" || !/^[A-Za-z]{1,16}$/.test(value)) return;
|
|
32331
|
+
return value.toUpperCase();
|
|
32332
|
+
}
|
|
32333
|
+
function normalizedStatus(value) {
|
|
32334
|
+
const status = typeof value === "number" ? value : Number(value);
|
|
32335
|
+
return Number.isInteger(status) && status >= 100 && status <= 599 ? status : void 0;
|
|
32336
|
+
}
|
|
32337
|
+
function normalizedBytes(value) {
|
|
32338
|
+
if (value === void 0 || value === null || value === "" || value === "-") return;
|
|
32339
|
+
const bytes = typeof value === "number" ? value : Number(value);
|
|
32340
|
+
return Number.isSafeInteger(bytes) && bytes >= 0 ? bytes : void 0;
|
|
32341
|
+
}
|
|
32342
|
+
function record(input) {
|
|
32343
|
+
let timestamp;
|
|
32344
|
+
if (typeof input.timestamp === "string") {
|
|
32345
|
+
const date = new Date(input.timestamp);
|
|
32346
|
+
if (!Number.isNaN(date.getTime())) timestamp = date.toISOString();
|
|
32347
|
+
}
|
|
32348
|
+
const method = normalizedMethod(input.method);
|
|
32349
|
+
const path = normalizedPath(input.path);
|
|
32350
|
+
const status = normalizedStatus(input.status);
|
|
32351
|
+
if (!timestamp || !method || !path || status === void 0) return void 0;
|
|
32352
|
+
const userAgent = typeof input.userAgent === "string" && input.userAgent.length <= 2e3 ? input.userAgent : void 0;
|
|
32353
|
+
return {
|
|
32354
|
+
timestamp,
|
|
32355
|
+
method,
|
|
32356
|
+
path,
|
|
32357
|
+
status,
|
|
32358
|
+
bytes: normalizedBytes(input.bytes),
|
|
32359
|
+
userAgent,
|
|
32360
|
+
crawler: classifyCrawler(userAgent)
|
|
32361
|
+
};
|
|
32362
|
+
}
|
|
32363
|
+
function parseCombinedLogLine(line) {
|
|
32364
|
+
const match = COMBINED_LOG.exec(line);
|
|
32365
|
+
if (!match) return void 0;
|
|
32366
|
+
const timestamp = logTimestamp(match[1] ?? "");
|
|
32367
|
+
if (!timestamp) return void 0;
|
|
32368
|
+
return record({
|
|
32369
|
+
timestamp,
|
|
32370
|
+
method: match[2],
|
|
32371
|
+
path: match[3],
|
|
32372
|
+
status: match[4],
|
|
32373
|
+
bytes: match[5],
|
|
32374
|
+
userAgent: match[6]
|
|
32375
|
+
});
|
|
32376
|
+
}
|
|
32377
|
+
function firstValue(value, names) {
|
|
32378
|
+
for (const name of names) if (value[name] !== void 0) return value[name];
|
|
32379
|
+
}
|
|
32380
|
+
function parseJsonLogLine(line) {
|
|
32381
|
+
let value;
|
|
32382
|
+
try {
|
|
32383
|
+
value = JSON.parse(line);
|
|
32384
|
+
} catch {
|
|
32385
|
+
return;
|
|
32386
|
+
}
|
|
32387
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return;
|
|
32388
|
+
const row = value;
|
|
32389
|
+
return record({
|
|
32390
|
+
timestamp: firstValue(row, [
|
|
32391
|
+
"timestamp",
|
|
32392
|
+
"time",
|
|
32393
|
+
"datetime",
|
|
32394
|
+
"@timestamp"
|
|
32395
|
+
]),
|
|
32396
|
+
method: firstValue(row, [
|
|
32397
|
+
"method",
|
|
32398
|
+
"requestMethod",
|
|
32399
|
+
"http_method"
|
|
32400
|
+
]),
|
|
32401
|
+
path: firstValue(row, [
|
|
32402
|
+
"path",
|
|
32403
|
+
"url",
|
|
32404
|
+
"requestUri",
|
|
32405
|
+
"request_uri"
|
|
32406
|
+
]),
|
|
32407
|
+
status: firstValue(row, [
|
|
32408
|
+
"status",
|
|
32409
|
+
"statusCode",
|
|
32410
|
+
"status_code"
|
|
32411
|
+
]),
|
|
32412
|
+
bytes: firstValue(row, [
|
|
32413
|
+
"bytes",
|
|
32414
|
+
"bodyBytesSent",
|
|
32415
|
+
"body_bytes_sent"
|
|
32416
|
+
]),
|
|
32417
|
+
userAgent: firstValue(row, [
|
|
32418
|
+
"userAgent",
|
|
32419
|
+
"user_agent",
|
|
32420
|
+
"httpUserAgent",
|
|
32421
|
+
"http_user_agent"
|
|
32422
|
+
])
|
|
32423
|
+
});
|
|
32424
|
+
}
|
|
32425
|
+
function emptyStatusBreakdown() {
|
|
32426
|
+
return {
|
|
32427
|
+
success: 0,
|
|
32428
|
+
redirect: 0,
|
|
32429
|
+
clientError: 0,
|
|
32430
|
+
serverError: 0,
|
|
32431
|
+
other: 0
|
|
32432
|
+
};
|
|
32433
|
+
}
|
|
32434
|
+
function addStatus(breakdown, status) {
|
|
32435
|
+
if (status >= 200 && status < 300) breakdown.success += 1;
|
|
32436
|
+
else if (status >= 300 && status < 400) breakdown.redirect += 1;
|
|
32437
|
+
else if (status >= 400 && status < 500) breakdown.clientError += 1;
|
|
32438
|
+
else if (status >= 500) breakdown.serverError += 1;
|
|
32439
|
+
else breakdown.other += 1;
|
|
32440
|
+
}
|
|
32441
|
+
//#endregion
|
|
32442
|
+
//#region packages/core/src/server-logs/import.ts
|
|
32443
|
+
const DEFAULT_ROW_LIMIT = 1e6;
|
|
32444
|
+
const MAX_ROW_LIMIT = 1e7;
|
|
32445
|
+
const DEFAULT_PATH_LIMIT = 25e3;
|
|
32446
|
+
const MAX_PATH_LIMIT = 1e5;
|
|
32447
|
+
const BYTE_LIMIT = 1e9;
|
|
32448
|
+
const MAX_LINE_BYTES = 65536;
|
|
32449
|
+
function boundedInteger(value, fallback, maximum, label) {
|
|
32450
|
+
const result = value ?? fallback;
|
|
32451
|
+
if (!Number.isInteger(result) || result < 1 || result > maximum) throw new SeoError("INVALID_INPUT", `${label} must be between 1 and ${maximum}.`);
|
|
32452
|
+
return result;
|
|
32453
|
+
}
|
|
32454
|
+
function formatFor(path, explicit) {
|
|
32455
|
+
if (explicit) return explicit;
|
|
32456
|
+
const extension = extname(path).toLowerCase();
|
|
32457
|
+
return [".jsonl", ".ndjson"].includes(extension) ? "jsonl" : "combined";
|
|
32458
|
+
}
|
|
32459
|
+
async function* lines(path, progress) {
|
|
32460
|
+
const stream = createReadStream(path, {
|
|
32461
|
+
encoding: "utf8",
|
|
32462
|
+
highWaterMark: 64 * 1024
|
|
32463
|
+
});
|
|
32464
|
+
let remainder = "";
|
|
32465
|
+
let discarding = false;
|
|
32466
|
+
for await (const chunk of stream) {
|
|
32467
|
+
const remainingBytes = BYTE_LIMIT - progress.bytesRead;
|
|
32468
|
+
if (remainingBytes <= 0) {
|
|
32469
|
+
progress.bytesCapped = true;
|
|
32470
|
+
break;
|
|
32471
|
+
}
|
|
32472
|
+
const chunkBytes = Buffer.byteLength(chunk);
|
|
32473
|
+
const accepted = chunkBytes > remainingBytes ? Buffer.from(chunk).subarray(0, remainingBytes).toString("utf8") : chunk;
|
|
32474
|
+
progress.bytesRead += Buffer.byteLength(accepted);
|
|
32475
|
+
if (chunkBytes > remainingBytes) progress.bytesCapped = true;
|
|
32476
|
+
remainder += accepted;
|
|
32477
|
+
let newline = remainder.indexOf("\n");
|
|
32478
|
+
while (newline >= 0) {
|
|
32479
|
+
const line = remainder.slice(0, newline).replace(/\r$/, "");
|
|
32480
|
+
remainder = remainder.slice(newline + 1);
|
|
32481
|
+
if (discarding || Buffer.byteLength(line) > MAX_LINE_BYTES) {
|
|
32482
|
+
yield { tooLong: true };
|
|
32483
|
+
discarding = false;
|
|
32484
|
+
} else if (line.trim()) yield { line };
|
|
32485
|
+
newline = remainder.indexOf("\n");
|
|
32486
|
+
}
|
|
32487
|
+
if (Buffer.byteLength(remainder) > MAX_LINE_BYTES) {
|
|
32488
|
+
remainder = "";
|
|
32489
|
+
discarding = true;
|
|
32490
|
+
}
|
|
32491
|
+
if (progress.bytesCapped) break;
|
|
32492
|
+
}
|
|
32493
|
+
if (discarding || Buffer.byteLength(remainder) > MAX_LINE_BYTES) yield { tooLong: true };
|
|
32494
|
+
else if (remainder.trim()) yield { line: remainder.replace(/\r$/, "") };
|
|
32495
|
+
}
|
|
32496
|
+
function compareCrawler(a, b) {
|
|
32497
|
+
return b.requests - a.requests || a.family.localeCompare(b.family, "en");
|
|
32498
|
+
}
|
|
32499
|
+
function compareCrawlerPath(a, b) {
|
|
32500
|
+
return b.requests - a.requests || a.family.localeCompare(b.family, "en") || a.path.localeCompare(b.path, "en");
|
|
32501
|
+
}
|
|
32502
|
+
function updatePeriod(timestamp, period) {
|
|
32503
|
+
if (!period.firstSeenAt || timestamp < period.firstSeenAt) period.firstSeenAt = timestamp;
|
|
32504
|
+
if (!period.lastSeenAt || timestamp > period.lastSeenAt) period.lastSeenAt = timestamp;
|
|
32505
|
+
}
|
|
32506
|
+
function newCrawler(record) {
|
|
32507
|
+
const crawler = record.crawler;
|
|
32508
|
+
if (!crawler) throw new Error("Crawler evidence is required.");
|
|
32509
|
+
return {
|
|
32510
|
+
family: crawler.family,
|
|
32511
|
+
category: crawler.category,
|
|
32512
|
+
requests: 0,
|
|
32513
|
+
lastSeenAt: record.timestamp,
|
|
32514
|
+
...emptyStatusBreakdown()
|
|
32515
|
+
};
|
|
32516
|
+
}
|
|
32517
|
+
function newCrawlerPath(record) {
|
|
32518
|
+
return {
|
|
32519
|
+
...newCrawler(record),
|
|
32520
|
+
path: record.path
|
|
32521
|
+
};
|
|
32522
|
+
}
|
|
32523
|
+
async function importServerLog(input) {
|
|
32524
|
+
const path = resolve(input.file);
|
|
32525
|
+
const file = await stat(path).catch(() => void 0);
|
|
32526
|
+
if (!file?.isFile()) throw new SeoError("INVALID_INPUT", `Server log was not found: ${path}`);
|
|
32527
|
+
const format = formatFor(path, input.format);
|
|
32528
|
+
const rowLimit = boundedInteger(input.rowLimit, DEFAULT_ROW_LIMIT, MAX_ROW_LIMIT, "Server log row limit");
|
|
32529
|
+
const pathLimit = boundedInteger(input.pathLimit, DEFAULT_PATH_LIMIT, MAX_PATH_LIMIT, "Server log path limit");
|
|
32530
|
+
const parse = format === "jsonl" ? parseJsonLogLine : parseCombinedLogLine;
|
|
32531
|
+
const progress = {
|
|
32532
|
+
bytesRead: 0,
|
|
32533
|
+
bytesCapped: false
|
|
32534
|
+
};
|
|
32535
|
+
const period = {};
|
|
32536
|
+
const statusCodes = /* @__PURE__ */ new Map();
|
|
32537
|
+
const crawlers = /* @__PURE__ */ new Map();
|
|
32538
|
+
const crawlerPaths = /* @__PURE__ */ new Map();
|
|
32539
|
+
let suppliedRows = 0;
|
|
32540
|
+
let parsedRows = 0;
|
|
32541
|
+
let invalidRows = 0;
|
|
32542
|
+
let crawlerRows = 0;
|
|
32543
|
+
let responseBytes = 0;
|
|
32544
|
+
let rowsCapped = false;
|
|
32545
|
+
let pathsCapped = false;
|
|
32546
|
+
let untrackedCrawlerPathRows = 0;
|
|
32547
|
+
for await (const item of lines(path, progress)) {
|
|
32548
|
+
if (suppliedRows >= rowLimit) {
|
|
32549
|
+
rowsCapped = true;
|
|
32550
|
+
break;
|
|
32551
|
+
}
|
|
32552
|
+
suppliedRows += 1;
|
|
32553
|
+
const parsed = item.line ? parse(item.line) : void 0;
|
|
32554
|
+
if (!parsed) {
|
|
32555
|
+
invalidRows += 1;
|
|
32556
|
+
continue;
|
|
32557
|
+
}
|
|
32558
|
+
parsedRows += 1;
|
|
32559
|
+
responseBytes += parsed.bytes ?? 0;
|
|
32560
|
+
updatePeriod(parsed.timestamp, period);
|
|
32561
|
+
statusCodes.set(parsed.status, (statusCodes.get(parsed.status) ?? 0) + 1);
|
|
32562
|
+
if (!parsed.crawler) continue;
|
|
32563
|
+
crawlerRows += 1;
|
|
32564
|
+
const crawler = crawlers.get(parsed.crawler.family) ?? newCrawler(parsed);
|
|
32565
|
+
crawler.requests += 1;
|
|
32566
|
+
if (parsed.timestamp > crawler.lastSeenAt) crawler.lastSeenAt = parsed.timestamp;
|
|
32567
|
+
addStatus(crawler, parsed.status);
|
|
32568
|
+
crawlers.set(parsed.crawler.family, crawler);
|
|
32569
|
+
const key = `${parsed.crawler.family}\u0000${parsed.path}`;
|
|
32570
|
+
let crawlerPath = crawlerPaths.get(key);
|
|
32571
|
+
if (!crawlerPath && crawlerPaths.size >= pathLimit) {
|
|
32572
|
+
pathsCapped = true;
|
|
32573
|
+
untrackedCrawlerPathRows += 1;
|
|
32574
|
+
continue;
|
|
32575
|
+
}
|
|
32576
|
+
crawlerPath ??= newCrawlerPath(parsed);
|
|
32577
|
+
crawlerPath.requests += 1;
|
|
32578
|
+
if (parsed.timestamp > crawlerPath.lastSeenAt) crawlerPath.lastSeenAt = parsed.timestamp;
|
|
32579
|
+
addStatus(crawlerPath, parsed.status);
|
|
32580
|
+
crawlerPaths.set(key, crawlerPath);
|
|
32581
|
+
}
|
|
32582
|
+
const fileReadCompletely = !rowsCapped && !progress.bytesCapped && progress.bytesRead >= file.size;
|
|
32583
|
+
const completeness = fileReadCompletely && invalidRows === 0 && !pathsCapped ? "complete" : "partial";
|
|
32584
|
+
const warnings = [];
|
|
32585
|
+
if (rowsCapped) warnings.push(`Analysis stopped after ${rowLimit} rows.`);
|
|
32586
|
+
if (progress.bytesCapped) warnings.push(`Analysis stopped after ${BYTE_LIMIT} input bytes.`);
|
|
32587
|
+
if (invalidRows) warnings.push(`${invalidRows} malformed or unsupported rows were skipped.`);
|
|
32588
|
+
if (pathsCapped) warnings.push(`Crawler path tracking stopped at ${pathLimit} unique crawler and path pairs. Overall crawler and status totals still include later valid rows.`);
|
|
32589
|
+
return {
|
|
32590
|
+
summary: {
|
|
32591
|
+
suppliedRows,
|
|
32592
|
+
parsedRows,
|
|
32593
|
+
invalidRows,
|
|
32594
|
+
crawlerRows,
|
|
32595
|
+
nonCrawlerRows: parsedRows - crawlerRows,
|
|
32596
|
+
responseBytes,
|
|
32597
|
+
...period
|
|
32598
|
+
},
|
|
32599
|
+
statusCodes: [...statusCodes].map(([status, requests]) => ({
|
|
32600
|
+
status,
|
|
32601
|
+
requests
|
|
32602
|
+
})).sort((a, b) => a.status - b.status),
|
|
32603
|
+
crawlers: [...crawlers.values()].sort(compareCrawler),
|
|
32604
|
+
crawlerPaths: [...crawlerPaths.values()].sort(compareCrawlerPath),
|
|
32605
|
+
provenance: {
|
|
32606
|
+
source: "local-server-log",
|
|
32607
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
32608
|
+
cached: false,
|
|
32609
|
+
file: {
|
|
32610
|
+
path,
|
|
32611
|
+
format,
|
|
32612
|
+
bytesRead: progress.bytesRead,
|
|
32613
|
+
fileBytes: file.size
|
|
32614
|
+
},
|
|
32615
|
+
limits: {
|
|
32616
|
+
rowLimit,
|
|
32617
|
+
pathLimit,
|
|
32618
|
+
byteLimit: BYTE_LIMIT,
|
|
32619
|
+
maxLineBytes: MAX_LINE_BYTES
|
|
32620
|
+
},
|
|
32621
|
+
coverage: {
|
|
32622
|
+
fileReadCompletely,
|
|
32623
|
+
rowsCapped,
|
|
32624
|
+
bytesCapped: progress.bytesCapped,
|
|
32625
|
+
pathsCapped,
|
|
32626
|
+
untrackedCrawlerPathRows
|
|
32627
|
+
},
|
|
32628
|
+
completeness
|
|
32629
|
+
},
|
|
32630
|
+
warnings
|
|
32631
|
+
};
|
|
32632
|
+
}
|
|
32633
|
+
//#endregion
|
|
32634
|
+
//#region packages/core/src/server-logs/report.ts
|
|
32635
|
+
const DEFAULT_LIMIT = 100;
|
|
32636
|
+
const MAX_LIMIT = 500;
|
|
32637
|
+
function outputLimit(value) {
|
|
32638
|
+
const limit = value ?? DEFAULT_LIMIT;
|
|
32639
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) throw new SeoError("INVALID_INPUT", `Server log output limit must be between 1 and ${MAX_LIMIT}.`);
|
|
32640
|
+
return limit;
|
|
32641
|
+
}
|
|
32642
|
+
function serverLogReport(input) {
|
|
32643
|
+
const limit = outputLimit(input.limit);
|
|
32644
|
+
const crawlerPaths = input.evidence.crawlerPaths.slice(0, limit);
|
|
32645
|
+
return {
|
|
32646
|
+
schemaVersion: 1,
|
|
32647
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
32648
|
+
dataStatus: input.evidence.provenance.completeness,
|
|
32649
|
+
summary: input.evidence.summary,
|
|
32650
|
+
statusCodes: input.evidence.statusCodes,
|
|
32651
|
+
crawlers: input.evidence.crawlers,
|
|
32652
|
+
crawlerPaths,
|
|
32653
|
+
selection: {
|
|
32654
|
+
availableCrawlerPaths: input.evidence.crawlerPaths.length,
|
|
32655
|
+
returnedCrawlerPaths: crawlerPaths.length,
|
|
32656
|
+
omittedCrawlerPaths: Math.max(0, input.evidence.crawlerPaths.length - crawlerPaths.length),
|
|
32657
|
+
limit
|
|
32658
|
+
},
|
|
32659
|
+
provenance: input.evidence.provenance,
|
|
32660
|
+
warnings: input.evidence.warnings,
|
|
32661
|
+
caveats: [
|
|
32662
|
+
"Crawler families are identified from user-agent strings, which can be missing or spoofed.",
|
|
32663
|
+
"This report describes only the supplied log file and cannot prove that every server or time period is represented.",
|
|
32664
|
+
"Crawler path rankings describe retained path aggregates when the path limit is reached."
|
|
32665
|
+
]
|
|
32666
|
+
};
|
|
32667
|
+
}
|
|
32668
|
+
//#endregion
|
|
30421
32669
|
//#region packages/core/src/telemetry.ts
|
|
30422
32670
|
const TELEMETRY_ENDPOINT = "https://seoskill.dev/api/t";
|
|
30423
32671
|
const TELEMETRY_SCHEMA_VERSION = 1;
|
|
@@ -30455,6 +32703,7 @@ const TELEMETRY_REPORTS = [
|
|
|
30455
32703
|
"ai-search-scorecard",
|
|
30456
32704
|
"audit-page",
|
|
30457
32705
|
"audit-urls",
|
|
32706
|
+
"bing-webmaster-overview",
|
|
30458
32707
|
"cannibalisation",
|
|
30459
32708
|
"community-intent",
|
|
30460
32709
|
"compare-crawls",
|
|
@@ -30473,6 +32722,8 @@ const TELEMETRY_REPORTS = [
|
|
|
30473
32722
|
"index-monitor",
|
|
30474
32723
|
"index-watch",
|
|
30475
32724
|
"internal-links",
|
|
32725
|
+
"link-evidence",
|
|
32726
|
+
"server-log-analysis",
|
|
30476
32727
|
"link-recovery",
|
|
30477
32728
|
"crawl-history",
|
|
30478
32729
|
"crawler-rules",
|
|
@@ -30814,6 +33065,6 @@ function isoDate(daysAgo = 0) {
|
|
|
30814
33065
|
return date.toISOString().slice(0, 10);
|
|
30815
33066
|
}
|
|
30816
33067
|
//#endregion
|
|
30817
|
-
export { AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, AI_SEARCH_SCORECARD_METHODOLOGY_ID, AI_SEARCH_SCORECARD_METHODOLOGY_VERSION, AUDIT_PAGE_RULE_IDS, AUTOMATIC_BASELINE_SITE_RETENTION, AUTOMATIC_BASELINE_TOTAL_RETENTION, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CRAWL_REPORT_STORAGE_VERSION, CRAWL_RUN_RETENTION, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_DEFAULT_MIN_IMPRESSIONS, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DataForSeoProvider, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, INDEX_WATCH_ATTEMPT_RETENTION, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LINK_RECOVER_RUN_RETENTION, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, OKF_DEFAULT_CONCEPTS, OKF_MAX_CONCEPTS, REPORT_BASELINE_MAX_AGE_MS, REPORT_BASELINE_MAX_DEPTH, REPORT_BASELINE_MAX_PAGES, ResponseSizeLimitError, SCORECARD_CHECK_WEIGHTS, SCORECARD_STATUS_CREDIT, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SemrushProvider, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_NOTICE, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, affectedUrls, agentReadiness, aiPromptsForQuery, aiReadiness, aiReferralSourceForValue, aiReferralsReport, aiSearchScorecard, allocateIndexUrlsToProperties, analyticsConnectionsSchema, analyzeAiReferralRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, attachCrawlRecommendations, auditCrawlPages, auditCrawlRequests, auditLlmsTxt, auditPage, authStatus, boundedSitemapInventory, buildOkfBundle, buildPseoAuditReportFromRows, buildTelemetryPayload, cannibalReport, changeMeasurementCsvRows, checkDatabaseReadiness, classifyCommunityIntent, clearCache, clientProfileSchema, clusterPseoTemplates, clusterQueryRows, collectAgentDiscovery, combineRobotsValues, communityIntentReport, compareCrawlPages, compareCrawlReports, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlDiff, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportStore, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createContentGroup, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, decayClusterDrafts, decayComparisonRange, decayingReport, deleteChange, deleteClient, deleteContentGroup, deleteCrawlReport, deleteTokens, deriveBrandTerms, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensureSeoCliDirs, entityReadiness, estimateSerpTitleWidth, expectedCtrForPosition, explainOkfValidation, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fetchText, fileSize, finalGscDateRange, findOverlappingSearchUpdates, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, generateLlmsTxt, geoGaps, geoGapsReport, getAuthModeStatus, getCacheStats, getChange, getClient, getClientConfig, getContentGroup, getDb, getKeywordProvider, getPreviousRun, getPrivacySnapshot, getRepeatedLinkRecoverUrls, getRunPages, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleAnalyticsPropertyId, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoveragePlan, indexCoverageSignals, indexMonitor, indexWatch, inferPseoTemplate, inferTemplateMovement, initializeTelemetry, insertCrawlRun, insertLinkRecoverRun, inspectUrl, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isCompatibleTechnicalBaseline, isCurrentTechnicalBaseline, isEntityEvidenceSchemaType, isSiteIdentitySchemaType, isSkippableReportError, isTelemetryCi, isTelemetryReport, isoDate, isoWeek, latestCrawlReport, latestCrawlSummaries, latestFinalGscDate, latestIndexWatchSummary, latestLinkRecoverSummary, linkEntries, linkRecover, linkRecoverRecommendation, listChanges, listClients, listContentGroups, listCrawlReports, listGa4AccountSummaries, listGa4DataStreams, listRules, listSearchUpdates, listSites, loadCrawlReport, localCredentialsProvider, loginWithLoopback, makeRunId, matchGa4WebStreams, measureChange, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, monitoringStatus, monthlyReport, monthlyReportWorkflow, narrativeCsvFiles, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, noteCacheWrite, okfConceptLimit, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, parsePseoPath, performanceAudit, performanceReportIsCacheable, planDueIndexUrls, planIndexCoverageFromUrls, plural, propertyMatchesUrl, providerPreferenceSchema, pruneIndexWatchSnapshots, pruneLogs, pseoAuditOptions, pseoAuditReport, pseoCsvFiles, pseoIndexStatus, pseoPresentation, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, readBoundedResponseText, readConfig, readOauthClient, readTokens, recommendCrawlDiffItem, recommendationForRule, recordChange, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderPseoMarkdown, renderWorkflowMarkdown, reportNarrative, reportPresentation, requestHeaders, resolveSegmentRanges, resolveSeoCliPaths, resolveTechnicalBaseline, reviewObservations, roundedPosition, runDoctor, runGa4Report, safeError, saveClient, saveCrawlReport, secondPage, segmentComparisonRange, segmentImpact, segmentRangeDays, selectAffectedUrls, selectDueIndexUrls, seoErrorEnvelope, seoToAiQueryReport, sessionSourceInListFilter, setDefaultClient, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteSchema, sqliteCrawlReportStore, strikingDistance, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, technicalWatchWorkflow, telemetryErrorCategory, templateForUrl, toSeoError, tokenSchema, topCrawlRecommendations, topFixes, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, unavailableSegmentImpactReport, unicodeTokens, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateOkfFiles, verifyQueryContent, workflowPresentation, writeConfig, writeOauthClient, writeTokens, xRobotsDirectives };
|
|
33068
|
+
export { AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, AI_SEARCH_SCORECARD_METHODOLOGY_ID, AI_SEARCH_SCORECARD_METHODOLOGY_VERSION, AUDIT_PAGE_RULE_IDS, AUTOMATIC_BASELINE_SITE_RETENTION, AUTOMATIC_BASELINE_TOTAL_RETENTION, BING_API_KEY_ENV, BING_API_KEY_SECRET, BROWSER_USER_AGENT, BingWebmasterClient, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CRAWL_REPORT_STORAGE_VERSION, CRAWL_RUN_RETENTION, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_DEFAULT_MIN_IMPRESSIONS, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DataForSeoProvider, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, INDEXNOW_ENDPOINT, INDEXNOW_KEYS_SECRET, INDEXNOW_KEY_ENV, INDEXNOW_MAX_URLS, INDEX_WATCH_ATTEMPT_RETENTION, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LINK_RECOVER_RUN_RETENTION, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, OKF_DEFAULT_CONCEPTS, OKF_MAX_CONCEPTS, REPORT_BASELINE_MAX_AGE_MS, REPORT_BASELINE_MAX_DEPTH, REPORT_BASELINE_MAX_PAGES, ResponseSizeLimitError, SCORECARD_CHECK_WEIGHTS, SCORECARD_STATUS_CREDIT, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SemrushProvider, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_NOTICE, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, addStatus, affectedUrls, agentReadiness, aiPromptsForQuery, aiReadiness, aiReferralSourceForValue, aiReferralsReport, aiSearchScorecard, allocateIndexUrlsToProperties, analyticsConnectionsSchema, analyzeAiReferralRows, analyzeBingCrawl, analyzeBingDimensions, analyzeBingTraffic, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, attachCrawlRecommendations, auditCrawlPages, auditCrawlRequests, auditLlmsTxt, auditPage, authStatus, bingAnalysisDefaults, bingWebmasterOverview, bingWebmasterSiteUrl, boundedSitemapInventory, buildOkfBundle, buildPseoAuditReportFromRows, buildTelemetryPayload, cannibalReport, changeMeasurementCsvRows, checkDatabaseReadiness, classifyCommunityIntent, classifyCrawler, clearCache, clientProfileSchema, clusterPseoTemplates, clusterQueryRows, collectAgentDiscovery, collectBingLinkEvidence, combineRobotsValues, communityIntentReport, compareCrawlPages, compareCrawlReports, compareLinkEvidence, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlDiff, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportStore, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createBingWebmasterClient, createContentGroup, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createIndexNowKey, createIndexNowKeyRecord, createLogger, createPageRenderer, ctrUnderperformersReport, decayClusterDrafts, decayComparisonRange, decayingReport, deleteChange, deleteClient, deleteContentGroup, deleteCrawlReport, deleteProviderSecret, deleteTokens, deriveBrandTerms, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, emptyStatusBreakdown, ensureSeoCliDirs, entityReadiness, estimateSerpTitleWidth, expectedCtrForPosition, explainOkfValidation, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fetchText, fileSize, finalGscDateRange, findOverlappingSearchUpdates, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, generateLlmsTxt, geoGaps, geoGapsReport, getAuthModeStatus, getCacheStats, getChange, getClient, getClientConfig, getContentGroup, getDb, getKeywordProvider, getPreviousRun, getPrivacySnapshot, getRepeatedLinkRecoverUrls, getRunPages, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleAnalyticsPropertyId, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, importLinkEvidence, importServerLog, indexCoveragePlan, indexCoverageSignals, indexMonitor, indexWatch, inferPseoTemplate, inferTemplateMovement, initializeTelemetry, insertCrawlRun, insertLinkRecoverRun, inspectUrl, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isCompatibleTechnicalBaseline, isCurrentTechnicalBaseline, isEntityEvidenceSchemaType, isSiteIdentitySchemaType, isSkippableReportError, isTelemetryCi, isTelemetryReport, isoDate, isoWeek, latestCrawlReport, latestCrawlSummaries, latestFinalGscDate, latestIndexWatchSummary, latestLinkRecoverSummary, linkEntries, linkEvidenceKey, linkEvidenceReport, linkRecover, linkRecoverRecommendation, listChanges, listClients, listContentGroups, listCrawlReports, listGa4AccountSummaries, listGa4DataStreams, listIndexNowKeys, listRules, listSearchUpdates, listSites, loadCrawlReport, localCredentialsProvider, loginWithLoopback, makeRunId, matchGa4WebStreams, measureChange, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, monitoringStatus, monthlyReport, monthlyReportWorkflow, narrativeCsvFiles, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLinkEvidenceRow, normalizeLoadedCrawlReport, noteCacheWrite, okfConceptLimit, pageEvidenceRecommendation, pageOpportunitiesReport, parseCombinedLogLine, parseIndexNowSite, parseJsonLogLine, parseOlderThan, parsePseoPath, performanceAudit, performanceReportIsCacheable, planDueIndexUrls, planIndexCoverageFromUrls, plural, propertyMatchesUrl, providerPreferenceSchema, pruneIndexWatchSnapshots, pruneLogs, pseoAuditOptions, pseoAuditReport, pseoCsvFiles, pseoIndexStatus, pseoPresentation, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, readBoundedResponseText, readConfig, readOauthClient, readProviderSecret, readTokens, recommendCrawlDiffItem, recommendationForRule, recordChange, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, removeIndexNowKey, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderPseoMarkdown, renderReportHtml, renderWorkflowMarkdown, reportNarrative, reportPresentation, requestHeaders, resolveIndexNowKey, resolveSegmentRanges, resolveSeoCliPaths, resolveTechnicalBaseline, reviewObservations, roundedPosition, runDoctor, runGa4Report, safeError, saveClient, saveCrawlReport, saveIndexNowKey, secondPage, segmentComparisonRange, segmentImpact, segmentRangeDays, selectAffectedUrls, selectDueIndexUrls, seoErrorEnvelope, seoToAiQueryReport, serverLogReport, sessionSourceInListFilter, setClientBingSite, setDefaultClient, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteSchema, sqliteCrawlReportStore, strikingDistance, submitIndexNow, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, technicalWatchWorkflow, telemetryErrorCategory, templateForUrl, toSeoError, tokenSchema, topCrawlRecommendations, topFixes, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, unavailableSegmentImpactReport, unicodeTokens, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateIndexNowKey, validateIndexNowKeyRecord, validateOkfFiles, verifyIndexNowKey, verifyQueryContent, workflowPresentation, writeConfig, writeOauthClient, writeProviderSecret, writeTokens, xRobotsDirectives };
|
|
30818
33069
|
|
|
30819
33070
|
//# sourceMappingURL=index.js.map
|