seo 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +23 -5
- package/README.md +1 -1
- package/dist/cli.js +82 -82
- package/dist/cli.js.map +1 -1
- package/dist/{dist-B3L_4U8E.js → dist-BL_yf1fw.js} +1 -1
- package/dist/dist-DqfdgVVU.js +361 -0
- package/dist/dist-DqfdgVVU.js.map +1 -0
- package/dist/{esm-gP6ovS6b.js → esm-ZQwQYL0U.js} +2 -2
- package/dist/{esm-gP6ovS6b.js.map → esm-ZQwQYL0U.js.map} +1 -1
- package/dist/index.d.ts +368 -210
- package/dist/index.js +1953 -962
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +2083 -994
- package/dist/mcp.js.map +1 -1
- package/package.json +1 -1
- package/skills/seo/SKILL.md +1 -1
- package/dist/dist-BdZyUFiw.js +0 -356
- package/dist/dist-BdZyUFiw.js.map +0 -1
package/dist/mcp.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-
|
|
2
|
+
import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-ZQwQYL0U.js";
|
|
3
3
|
import { createHash, randomUUID, sign } from "node:crypto";
|
|
4
4
|
import { ZodOptional, z } from "zod";
|
|
5
|
-
import { arch, homedir, platform, totalmem } from "node:os";
|
|
5
|
+
import { arch, homedir, platform, tmpdir, totalmem } from "node:os";
|
|
6
6
|
import { dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
7
7
|
import "open";
|
|
8
8
|
import { chmodSync, createReadStream, existsSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
@@ -21,11 +21,12 @@ import { chromium } from "playwright-core";
|
|
|
21
21
|
import { gunzipSync } from "node:zlib";
|
|
22
22
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
23
23
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
24
|
-
import { open as open$1, readFile, stat } from "node:fs/promises";
|
|
24
|
+
import { mkdtemp, open as open$1, readFile, rm, stat } from "node:fs/promises";
|
|
25
25
|
import { TextDecoder as TextDecoder$1, promisify } from "node:util";
|
|
26
26
|
import { execFile } from "node:child_process";
|
|
27
27
|
import "pino";
|
|
28
28
|
import "pino-pretty";
|
|
29
|
+
import { request } from "node:https";
|
|
29
30
|
import * as z3rt from "zod/v3";
|
|
30
31
|
import { ZodFirstPartyTypeKind } from "zod/v3";
|
|
31
32
|
import * as z4mini from "zod/v4-mini";
|
|
@@ -13634,12 +13635,12 @@ var SeoError = class extends Error {
|
|
|
13634
13635
|
this.retryable = errorMetadata[code].retryable;
|
|
13635
13636
|
}
|
|
13636
13637
|
};
|
|
13637
|
-
function errorMessage$
|
|
13638
|
+
function errorMessage$1(error) {
|
|
13638
13639
|
return error instanceof Error ? error.message : String(error);
|
|
13639
13640
|
}
|
|
13640
13641
|
function toSeoError(error) {
|
|
13641
13642
|
if (error instanceof SeoError) return error;
|
|
13642
|
-
const message = errorMessage$
|
|
13643
|
+
const message = errorMessage$1(error);
|
|
13643
13644
|
if (/not logged in/i.test(message)) return new SeoError("AUTH_REQUIRED", message);
|
|
13644
13645
|
if (/oauth client config.*missing|shared seo google app/i.test(message)) return new SeoError("AUTH_CONFIG_REQUIRED", message);
|
|
13645
13646
|
if (/invalid_grant|refresh token.*no longer valid/i.test(message)) return new SeoError("AUTH_EXPIRED", message);
|
|
@@ -13877,7 +13878,7 @@ async function runAnalyticsReport(gfetch, propertyId, body, options) {
|
|
|
13877
13878
|
//#endregion
|
|
13878
13879
|
//#region packages/core/dist/bing/api.js
|
|
13879
13880
|
const BASE = "https://ssl.bing.com/webmaster/api.svc/json";
|
|
13880
|
-
const MAX_BYTES$
|
|
13881
|
+
const MAX_BYTES$3 = 8e6;
|
|
13881
13882
|
function bingCall(auth, options) {
|
|
13882
13883
|
return async (method, params = {}) => {
|
|
13883
13884
|
const url = new URL(`${BASE}/${method}`);
|
|
@@ -13902,7 +13903,7 @@ function bingCall(auth, options) {
|
|
|
13902
13903
|
if (response.status === 429 || /throttle/i.test(detail)) throw new SeoError("RATE_LIMITED", "Bing Webmaster is limiting requests. Try again in a few minutes.");
|
|
13903
13904
|
throw new SeoError("OPTIONAL_PROVIDER_UNAVAILABLE", `Bing Webmaster answered ${response.status}. ${detail}`.trim());
|
|
13904
13905
|
}
|
|
13905
|
-
return (await readJson(response, "Bing Webmaster", { maxBytes: MAX_BYTES$
|
|
13906
|
+
return (await readJson(response, "Bing Webmaster", { maxBytes: MAX_BYTES$3 })).d;
|
|
13906
13907
|
};
|
|
13907
13908
|
}
|
|
13908
13909
|
function day(value) {
|
|
@@ -14100,6 +14101,262 @@ function decayComparisonRange(current, comparison = "previous-period") {
|
|
|
14100
14101
|
};
|
|
14101
14102
|
}
|
|
14102
14103
|
//#endregion
|
|
14104
|
+
//#region packages/core/dist/updates/ranking-history.js
|
|
14105
|
+
const RANKING_HISTORY = [
|
|
14106
|
+
{
|
|
14107
|
+
id: "XhUDXP7A67iHCD2kmbVu",
|
|
14108
|
+
begin: "2026-09-24T16:15:00+00:00",
|
|
14109
|
+
end: "2026-10-08T08:00:00+00:00",
|
|
14110
|
+
external_desc: "September 2026 spam update"
|
|
14111
|
+
},
|
|
14112
|
+
{
|
|
14113
|
+
id: "LEubPCm2octf2uMqCFKE",
|
|
14114
|
+
begin: "2026-08-18T16:27:00+00:00",
|
|
14115
|
+
end: "2026-08-21T08:49:00+00:00",
|
|
14116
|
+
external_desc: "August 2026 spam update"
|
|
14117
|
+
},
|
|
14118
|
+
{
|
|
14119
|
+
id: "YUX1peHev5a4fkxLDiUQ",
|
|
14120
|
+
begin: "2026-06-24T16:00:00+00:00",
|
|
14121
|
+
end: "2026-06-26T17:00:00+00:00",
|
|
14122
|
+
external_desc: "June 2026 spam update"
|
|
14123
|
+
},
|
|
14124
|
+
{
|
|
14125
|
+
id: "wdAXJk6LRRihEjpzEeWE",
|
|
14126
|
+
begin: "2026-05-21T15:40:00+00:00",
|
|
14127
|
+
end: "2026-06-02T12:40:00+00:00",
|
|
14128
|
+
external_desc: "May 2026 core update"
|
|
14129
|
+
},
|
|
14130
|
+
{
|
|
14131
|
+
id: "7eTbAa2jWdToLkraZj5y",
|
|
14132
|
+
begin: "2026-03-27T09:00:00+00:00",
|
|
14133
|
+
end: "2026-04-08T13:00:00+00:00",
|
|
14134
|
+
external_desc: "March 2026 core update"
|
|
14135
|
+
},
|
|
14136
|
+
{
|
|
14137
|
+
id: "VbnSXAH4SmEcxPtx4YSD",
|
|
14138
|
+
begin: "2026-03-24T19:00:00+00:00",
|
|
14139
|
+
end: "2026-03-25T14:30:00+00:00",
|
|
14140
|
+
external_desc: "March 2026 spam update"
|
|
14141
|
+
},
|
|
14142
|
+
{
|
|
14143
|
+
id: "mYbNTqV1ytDc2fA8hUz4",
|
|
14144
|
+
begin: "2026-02-05T17:00:00+00:00",
|
|
14145
|
+
end: "2026-02-27T10:00:00+00:00",
|
|
14146
|
+
external_desc: "February 2026 Discover update"
|
|
14147
|
+
},
|
|
14148
|
+
{
|
|
14149
|
+
id: "DsirqJ1gpPRgVQeccPRv",
|
|
14150
|
+
begin: "2025-12-11T17:25:00+00:00",
|
|
14151
|
+
end: "2025-12-29T19:00:00+00:00",
|
|
14152
|
+
external_desc: "December 2025 core update"
|
|
14153
|
+
},
|
|
14154
|
+
{
|
|
14155
|
+
id: "a7Aainy6E9rZsmfq82xt",
|
|
14156
|
+
begin: "2025-08-26T16:00:00+00:00",
|
|
14157
|
+
end: "2025-09-22T07:00:00+00:00",
|
|
14158
|
+
external_desc: "August 2025 spam update"
|
|
14159
|
+
},
|
|
14160
|
+
{
|
|
14161
|
+
id: "riq1AuqETW46NfBCe5NT",
|
|
14162
|
+
begin: "2025-06-30T14:30:00+00:00",
|
|
14163
|
+
end: "2025-07-17T08:00:00+00:00",
|
|
14164
|
+
external_desc: "June 2025 core update"
|
|
14165
|
+
},
|
|
14166
|
+
{
|
|
14167
|
+
id: "zpmwuSwifjDjfrVdaZUx",
|
|
14168
|
+
begin: "2025-03-13T16:00:00+00:00",
|
|
14169
|
+
end: "2025-03-27T12:35:00+00:00",
|
|
14170
|
+
external_desc: "March 2025 core update"
|
|
14171
|
+
},
|
|
14172
|
+
{
|
|
14173
|
+
id: "UUq2WSouY7PhSm8zvtD1",
|
|
14174
|
+
begin: "2024-12-19T17:00:00+00:00",
|
|
14175
|
+
end: "2024-12-26T19:00:00+00:00",
|
|
14176
|
+
external_desc: "December 2024 spam update"
|
|
14177
|
+
},
|
|
14178
|
+
{
|
|
14179
|
+
id: "V9nDKuo6nWKh2ThBALgA",
|
|
14180
|
+
begin: "2024-12-12T15:30:00+00:00",
|
|
14181
|
+
end: "2024-12-18T19:00:00+00:00",
|
|
14182
|
+
external_desc: "December 2024 core update"
|
|
14183
|
+
},
|
|
14184
|
+
{
|
|
14185
|
+
id: "G7rp11wqTaTGn6JjiPF3",
|
|
14186
|
+
begin: "2024-11-11T19:00:00+00:00",
|
|
14187
|
+
end: "2024-12-05T08:00:00+00:00",
|
|
14188
|
+
external_desc: "November 2024 core update"
|
|
14189
|
+
},
|
|
14190
|
+
{
|
|
14191
|
+
id: "42XCLGtpuCZkzqe4dc6S",
|
|
14192
|
+
begin: "2024-08-16T00:15:00+00:00",
|
|
14193
|
+
end: "2024-08-20T11:30:00+00:00",
|
|
14194
|
+
external_desc: "Ranking is experiencing an ongoing issue."
|
|
14195
|
+
},
|
|
14196
|
+
{
|
|
14197
|
+
id: "gVx6b2o78zke7GrMidGy",
|
|
14198
|
+
begin: "2024-08-15T14:58:00+00:00",
|
|
14199
|
+
end: "2024-09-03T19:00:00+00:00",
|
|
14200
|
+
external_desc: "August 2024 core update"
|
|
14201
|
+
},
|
|
14202
|
+
{
|
|
14203
|
+
id: "QdUeCQx3LRVbzQ3E7FiD",
|
|
14204
|
+
begin: "2024-06-20T15:12:00+00:00",
|
|
14205
|
+
end: "2024-06-27T16:11:00+00:00",
|
|
14206
|
+
external_desc: "June 2024 spam update"
|
|
14207
|
+
},
|
|
14208
|
+
{
|
|
14209
|
+
id: "iXz2PJfodvyjaVUeqxZE",
|
|
14210
|
+
begin: "2024-03-05T17:03:00+00:00",
|
|
14211
|
+
end: "2024-03-20T14:09:00+00:00",
|
|
14212
|
+
external_desc: "March 2024 spam update"
|
|
14213
|
+
},
|
|
14214
|
+
{
|
|
14215
|
+
id: "1jW2F89qC2NxJBWGiKxE",
|
|
14216
|
+
begin: "2024-03-05T17:02:00+00:00",
|
|
14217
|
+
end: "2024-04-19T17:00:00+00:00",
|
|
14218
|
+
external_desc: "March 2024 core update"
|
|
14219
|
+
},
|
|
14220
|
+
{
|
|
14221
|
+
id: "VqRTcmyQCwfpuYeaVNfn",
|
|
14222
|
+
begin: "2023-11-08T16:58:00+00:00",
|
|
14223
|
+
end: "2023-12-07T17:00:00+00:00",
|
|
14224
|
+
external_desc: "November 2023 reviews update"
|
|
14225
|
+
},
|
|
14226
|
+
{
|
|
14227
|
+
id: "WtY1CQgfqrr2cjxKbyUk",
|
|
14228
|
+
begin: "2023-11-02T19:09:00+00:00",
|
|
14229
|
+
end: "2023-11-28T16:30:00+00:00",
|
|
14230
|
+
external_desc: "November 2023 core update"
|
|
14231
|
+
},
|
|
14232
|
+
{
|
|
14233
|
+
id: "VKyoS53ULWzSp8BoDxqk",
|
|
14234
|
+
begin: "2023-10-05T16:00:00+00:00",
|
|
14235
|
+
end: "2023-10-19T15:00:00+00:00",
|
|
14236
|
+
external_desc: "October 2023 core update"
|
|
14237
|
+
},
|
|
14238
|
+
{
|
|
14239
|
+
id: "fCUAy6TvbDMkkLAGcwkj",
|
|
14240
|
+
begin: "2023-10-05T07:00:00+00:00",
|
|
14241
|
+
end: "2023-10-31T07:00:00+00:00",
|
|
14242
|
+
external_desc: "Ranking is experiencing an ongoing issue"
|
|
14243
|
+
},
|
|
14244
|
+
{
|
|
14245
|
+
id: "NzcEhGMDhbQEdXCS35xL",
|
|
14246
|
+
begin: "2023-10-04T16:51:00+00:00",
|
|
14247
|
+
end: "2023-10-20T05:15:00+00:00",
|
|
14248
|
+
external_desc: "October 2023 spam update"
|
|
14249
|
+
},
|
|
14250
|
+
{
|
|
14251
|
+
id: "53diuQvcEsgzqXTPBb8p",
|
|
14252
|
+
begin: "2023-09-14T20:19:00+00:00",
|
|
14253
|
+
end: "2023-09-28T07:07:00+00:00",
|
|
14254
|
+
external_desc: "September 2023 helpful content update"
|
|
14255
|
+
},
|
|
14256
|
+
{
|
|
14257
|
+
id: "nBtYtBeex4GYBbdDS2LX",
|
|
14258
|
+
begin: "2023-08-22T17:30:00+00:00",
|
|
14259
|
+
end: "2023-09-07T20:03:00+00:00",
|
|
14260
|
+
external_desc: "August 2023 core update"
|
|
14261
|
+
},
|
|
14262
|
+
{
|
|
14263
|
+
id: "5XRfC46rorevFt8yN8iR",
|
|
14264
|
+
begin: "2023-04-12T16:00:00+00:00",
|
|
14265
|
+
end: "2023-04-25T18:24:00+00:00",
|
|
14266
|
+
external_desc: "April 2023 reviews update"
|
|
14267
|
+
},
|
|
14268
|
+
{
|
|
14269
|
+
id: "Cou8Tr74r7EXNthuEsaG",
|
|
14270
|
+
begin: "2023-03-15T07:00:00+00:00",
|
|
14271
|
+
end: "2023-03-28T14:26:00+00:00",
|
|
14272
|
+
external_desc: "March 2023 core update"
|
|
14273
|
+
},
|
|
14274
|
+
{
|
|
14275
|
+
id: "NDjZ11MEUJb5i62hy5aN",
|
|
14276
|
+
begin: "2023-02-21T08:00:00+00:00",
|
|
14277
|
+
end: "2023-03-07T08:00:00+00:00",
|
|
14278
|
+
external_desc: "February 2023 product reviews update"
|
|
14279
|
+
},
|
|
14280
|
+
{
|
|
14281
|
+
id: "CxXUEK28vCKqkNocPhKR",
|
|
14282
|
+
begin: "2022-12-14T08:00:00+00:00",
|
|
14283
|
+
end: "2023-01-12T08:00:00+00:00",
|
|
14284
|
+
external_desc: "December 2022 link spam update"
|
|
14285
|
+
},
|
|
14286
|
+
{
|
|
14287
|
+
id: "XMdX5PmbMvKW4fv3CRvn",
|
|
14288
|
+
begin: "2022-12-05T08:00:00+00:00",
|
|
14289
|
+
end: "2023-01-12T08:00:00+00:00",
|
|
14290
|
+
external_desc: "December 2022 helpful content update"
|
|
14291
|
+
},
|
|
14292
|
+
{
|
|
14293
|
+
id: "SAeYuxVSo39HHotQNXHr",
|
|
14294
|
+
begin: "2022-10-19T07:00:00+00:00",
|
|
14295
|
+
end: "2022-10-21T07:00:00+00:00",
|
|
14296
|
+
external_desc: "October 2022 spam update"
|
|
14297
|
+
},
|
|
14298
|
+
{
|
|
14299
|
+
id: "ZYwuBXDhkwaoop2V5epM",
|
|
14300
|
+
begin: "2022-09-20T07:00:00+00:00",
|
|
14301
|
+
end: "2022-09-26T07:00:00+00:00",
|
|
14302
|
+
external_desc: "September 2022 product reviews update"
|
|
14303
|
+
},
|
|
14304
|
+
{
|
|
14305
|
+
id: "Y2Cg5k2xLJvfNqR58kEt",
|
|
14306
|
+
begin: "2022-09-12T07:00:00+00:00",
|
|
14307
|
+
end: "2022-09-26T07:00:00+00:00",
|
|
14308
|
+
external_desc: "September 2022 core update"
|
|
14309
|
+
},
|
|
14310
|
+
{
|
|
14311
|
+
id: "sbKc4YSptrSxcUZFDKwU",
|
|
14312
|
+
begin: "2022-08-25T07:00:00+00:00",
|
|
14313
|
+
end: "2022-09-09T07:00:00+00:00",
|
|
14314
|
+
external_desc: "August 2022 helpful content update"
|
|
14315
|
+
},
|
|
14316
|
+
{
|
|
14317
|
+
id: "Xiid524od7fpCak86sNS",
|
|
14318
|
+
begin: "2022-07-27T07:00:00+00:00",
|
|
14319
|
+
end: "2022-08-02T07:00:00+00:00",
|
|
14320
|
+
external_desc: "July 2022 product reviews update"
|
|
14321
|
+
},
|
|
14322
|
+
{
|
|
14323
|
+
id: "GHwLLtDS64a7nDng8xhD",
|
|
14324
|
+
begin: "2022-05-25T07:00:00+00:00",
|
|
14325
|
+
end: "2022-06-09T07:00:00+00:00",
|
|
14326
|
+
external_desc: "May 2022 core update"
|
|
14327
|
+
},
|
|
14328
|
+
{
|
|
14329
|
+
id: "iYWoNu4QXRsQUrzxrbLv",
|
|
14330
|
+
begin: "2022-03-23T07:00:00+00:00",
|
|
14331
|
+
end: "2022-04-06T07:00:00+00:00",
|
|
14332
|
+
external_desc: "March 2022 product reviews update"
|
|
14333
|
+
},
|
|
14334
|
+
{
|
|
14335
|
+
id: "zDCQR5mxLU3ZrQeQvwqN",
|
|
14336
|
+
begin: "2022-02-22T08:00:00+00:00",
|
|
14337
|
+
end: "2022-03-03T08:00:00+00:00",
|
|
14338
|
+
external_desc: "Page experience update for desktop"
|
|
14339
|
+
},
|
|
14340
|
+
{
|
|
14341
|
+
id: "hgwP7ei5H1uKbYSeZrBZ",
|
|
14342
|
+
begin: "2021-12-01T08:00:00+00:00",
|
|
14343
|
+
end: "2021-12-21T08:00:00+00:00",
|
|
14344
|
+
external_desc: "December 2021 product reviews update"
|
|
14345
|
+
},
|
|
14346
|
+
{
|
|
14347
|
+
id: "FncPfR7zRSZ3diemah4o",
|
|
14348
|
+
begin: "2021-11-17T08:00:00+00:00",
|
|
14349
|
+
end: "2021-11-30T08:00:00+00:00",
|
|
14350
|
+
external_desc: "November 2021 core update"
|
|
14351
|
+
},
|
|
14352
|
+
{
|
|
14353
|
+
id: "AFdXceQVsh1GEfKFvHyD",
|
|
14354
|
+
begin: "2021-11-03T07:00:00+00:00",
|
|
14355
|
+
end: "2021-11-11T08:00:00+00:00",
|
|
14356
|
+
external_desc: "November 2021 spam update"
|
|
14357
|
+
}
|
|
14358
|
+
];
|
|
14359
|
+
//#endregion
|
|
14103
14360
|
//#region packages/core/dist/updates/search-updates.js
|
|
14104
14361
|
function classifyUpdate(name, product) {
|
|
14105
14362
|
const lower = `${name} ${product}`.toLowerCase();
|
|
@@ -14127,6 +14384,18 @@ function searchUpdatesFromIncidents(incidents, input = {}) {
|
|
|
14127
14384
|
};
|
|
14128
14385
|
}).slice(0, input.limit ?? 25);
|
|
14129
14386
|
}
|
|
14387
|
+
function withRankingHistory(live) {
|
|
14388
|
+
const byId = /* @__PURE__ */ new Map();
|
|
14389
|
+
for (const saved of RANKING_HISTORY) byId.set(saved.id, {
|
|
14390
|
+
...saved,
|
|
14391
|
+
service_name: "Ranking",
|
|
14392
|
+
status_impact: "SERVICE_INFORMATION",
|
|
14393
|
+
severity: "low",
|
|
14394
|
+
uri: `incidents/${saved.id}`
|
|
14395
|
+
});
|
|
14396
|
+
for (const incident of live) byId.set(incident.id, incident);
|
|
14397
|
+
return [...byId.values()].sort((left, right) => Date.parse(right.begin) - Date.parse(left.begin) || (left.id < right.id ? -1 : 1));
|
|
14398
|
+
}
|
|
14130
14399
|
function findOverlappingSearchUpdates(input) {
|
|
14131
14400
|
const paddingMs = (input.paddingDays ?? 3) * 864e5;
|
|
14132
14401
|
const windowStart = new Date(input.startDate).getTime() - paddingMs;
|
|
@@ -14225,9 +14494,9 @@ async function googleUpdates(window) {
|
|
|
14225
14494
|
const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? readJson(response, "Google Search Status", { maxBytes: MAX_STATUS_BYTES }) : null).catch(() => null);
|
|
14226
14495
|
if (!Array.isArray(incidents)) return "Google Search Status could not be read.";
|
|
14227
14496
|
return findOverlappingSearchUpdates({
|
|
14228
|
-
updates: searchUpdatesFromIncidents(incidents, {
|
|
14497
|
+
updates: searchUpdatesFromIncidents(withRankingHistory(incidents), {
|
|
14229
14498
|
product: "Ranking",
|
|
14230
|
-
limit:
|
|
14499
|
+
limit: Number.POSITIVE_INFINITY
|
|
14231
14500
|
}),
|
|
14232
14501
|
startDate: window.start,
|
|
14233
14502
|
endDate: window.end,
|
|
@@ -14332,7 +14601,7 @@ const METRIC_COLUMNS = [
|
|
|
14332
14601
|
"score",
|
|
14333
14602
|
"priority.score"
|
|
14334
14603
|
];
|
|
14335
|
-
const SUMMARY_ROWS = 10;
|
|
14604
|
+
const SUMMARY_ROWS$1 = 10;
|
|
14336
14605
|
const CELL_MAX = 80;
|
|
14337
14606
|
const isObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
14338
14607
|
const isPrimitive = (value) => value === null || [
|
|
@@ -14491,7 +14760,7 @@ function presentReport(input) {
|
|
|
14491
14760
|
}
|
|
14492
14761
|
};
|
|
14493
14762
|
}
|
|
14494
|
-
const shown = flat.slice(0, Math.min(input.limit, SUMMARY_ROWS));
|
|
14763
|
+
const shown = flat.slice(0, Math.min(input.limit, SUMMARY_ROWS$1));
|
|
14495
14764
|
const columns = columnsFor(flat, 8);
|
|
14496
14765
|
return {
|
|
14497
14766
|
text: [
|
|
@@ -16495,10 +16764,12 @@ function initDb(database, isNewDatabase) {
|
|
|
16495
16764
|
database.exec("DROP TABLE IF EXISTS provider_spend_notices");
|
|
16496
16765
|
ensureColumn(database, "http_cache", "metadata_json", "TEXT");
|
|
16497
16766
|
}).immediate();
|
|
16498
|
-
|
|
16499
|
-
|
|
16500
|
-
|
|
16501
|
-
|
|
16767
|
+
try {
|
|
16768
|
+
runCacheMaintenance(database, {
|
|
16769
|
+
compact: true,
|
|
16770
|
+
allowFullVacuum: true
|
|
16771
|
+
});
|
|
16772
|
+
} catch {}
|
|
16502
16773
|
}
|
|
16503
16774
|
function getDb() {
|
|
16504
16775
|
if (db) return db;
|
|
@@ -16509,12 +16780,41 @@ function getDb() {
|
|
|
16509
16780
|
mode: 448
|
|
16510
16781
|
});
|
|
16511
16782
|
ensurePrivateDatabaseFiles(path);
|
|
16512
|
-
|
|
16513
|
-
|
|
16514
|
-
|
|
16515
|
-
|
|
16783
|
+
const opened = new SqliteDatabase(path);
|
|
16784
|
+
try {
|
|
16785
|
+
ensurePrivateDatabaseFiles(path);
|
|
16786
|
+
initDb(opened, isNewDatabase);
|
|
16787
|
+
ensurePrivateDatabaseFiles(path);
|
|
16788
|
+
} catch (error) {
|
|
16789
|
+
opened.close();
|
|
16790
|
+
throw error;
|
|
16791
|
+
}
|
|
16792
|
+
db = opened;
|
|
16516
16793
|
return db;
|
|
16517
16794
|
}
|
|
16795
|
+
let memoryDb;
|
|
16796
|
+
function memoryCache() {
|
|
16797
|
+
if (!memoryDb) {
|
|
16798
|
+
memoryDb = new SqliteDatabase(":memory:");
|
|
16799
|
+
memoryDb.exec(CREATE_SQL);
|
|
16800
|
+
}
|
|
16801
|
+
return memoryDb;
|
|
16802
|
+
}
|
|
16803
|
+
function cacheDb() {
|
|
16804
|
+
if (memoryDb) return memoryDb;
|
|
16805
|
+
try {
|
|
16806
|
+
return getDb();
|
|
16807
|
+
} catch {
|
|
16808
|
+
return memoryCache();
|
|
16809
|
+
}
|
|
16810
|
+
}
|
|
16811
|
+
function writeToCache(write) {
|
|
16812
|
+
try {
|
|
16813
|
+
write(cacheDb());
|
|
16814
|
+
} catch {
|
|
16815
|
+
memoryCache();
|
|
16816
|
+
}
|
|
16817
|
+
}
|
|
16518
16818
|
function noteCacheWrite(sizeBytes = 0, database = getDb()) {
|
|
16519
16819
|
const counter = cacheMaintenanceCounters.get(database) ?? {
|
|
16520
16820
|
writes: 0,
|
|
@@ -16542,7 +16842,7 @@ function ga4RequestCanUseCache(body) {
|
|
|
16542
16842
|
}
|
|
16543
16843
|
async function runGa4Report(propertyId, body, opts = {}) {
|
|
16544
16844
|
const { client, identity } = await createGoogleAccessTokenClient(opts.accountEmail);
|
|
16545
|
-
const db =
|
|
16845
|
+
const db = cacheDb();
|
|
16546
16846
|
const queryHash = hashKey([
|
|
16547
16847
|
propertyId,
|
|
16548
16848
|
identity,
|
|
@@ -16555,10 +16855,12 @@ async function runGa4Report(propertyId, body, opts = {}) {
|
|
|
16555
16855
|
if (cacheable) {
|
|
16556
16856
|
const requestJson = JSON.stringify(body);
|
|
16557
16857
|
const responseJson = JSON.stringify(result);
|
|
16558
|
-
|
|
16559
|
-
|
|
16560
|
-
|
|
16561
|
-
|
|
16858
|
+
writeToCache((database) => {
|
|
16859
|
+
database.prepare(`INSERT OR REPLACE INTO ga4_cache
|
|
16860
|
+
(property_id, query_hash, request_json, response_json, row_count, fetched_at, expires_at)
|
|
16861
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`).run(propertyId, queryHash, requestJson, responseJson, result.rowCount ?? result.rows?.length ?? 0, Date.now(), Date.now() + 864e5);
|
|
16862
|
+
noteCacheWrite(Buffer.byteLength(requestJson) + Buffer.byteLength(responseJson), database);
|
|
16863
|
+
});
|
|
16562
16864
|
}
|
|
16563
16865
|
return result;
|
|
16564
16866
|
}
|
|
@@ -16849,11 +17151,11 @@ function analyzeAiReferralRows(input) {
|
|
|
16849
17151
|
}
|
|
16850
17152
|
//#endregion
|
|
16851
17153
|
//#region packages/core/dist/analyze/ai-referrals-query.js
|
|
16852
|
-
const PAGE_SIZE$
|
|
17154
|
+
const PAGE_SIZE$3 = 25e3;
|
|
16853
17155
|
async function fetchAiReferralRows(input) {
|
|
16854
|
-
const pageSize = input.pageSize ?? PAGE_SIZE$
|
|
17156
|
+
const pageSize = input.pageSize ?? PAGE_SIZE$3;
|
|
16855
17157
|
if (!Number.isInteger(input.maxRows) || input.maxRows < 1) throw new Error("maxRows must be a positive whole number.");
|
|
16856
|
-
if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > PAGE_SIZE$
|
|
17158
|
+
if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > PAGE_SIZE$3) throw new Error(`pageSize must be a whole number between 1 and ${PAGE_SIZE$3}.`);
|
|
16857
17159
|
const rows = [];
|
|
16858
17160
|
const warnings = /* @__PURE__ */ new Set();
|
|
16859
17161
|
const retainedKeys = /* @__PURE__ */ new Set();
|
|
@@ -18930,7 +19232,7 @@ async function fetchRobotsSource(origin, refresh, options) {
|
|
|
18930
19232
|
const robotsUrl = new URL("/robots.txt", origin).toString();
|
|
18931
19233
|
const key = hashKey(["robots", robotsUrl]);
|
|
18932
19234
|
if (!refresh) {
|
|
18933
|
-
const cached =
|
|
19235
|
+
const cached = cacheDb().prepare("SELECT status, body_blob, expires_at FROM http_cache WHERE url_hash = ?").get(key);
|
|
18934
19236
|
if (cached?.body_blob && cached.status !== void 0 && cacheableStatus(cached.status) && cached.expires_at && cached.expires_at > Date.now()) return {
|
|
18935
19237
|
robotsUrl,
|
|
18936
19238
|
status: cached.status,
|
|
@@ -18949,12 +19251,12 @@ async function fetchRobotsSource(origin, refresh, options) {
|
|
|
18949
19251
|
signal: controller.signal
|
|
18950
19252
|
});
|
|
18951
19253
|
const text = await readBoundedResponseText(response, MAX_ROBOTS_RESPONSE_BYTES, "robots.txt response");
|
|
18952
|
-
if (options.writeCache !== false && cacheableStatus(response.status)) {
|
|
18953
|
-
|
|
18954
|
-
|
|
18955
|
-
|
|
18956
|
-
noteCacheWrite(Buffer.byteLength(text));
|
|
18957
|
-
}
|
|
19254
|
+
if (options.writeCache !== false && cacheableStatus(response.status)) writeToCache((database) => {
|
|
19255
|
+
database.prepare(`INSERT OR REPLACE INTO http_cache
|
|
19256
|
+
(url_hash, url, status, headers_json, body_blob, etag, fetched_at, expires_at)
|
|
19257
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(key, robotsUrl, response.status, "{}", Buffer.from(text), null, Date.now(), Date.now() + 864e5);
|
|
19258
|
+
noteCacheWrite(Buffer.byteLength(text), database);
|
|
19259
|
+
});
|
|
18958
19260
|
return {
|
|
18959
19261
|
robotsUrl,
|
|
18960
19262
|
status: response.status,
|
|
@@ -18992,6 +19294,12 @@ function createRobotsSession(input = {}) {
|
|
|
18992
19294
|
};
|
|
18993
19295
|
return {
|
|
18994
19296
|
resolve: async (origin, targetUrl) => resultFromSource(await sourceFor(origin), targetUrl),
|
|
19297
|
+
crawlDelay: async (origin) => {
|
|
19298
|
+
const source = await sourceFor(origin);
|
|
19299
|
+
if (source.status === void 0 || source.status >= 300 || !source.text) return;
|
|
19300
|
+
const delay = parseRobots$1(source.robotsUrl, source.text).getCrawlDelay(SEO_CRAWLER_USER_AGENT);
|
|
19301
|
+
return delay !== void 0 && Number.isFinite(delay) && delay > 0 ? delay : void 0;
|
|
19302
|
+
},
|
|
18995
19303
|
sitemapUrls: async (origin) => {
|
|
18996
19304
|
const source = await sourceFor(origin);
|
|
18997
19305
|
if (source.status === void 0 || source.status >= 300 || !source.text) return [];
|
|
@@ -19055,7 +19363,6 @@ function decodePageFetchCacheEvidence(value) {
|
|
|
19055
19363
|
}
|
|
19056
19364
|
async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, respectRobots = false, writeCache = true) {
|
|
19057
19365
|
const startedAt = Date.now();
|
|
19058
|
-
const db = getDb();
|
|
19059
19366
|
const key = hashKey(["page", url]);
|
|
19060
19367
|
const host = new URL(url).host;
|
|
19061
19368
|
let robots = respectRobots ? await fetchRobots(new URL(url).origin, url, refresh, {
|
|
@@ -19063,7 +19370,7 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
|
|
|
19063
19370
|
signal
|
|
19064
19371
|
}) : void 0;
|
|
19065
19372
|
if (robots && robots.allowed !== true) throw new RobotsAccessError(robots.allowed === false ? "robots-disallowed" : "robots-deferred", robots);
|
|
19066
|
-
const cached =
|
|
19373
|
+
const cached = cacheDb().prepare("SELECT status, headers_json, body_blob, metadata_json, expires_at FROM http_cache WHERE url_hash = ? AND expires_at > ?").get(key, Date.now());
|
|
19067
19374
|
if (!refresh && cached) {
|
|
19068
19375
|
const evidence = decodePageFetchCacheEvidence(cached.metadata_json);
|
|
19069
19376
|
return {
|
|
@@ -19165,12 +19472,12 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
|
|
|
19165
19472
|
warnings: [...robots.allowed === null ? [`robots.txt availability is unknown${robots.status ? ` after HTTP ${robots.status}` : ""}; this fetch does not prove crawler access is allowed.`] : [], ...accessBlock ? [`${accessBlock.guidance.summary} Identify the crawler as ${accessBlock.crawler.userAgent}.`] : []],
|
|
19166
19473
|
robotsTxt: pageRobotsEvidence(robots)
|
|
19167
19474
|
};
|
|
19168
|
-
if (!accessBlock && writeCache) {
|
|
19169
|
-
|
|
19170
|
-
|
|
19171
|
-
|
|
19172
|
-
noteCacheWrite(Buffer.byteLength(html));
|
|
19173
|
-
}
|
|
19475
|
+
if (!accessBlock && writeCache) writeToCache((database) => {
|
|
19476
|
+
database.prepare(`INSERT OR REPLACE INTO http_cache
|
|
19477
|
+
(url_hash, url, status, headers_json, body_blob, metadata_json, etag, fetched_at, expires_at)
|
|
19478
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(key, url, response.response.status, JSON.stringify(headerMap), Buffer.from(html), encodePageFetchCacheEvidence(result), response.response.headers.get("etag"), Date.now(), Date.now() + 36e5);
|
|
19479
|
+
noteCacheWrite(Buffer.byteLength(html), database);
|
|
19480
|
+
});
|
|
19174
19481
|
return result;
|
|
19175
19482
|
}
|
|
19176
19483
|
//#endregion
|
|
@@ -19952,7 +20259,7 @@ function finalizeUrlInspectionQuota(input) {
|
|
|
19952
20259
|
function credentialKey(input) {
|
|
19953
20260
|
return createHash("sha256").update(`${input.clientId}\n${input.accountEmail.toLowerCase()}`).digest("base64url");
|
|
19954
20261
|
}
|
|
19955
|
-
function retryAfter(response, now) {
|
|
20262
|
+
function retryAfter$1(response, now) {
|
|
19956
20263
|
const value = response.headers.get("retry-after");
|
|
19957
20264
|
if (!value) return void 0;
|
|
19958
20265
|
const seconds = Number(value);
|
|
@@ -19995,7 +20302,7 @@ async function inspectUrl(input) {
|
|
|
19995
20302
|
}
|
|
19996
20303
|
if (response.status === 429) {
|
|
19997
20304
|
const now = /* @__PURE__ */ new Date();
|
|
19998
|
-
const blockedUntil = retryAfter(response, now);
|
|
20305
|
+
const blockedUntil = retryAfter$1(response, now);
|
|
19999
20306
|
finalizeUrlInspectionQuota({
|
|
20000
20307
|
reservation,
|
|
20001
20308
|
outcome: "exhausted",
|
|
@@ -20028,7 +20335,7 @@ function resolveSearchDateWindow(input = 28) {
|
|
|
20028
20335
|
}
|
|
20029
20336
|
async function querySearchAnalytics(site, body, opts = {}) {
|
|
20030
20337
|
const { client, identity } = await getAuthorized(opts.accountEmail ?? await cloudAccountFor(site));
|
|
20031
|
-
const db =
|
|
20338
|
+
const db = cacheDb();
|
|
20032
20339
|
const queryHash = hashKey([
|
|
20033
20340
|
site,
|
|
20034
20341
|
identity,
|
|
@@ -20048,10 +20355,12 @@ async function querySearchAnalytics(site, body, opts = {}) {
|
|
|
20048
20355
|
const calls = result.calls;
|
|
20049
20356
|
const requestJson = JSON.stringify(body);
|
|
20050
20357
|
const responseJson = JSON.stringify(allRows);
|
|
20051
|
-
|
|
20052
|
-
|
|
20053
|
-
|
|
20054
|
-
|
|
20358
|
+
writeToCache((database) => {
|
|
20359
|
+
database.prepare(`INSERT OR REPLACE INTO gsc_cache
|
|
20360
|
+
(site_url, query_hash, request_json, response_json, row_count, fetched_at, expires_at)
|
|
20361
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`).run(site, queryHash, requestJson, responseJson, allRows.length, Date.now(), Date.now() + 864e5);
|
|
20362
|
+
noteCacheWrite(Buffer.byteLength(requestJson) + Buffer.byteLength(responseJson), database);
|
|
20363
|
+
});
|
|
20055
20364
|
return {
|
|
20056
20365
|
rows: allRows,
|
|
20057
20366
|
calls,
|
|
@@ -20187,8 +20496,10 @@ async function queryPagesTopQueriesBatch(site, pageUrls, range = 28) {
|
|
|
20187
20496
|
//#region packages/core/dist/gsc/client/sites.js
|
|
20188
20497
|
async function listSites(_refresh = false, accountEmail) {
|
|
20189
20498
|
const entries = !accountEmail && cloudLinked() ? await allCloudSites() : await listSearchConsoleSites(googleFetchFor((await getAuthorized(accountEmail)).client), LOCAL_SIGN_IN);
|
|
20190
|
-
|
|
20191
|
-
|
|
20499
|
+
writeToCache((database) => {
|
|
20500
|
+
const insert = database.prepare("INSERT OR REPLACE INTO sites (site_url, display_name, permission, added_at, is_default) VALUES (?, ?, ?, ?, COALESCE((SELECT is_default FROM sites WHERE site_url = ?), 0))");
|
|
20501
|
+
for (const site of entries) insert.run(site.siteUrl, site.siteUrl, site.permissionLevel ?? null, Date.now(), site.siteUrl);
|
|
20502
|
+
});
|
|
20192
20503
|
return entries;
|
|
20193
20504
|
}
|
|
20194
20505
|
async function allCloudSites() {
|
|
@@ -20366,7 +20677,7 @@ function estimateSerpTitleWidth(title) {
|
|
|
20366
20677
|
}
|
|
20367
20678
|
//#endregion
|
|
20368
20679
|
//#region packages/core/dist/analyze/audit-page.js
|
|
20369
|
-
const defaultDependencies$
|
|
20680
|
+
const defaultDependencies$12 = {
|
|
20370
20681
|
fetchPage,
|
|
20371
20682
|
queryPageMetrics,
|
|
20372
20683
|
now: () => /* @__PURE__ */ new Date()
|
|
@@ -20397,7 +20708,7 @@ function buildRecommendations(report) {
|
|
|
20397
20708
|
});
|
|
20398
20709
|
return recommendations;
|
|
20399
20710
|
}
|
|
20400
|
-
async function auditPage(input, dependencies = defaultDependencies$
|
|
20711
|
+
async function auditPage(input, dependencies = defaultDependencies$12) {
|
|
20401
20712
|
const fetched = await dependencies.fetchPage(input.url, {
|
|
20402
20713
|
js: input.js,
|
|
20403
20714
|
refresh: input.refresh
|
|
@@ -20516,7 +20827,7 @@ async function auditPage(input, dependencies = defaultDependencies$13) {
|
|
|
20516
20827
|
}
|
|
20517
20828
|
//#endregion
|
|
20518
20829
|
//#region packages/core/dist/analyze/content-coverage-analysis.js
|
|
20519
|
-
const STOPWORDS
|
|
20830
|
+
const STOPWORDS = /* @__PURE__ */ new Set([
|
|
20520
20831
|
"a",
|
|
20521
20832
|
"an",
|
|
20522
20833
|
"and",
|
|
@@ -20545,7 +20856,7 @@ function hasNonAscii(value) {
|
|
|
20545
20856
|
}
|
|
20546
20857
|
function queryTerms$1(query) {
|
|
20547
20858
|
const tokens = normalizeForCoverage(query).split(" ").filter((token) => token.length > 1 || hasNonAscii(token));
|
|
20548
|
-
const meaningful = tokens.filter((token) => (token.length > 2 || hasNonAscii(token)) && !STOPWORDS
|
|
20859
|
+
const meaningful = tokens.filter((token) => (token.length > 2 || hasNonAscii(token)) && !STOPWORDS.has(token));
|
|
20549
20860
|
return [...new Set(meaningful.length ? meaningful : tokens)];
|
|
20550
20861
|
}
|
|
20551
20862
|
function countPhrase$1(query, text) {
|
|
@@ -21809,7 +22120,7 @@ const actionPriority = new Map([
|
|
|
21809
22120
|
"unverified",
|
|
21810
22121
|
"covered"
|
|
21811
22122
|
].map((type, index) => [type, index]));
|
|
21812
|
-
function plural$
|
|
22123
|
+
function plural$6(count, singular, pluralLabel) {
|
|
21813
22124
|
return count === 1 ? singular : pluralLabel;
|
|
21814
22125
|
}
|
|
21815
22126
|
function actionable(item) {
|
|
@@ -21843,7 +22154,7 @@ function pageOpportunityVerdict(input) {
|
|
|
21843
22154
|
if (focus === "unverified") return "GSC visibility exists, but the page was not verified and no specific change is justified yet.";
|
|
21844
22155
|
if (focus === "covered") return "The verified page covers the sampled queries and CTR is not materially below its directional benchmark.";
|
|
21845
22156
|
const count = analysis.summary.opportunities;
|
|
21846
|
-
const prefix = `${count} ${plural$
|
|
22157
|
+
const prefix = `${count} ${plural$6(count, "actionable observation", "actionable observations")} found.`;
|
|
21847
22158
|
if (focus === "content-gap") return `${prefix} Verified body coverage is the main issue; confirm intent before expanding the page.`;
|
|
21848
22159
|
if (focus === "serp-framing") return `${prefix} The body is relevant, but its title, meta description, or H1 framing is weak.`;
|
|
21849
22160
|
if (focus === "ranking") return `${prefix} Most upside requires stronger relevance or authority; no CTR-only lift is claimed for page-two queries.`;
|
|
@@ -21858,7 +22169,7 @@ function pageOpportunityRecommendations(analysis) {
|
|
|
21858
22169
|
}
|
|
21859
22170
|
const recommendations = [`Start with "${top.query}". ${top.recommendation}`];
|
|
21860
22171
|
const technical = actions.filter((item) => item.opportunityType === "technical-check").length;
|
|
21861
|
-
if (technical) recommendations.push(`Resolve the technical evidence affecting ${technical} ${plural$
|
|
22172
|
+
if (technical) recommendations.push(`Resolve the technical evidence affecting ${technical} ${plural$6(technical, "row", "rows")} before content work.`);
|
|
21862
22173
|
const gaps = actions.filter((item) => item.opportunityType === "content-gap").length;
|
|
21863
22174
|
if (gaps >= 3) recommendations.push(`Group the ${gaps} verified content gaps into useful sections rather than adding one thin paragraph per query.`);
|
|
21864
22175
|
return recommendations;
|
|
@@ -21866,13 +22177,13 @@ function pageOpportunityRecommendations(analysis) {
|
|
|
21866
22177
|
//#endregion
|
|
21867
22178
|
//#region packages/core/dist/analyze/page-opportunities.js
|
|
21868
22179
|
const MAX_BENCHMARK_ROWS = 1e5;
|
|
21869
|
-
const defaultDependencies$
|
|
22180
|
+
const defaultDependencies$11 = {
|
|
21870
22181
|
searchAnalytics: querySearchAnalytics,
|
|
21871
22182
|
fetch: fetchPage,
|
|
21872
22183
|
extract: extractPage,
|
|
21873
22184
|
now: () => /* @__PURE__ */ new Date()
|
|
21874
22185
|
};
|
|
21875
|
-
function errorMessage
|
|
22186
|
+
function errorMessage(error) {
|
|
21876
22187
|
return error instanceof Error ? error.message : String(error);
|
|
21877
22188
|
}
|
|
21878
22189
|
function unique$3(values) {
|
|
@@ -21893,7 +22204,7 @@ async function verifyPage(input) {
|
|
|
21893
22204
|
});
|
|
21894
22205
|
warnings.push(...fetched.warnings);
|
|
21895
22206
|
} catch (error) {
|
|
21896
|
-
const reason = `Page fetch failed: ${errorMessage
|
|
22207
|
+
const reason = `Page fetch failed: ${errorMessage(error)}`;
|
|
21897
22208
|
return {
|
|
21898
22209
|
verification: {
|
|
21899
22210
|
status: "failed",
|
|
@@ -21915,7 +22226,7 @@ async function verifyPage(input) {
|
|
|
21915
22226
|
warnings: unique$3(warnings)
|
|
21916
22227
|
};
|
|
21917
22228
|
} catch (error) {
|
|
21918
|
-
const reason = `Page extraction failed: ${errorMessage
|
|
22229
|
+
const reason = `Page extraction failed: ${errorMessage(error)}`;
|
|
21919
22230
|
return {
|
|
21920
22231
|
fetched,
|
|
21921
22232
|
verification: {
|
|
@@ -21938,7 +22249,7 @@ function reportCaveats$1(input) {
|
|
|
21938
22249
|
input.benchmarkTruncated ? `The site-wide benchmark reached its ${MAX_BENCHMARK_ROWS.toLocaleString("en-US")}-row cap, so provenance is complete but the peer sample is truncated.` : ""
|
|
21939
22250
|
].filter(Boolean);
|
|
21940
22251
|
}
|
|
21941
|
-
async function pageOpportunitiesReport(input, dependencies = defaultDependencies$
|
|
22252
|
+
async function pageOpportunitiesReport(input, dependencies = defaultDependencies$11) {
|
|
21942
22253
|
const url = assertUrlMatchesGscProperty(input.site, input.url);
|
|
21943
22254
|
const days = reportDays$1(input.days);
|
|
21944
22255
|
const range = defaultDateRange(days);
|
|
@@ -26313,7 +26624,7 @@ function taskIds(value) {
|
|
|
26313
26624
|
}
|
|
26314
26625
|
}
|
|
26315
26626
|
function readProviderCache(key, schema, options = {}) {
|
|
26316
|
-
const database = options.database ??
|
|
26627
|
+
const database = options.database ?? cacheDb();
|
|
26317
26628
|
const now = options.now ?? Date.now();
|
|
26318
26629
|
const hash = requestHash(key, stableProviderRequestJson(key.request));
|
|
26319
26630
|
const row = database.prepare(`SELECT response_json, row_count, source_cost_micros, task_ids_json,
|
|
@@ -26334,9 +26645,11 @@ function readProviderCache(key, schema, options = {}) {
|
|
|
26334
26645
|
taskIds: taskIds(row.task_ids_json)
|
|
26335
26646
|
};
|
|
26336
26647
|
} catch {
|
|
26337
|
-
|
|
26338
|
-
|
|
26339
|
-
AND
|
|
26648
|
+
try {
|
|
26649
|
+
database.prepare(`DELETE FROM provider_cache
|
|
26650
|
+
WHERE provider = ? AND credential_scope = ? AND operation = ?
|
|
26651
|
+
AND request_hash = ?`).run(key.provider, key.credentialScope, key.operation, hash);
|
|
26652
|
+
} catch {}
|
|
26340
26653
|
return null;
|
|
26341
26654
|
}
|
|
26342
26655
|
}
|
|
@@ -26347,19 +26660,22 @@ function writeProviderCache(key, input, options = {}) {
|
|
|
26347
26660
|
code: "configuration",
|
|
26348
26661
|
message: `Provider cache TTL must be between 1 and ${CACHE_MAX_AGE_MS} milliseconds.`
|
|
26349
26662
|
});
|
|
26350
|
-
const database = options.database ?? getDb();
|
|
26351
26663
|
const now = options.now ?? Date.now();
|
|
26352
26664
|
const expiresAt = now + input.ttlMs;
|
|
26353
26665
|
const requestJson = stableProviderRequestJson(key.request);
|
|
26354
26666
|
const responseJson = JSON.stringify(input.data);
|
|
26355
26667
|
const normalizedTaskIds = [...new Set(input.taskIds)].filter((id) => id.length > 0 && id.length <= 100).sort((left, right) => left < right ? -1 : left > right ? 1 : 0).slice(0, 20);
|
|
26356
26668
|
const taskIdsJson = JSON.stringify(normalizedTaskIds);
|
|
26357
|
-
|
|
26358
|
-
|
|
26359
|
-
|
|
26360
|
-
|
|
26361
|
-
|
|
26362
|
-
|
|
26669
|
+
const save = (database) => {
|
|
26670
|
+
database.prepare(`INSERT OR REPLACE INTO provider_cache (
|
|
26671
|
+
provider, credential_scope, operation, request_hash, request_json,
|
|
26672
|
+
response_json, row_count, source_cost_micros, task_ids_json,
|
|
26673
|
+
fetched_at, expires_at
|
|
26674
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(key.provider, key.credentialScope, key.operation, requestHash(key, requestJson), requestJson, responseJson, input.rowCount, input.sourceCostMicros, taskIdsJson, now, expiresAt);
|
|
26675
|
+
noteCacheWrite(Buffer.byteLength(requestJson) + Buffer.byteLength(responseJson) + Buffer.byteLength(taskIdsJson), database);
|
|
26676
|
+
};
|
|
26677
|
+
if (options.database) save(options.database);
|
|
26678
|
+
else writeToCache(save);
|
|
26363
26679
|
return {
|
|
26364
26680
|
data: input.data,
|
|
26365
26681
|
storedAt: new Date(now).toISOString(),
|
|
@@ -26980,9 +27296,9 @@ async function loadInstalledProviderExtensions(options = {}) {
|
|
|
26980
27296
|
//#endregion
|
|
26981
27297
|
//#region packages/core/dist/provider-extensions/runtime.js
|
|
26982
27298
|
const DEFAULT_MAX_REQUESTS = 8;
|
|
26983
|
-
const MAX_RESPONSE_BYTES = 10 * 1024 * 1024;
|
|
27299
|
+
const MAX_RESPONSE_BYTES$1 = 10 * 1024 * 1024;
|
|
26984
27300
|
const MAX_REQUEST_BODY_BYTES = 64 * 1024;
|
|
26985
|
-
const TIMEOUT_MS$
|
|
27301
|
+
const TIMEOUT_MS$2 = 15e3;
|
|
26986
27302
|
function normalizeProviderExtensionAccount(provider, account) {
|
|
26987
27303
|
const fields = provider.connection.fields.filter((field) => field.kind === "account");
|
|
26988
27304
|
const allowed = new Set(fields.map((field) => field.id));
|
|
@@ -27028,8 +27344,8 @@ function runtimeForProvider(provider, options = {}) {
|
|
|
27028
27344
|
...input.body ? { body: input.body } : {}
|
|
27029
27345
|
},
|
|
27030
27346
|
fetch: providerFetch,
|
|
27031
|
-
maxResponseBytes: MAX_RESPONSE_BYTES,
|
|
27032
|
-
timeoutMs: TIMEOUT_MS$
|
|
27347
|
+
maxResponseBytes: MAX_RESPONSE_BYTES$1,
|
|
27348
|
+
timeoutMs: TIMEOUT_MS$2,
|
|
27033
27349
|
retry: (input.method ?? "GET") === "GET" ? "safe" : "never",
|
|
27034
27350
|
schema: z.unknown()
|
|
27035
27351
|
});
|
|
@@ -27117,7 +27433,7 @@ async function runAnalyticsProviderLandingPages(input) {
|
|
|
27117
27433
|
}
|
|
27118
27434
|
//#endregion
|
|
27119
27435
|
//#region packages/core/dist/analyze/workflows/analytics-value.js
|
|
27120
|
-
const defaultDependencies$
|
|
27436
|
+
const defaultDependencies$10 = {
|
|
27121
27437
|
runGa4Report,
|
|
27122
27438
|
analyticsLandingPages: runAnalyticsProviderLandingPages
|
|
27123
27439
|
};
|
|
@@ -27188,7 +27504,7 @@ function landingPageRankingPolicy(input) {
|
|
|
27188
27504
|
warnings
|
|
27189
27505
|
};
|
|
27190
27506
|
}
|
|
27191
|
-
async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
27507
|
+
async function fetchLandingPageValues(input, dependencies = defaultDependencies$10) {
|
|
27192
27508
|
const retainedRowLimit = input.limit ?? 5e3;
|
|
27193
27509
|
const connection = input.connection ?? (input.propertyId ? {
|
|
27194
27510
|
provider: "google",
|
|
@@ -27207,7 +27523,7 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
27207
27523
|
}
|
|
27208
27524
|
};
|
|
27209
27525
|
if (connection.provider === "extension") try {
|
|
27210
|
-
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$
|
|
27526
|
+
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$10.analyticsLandingPages)({
|
|
27211
27527
|
providerId: connection.providerId,
|
|
27212
27528
|
account: connection.account,
|
|
27213
27529
|
startDate: input.startDate,
|
|
@@ -27247,7 +27563,7 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
27247
27563
|
clicky: async (selected) => {
|
|
27248
27564
|
if (selected.provider !== "clicky") throw new Error("Clicky analytics connection mismatch.");
|
|
27249
27565
|
try {
|
|
27250
|
-
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$
|
|
27566
|
+
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$10.analyticsLandingPages)({
|
|
27251
27567
|
providerId: "clicky",
|
|
27252
27568
|
account: { siteId: selected.siteId },
|
|
27253
27569
|
startDate: input.startDate,
|
|
@@ -27286,7 +27602,7 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
27286
27602
|
google: async (selected) => {
|
|
27287
27603
|
if (selected.provider !== "google") throw new Error("Google Analytics connection mismatch.");
|
|
27288
27604
|
try {
|
|
27289
|
-
const result = await (dependencies.runGa4Report ?? defaultDependencies$
|
|
27605
|
+
const result = await (dependencies.runGa4Report ?? defaultDependencies$10.runGa4Report)(selected.propertyId, {
|
|
27290
27606
|
dateRanges: [{
|
|
27291
27607
|
startDate: input.startDate,
|
|
27292
27608
|
endDate: input.endDate
|
|
@@ -28067,7 +28383,7 @@ const NAVIGATIONAL_STEM_THRESHOLDS = {
|
|
|
28067
28383
|
function compact(value) {
|
|
28068
28384
|
return normalizeBrandText(value).replace(/\s+/g, "");
|
|
28069
28385
|
}
|
|
28070
|
-
function round$
|
|
28386
|
+
function round$2(value, digits) {
|
|
28071
28387
|
return Number(value.toFixed(digits));
|
|
28072
28388
|
}
|
|
28073
28389
|
function stemQueryEvidence(rows, stem) {
|
|
@@ -28094,8 +28410,8 @@ function stemQueryEvidence(rows, stem) {
|
|
|
28094
28410
|
query,
|
|
28095
28411
|
clicks,
|
|
28096
28412
|
impressions,
|
|
28097
|
-
ctr: round$
|
|
28098
|
-
position: round$
|
|
28413
|
+
ctr: round$2(clicks / impressions, 4),
|
|
28414
|
+
position: round$2(weightedPosition / impressions, 2),
|
|
28099
28415
|
sourceRows
|
|
28100
28416
|
};
|
|
28101
28417
|
}
|
|
@@ -28590,9 +28906,9 @@ function analyzeCtrUnderperformersFromRows(input) {
|
|
|
28590
28906
|
//#endregion
|
|
28591
28907
|
//#region packages/core/dist/analyze/ctr-underperformers-report.js
|
|
28592
28908
|
const MAX_GSC_ROWS$3 = 1e5;
|
|
28593
|
-
const DEFAULT_DAYS$
|
|
28909
|
+
const DEFAULT_DAYS$3 = 28;
|
|
28594
28910
|
const MAX_DAYS$2 = 548;
|
|
28595
|
-
function plural$
|
|
28911
|
+
function plural$5(count, singular, pluralLabel = `${singular}s`) {
|
|
28596
28912
|
return count === 1 ? singular : pluralLabel;
|
|
28597
28913
|
}
|
|
28598
28914
|
async function ctrUnderperformersReportWith(input, dependencies) {
|
|
@@ -28613,7 +28929,7 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
28613
28929
|
const explicitRange = explicitDateRange(input, MAX_DAYS$2);
|
|
28614
28930
|
const days = explicitRange?.days ?? integerOption$1({
|
|
28615
28931
|
value: input.days,
|
|
28616
|
-
fallback: DEFAULT_DAYS$
|
|
28932
|
+
fallback: DEFAULT_DAYS$3,
|
|
28617
28933
|
minimum: 1,
|
|
28618
28934
|
maximum: MAX_DAYS$2,
|
|
28619
28935
|
label: "Days"
|
|
@@ -28695,11 +29011,11 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
28695
29011
|
minImpressions,
|
|
28696
29012
|
limit,
|
|
28697
29013
|
brandFiltering: input.includeBrand ? "included" : "excluded",
|
|
28698
|
-
verdict: top ? `${selection.eligibleUnderperformers} CTR ${plural$
|
|
29014
|
+
verdict: top ? `${selection.eligibleUnderperformers} CTR ${plural$5(selection.eligibleUnderperformers, "underperformer")} found in retained rows, with a calculated heuristic shortfall of ${totalClickShortfall.toFixed(0)} clicks for this window. Start review with "${top.query}" because it has the largest gap.` : dataStatus === "partial" ? "No material CTR underperformers remained in the validated retained rows, but partial evidence prevents an all-clear." : `No high-impression rows at average position below 4 in the retained rows are materially below the heuristic CTR benchmark.`
|
|
28699
29015
|
},
|
|
28700
29016
|
items,
|
|
28701
29017
|
caveats: [
|
|
28702
|
-
`Date window: ${range.startDate} to ${range.endDate} (${days} ${plural$
|
|
29018
|
+
`Date window: ${range.startDate} to ${range.endDate} (${days} ${plural$5(days, "day")}), using final GSC data where available.`,
|
|
28703
29019
|
`Brand queries: ${input.includeBrand ? "included" : "excluded"}. Brand terms used: ${brand.terms.join(", ") || "none"} (${brand.source === "provided" ? "provided" : "host forms, plus the bare domain stem only when Search Console shows the site is its navigational answer"}).`,
|
|
28704
29020
|
`Only rows at average position below 4 with at least ${minImpressions} impressions were checked. Rows at position 4 to 10 belong to quick-wins, which uses the same site CTR curve.`,
|
|
28705
29021
|
minImpressionsCaveat({
|
|
@@ -28707,13 +29023,13 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
28707
29023
|
belowMinimumRows: selection.belowMinimumRows
|
|
28708
29024
|
}),
|
|
28709
29025
|
"Expected CTR uses the site CTR curve from all page-one rows (75th percentile of qualified URL CTRs per position) when enough peer data exists, otherwise the fallback position curve.",
|
|
28710
|
-
...selection.fragmentRows ? [`${selection.fragmentRows} source ${plural$
|
|
28711
|
-
...selection.flaggedRows ? [`${selection.flaggedRows} eligible ${plural$
|
|
29026
|
+
...selection.fragmentRows ? [`${selection.fragmentRows} source ${plural$5(selection.fragmentRows, "row")} with a #fragment URL ${selection.fragmentRows === 1 ? "was" : "were"} merged into the base URL.`] : [],
|
|
29027
|
+
...selection.flaggedRows ? [`${selection.flaggedRows} eligible ${plural$5(selection.flaggedRows, "row")} carry heuristic query flags (likely-automated or navigational-other) and rank after unflagged rows.`] : [],
|
|
28712
29028
|
"CTR benchmarks and calculated click shortfalls are directional heuristics, not forecasts or proof that clicks are available. Validate the search intent and live SERP before editing.",
|
|
28713
|
-
...selection.invalidRows ? [`${selection.invalidRows} invalid provider ${plural$
|
|
28714
|
-
...selection.duplicateRows ? [`${selection.duplicateRows} repeated query/page ${plural$
|
|
29029
|
+
...selection.invalidRows ? [`${selection.invalidRows} invalid provider ${plural$5(selection.invalidRows, "row")} ${selection.invalidRows === 1 ? "was" : "were"} excluded, so this report is partial.`] : [],
|
|
29030
|
+
...selection.duplicateRows ? [`${selection.duplicateRows} repeated query/page ${plural$5(selection.duplicateRows, "row")} ${selection.duplicateRows === 1 ? "was" : "were"} aggregated before CTR and average position were calculated.`] : [],
|
|
28715
29031
|
...possiblyTruncated ? [`The retained-row limit of ${MAX_GSC_ROWS$3} was reached, so lower-ranked source rows may be absent.`] : [],
|
|
28716
|
-
...selection.limitedUnderperformers ? [`${selection.limitedUnderperformers} eligible ${plural$
|
|
29032
|
+
...selection.limitedUnderperformers ? [`${selection.limitedUnderperformers} eligible ${plural$5(selection.limitedUnderperformers, "underperformer")} ${selection.limitedUnderperformers === 1 ? "was" : "were"} omitted by the output limit.`] : []
|
|
28717
29033
|
],
|
|
28718
29034
|
recommendations: top ? [
|
|
28719
29035
|
`Review the live SERP and displayed snippet for "${top.query}" first. Test the title or meta description only if the observed framing does not match the query intent.`,
|
|
@@ -28724,20 +29040,20 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
28724
29040
|
}
|
|
28725
29041
|
//#endregion
|
|
28726
29042
|
//#region packages/core/dist/analyze/ctr-underperformers.js
|
|
28727
|
-
const defaultDependencies$
|
|
29043
|
+
const defaultDependencies$9 = {
|
|
28728
29044
|
searchAnalytics: querySearchAnalytics,
|
|
28729
29045
|
now: () => /* @__PURE__ */ new Date()
|
|
28730
29046
|
};
|
|
28731
|
-
function ctrUnderperformersReport(input, dependencies = defaultDependencies$
|
|
29047
|
+
function ctrUnderperformersReport(input, dependencies = defaultDependencies$9) {
|
|
28732
29048
|
return ctrUnderperformersReportWith(input, dependencies);
|
|
28733
29049
|
}
|
|
28734
29050
|
//#endregion
|
|
28735
29051
|
//#region packages/core/dist/phrasing.js
|
|
28736
|
-
function plural$
|
|
29052
|
+
function plural$4(count, singular, pluralLabel = `${singular}s`) {
|
|
28737
29053
|
return count === 1 ? singular : pluralLabel;
|
|
28738
29054
|
}
|
|
28739
29055
|
function countLabel(count, singular, pluralLabel = `${singular}s`) {
|
|
28740
|
-
return `${count.toLocaleString("en-GB")} ${plural$
|
|
29056
|
+
return `${count.toLocaleString("en-GB")} ${plural$4(count, singular, pluralLabel)}`;
|
|
28741
29057
|
}
|
|
28742
29058
|
//#endregion
|
|
28743
29059
|
//#region packages/core/dist/analyze/diagnosis-priorities.js
|
|
@@ -29185,11 +29501,11 @@ function unavailableSegmentImpactReport(input) {
|
|
|
29185
29501
|
//#endregion
|
|
29186
29502
|
//#region packages/core/dist/analyze/segment-impact.js
|
|
29187
29503
|
const DEFAULT_MAX_ROWS = 1e5;
|
|
29188
|
-
const defaultDependencies$
|
|
29504
|
+
const defaultDependencies$8 = {
|
|
29189
29505
|
searchAnalytics: querySearchAnalytics,
|
|
29190
29506
|
now: () => /* @__PURE__ */ new Date()
|
|
29191
29507
|
};
|
|
29192
|
-
async function segmentImpact(input, dependencies = defaultDependencies$
|
|
29508
|
+
async function segmentImpact(input, dependencies = defaultDependencies$8) {
|
|
29193
29509
|
validateSegmentSite(input.site);
|
|
29194
29510
|
const now = dependencies.now();
|
|
29195
29511
|
const ranges = resolveSegmentRanges({
|
|
@@ -31042,11 +31358,11 @@ async function cannibalReportWith(input, dependencies) {
|
|
|
31042
31358
|
}
|
|
31043
31359
|
//#endregion
|
|
31044
31360
|
//#region packages/core/dist/analyze/site-diagnostics/cannibal.js
|
|
31045
|
-
const defaultDependencies$
|
|
31361
|
+
const defaultDependencies$7 = {
|
|
31046
31362
|
searchAnalytics: querySearchAnalytics,
|
|
31047
31363
|
now: () => /* @__PURE__ */ new Date()
|
|
31048
31364
|
};
|
|
31049
|
-
async function cannibalReport(input, dependencies = defaultDependencies$
|
|
31365
|
+
async function cannibalReport(input, dependencies = defaultDependencies$7) {
|
|
31050
31366
|
if (input.researchFiles) {
|
|
31051
31367
|
const conflicts = [
|
|
31052
31368
|
["days", input.days],
|
|
@@ -31504,11 +31820,11 @@ async function decayingReportWith(input, dependencies) {
|
|
|
31504
31820
|
}
|
|
31505
31821
|
//#endregion
|
|
31506
31822
|
//#region packages/core/dist/analyze/site-diagnostics/decaying.js
|
|
31507
|
-
const defaultDependencies$
|
|
31823
|
+
const defaultDependencies$6 = {
|
|
31508
31824
|
searchAnalytics: querySearchAnalytics,
|
|
31509
31825
|
now: () => /* @__PURE__ */ new Date()
|
|
31510
31826
|
};
|
|
31511
|
-
function decayingReport(input, dependencies = defaultDependencies$
|
|
31827
|
+
function decayingReport(input, dependencies = defaultDependencies$6) {
|
|
31512
31828
|
return decayingReportWith(input, dependencies);
|
|
31513
31829
|
}
|
|
31514
31830
|
//#endregion
|
|
@@ -31818,7 +32134,7 @@ function quickWinTemplateSummaries(items) {
|
|
|
31818
32134
|
}
|
|
31819
32135
|
//#endregion
|
|
31820
32136
|
//#region packages/core/dist/analyze/site-diagnostics/quick-wins-report.js
|
|
31821
|
-
const DEFAULT_DAYS$
|
|
32137
|
+
const DEFAULT_DAYS$2 = 28;
|
|
31822
32138
|
const MAX_DAYS$1 = 548;
|
|
31823
32139
|
const DEFAULT_VERIFY_LIMIT$1 = 5;
|
|
31824
32140
|
const MAX_VERIFY_LIMIT = 100;
|
|
@@ -31840,7 +32156,7 @@ async function quickWinsReportWith(input, dependencies) {
|
|
|
31840
32156
|
const explicitRange = explicitDateRange(input, MAX_DAYS$1);
|
|
31841
32157
|
const days = explicitRange?.days ?? integerOption$1({
|
|
31842
32158
|
value: input.days,
|
|
31843
|
-
fallback: DEFAULT_DAYS$
|
|
32159
|
+
fallback: DEFAULT_DAYS$2,
|
|
31844
32160
|
minimum: 1,
|
|
31845
32161
|
maximum: MAX_DAYS$1,
|
|
31846
32162
|
label: "Days"
|
|
@@ -32005,13 +32321,13 @@ async function quickWinsReportWith(input, dependencies) {
|
|
|
32005
32321
|
}
|
|
32006
32322
|
//#endregion
|
|
32007
32323
|
//#region packages/core/dist/analyze/site-diagnostics/quick-wins.js
|
|
32008
|
-
const defaultDependencies$
|
|
32324
|
+
const defaultDependencies$5 = {
|
|
32009
32325
|
searchAnalytics: querySearchAnalytics,
|
|
32010
32326
|
fetch: fetchPage,
|
|
32011
32327
|
extract: extractPage,
|
|
32012
32328
|
now: () => /* @__PURE__ */ new Date()
|
|
32013
32329
|
};
|
|
32014
|
-
function quickWinsReport(input, dependencies = defaultDependencies$
|
|
32330
|
+
function quickWinsReport(input, dependencies = defaultDependencies$5) {
|
|
32015
32331
|
return quickWinsReportWith(input, dependencies);
|
|
32016
32332
|
}
|
|
32017
32333
|
//#endregion
|
|
@@ -32264,7 +32580,7 @@ function analyzeStrikingDistanceRows(input) {
|
|
|
32264
32580
|
}
|
|
32265
32581
|
//#endregion
|
|
32266
32582
|
//#region packages/core/dist/analyze/striking-distance-report.js
|
|
32267
|
-
const DEFAULT_DAYS$
|
|
32583
|
+
const DEFAULT_DAYS$1 = 28;
|
|
32268
32584
|
const MAX_DAYS = 548;
|
|
32269
32585
|
const DEFAULT_VERIFY_LIMIT = 5;
|
|
32270
32586
|
const MAX_SOURCE_ROWS$1 = 1e5;
|
|
@@ -32273,7 +32589,7 @@ function boundedInteger$2(value, fallback, minimum, maximum) {
|
|
|
32273
32589
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
32274
32590
|
}
|
|
32275
32591
|
function reportDays(value) {
|
|
32276
|
-
if (value === void 0) return DEFAULT_DAYS$
|
|
32592
|
+
if (value === void 0) return DEFAULT_DAYS$1;
|
|
32277
32593
|
if (!Number.isInteger(value) || value < 1 || value > MAX_DAYS) throw new SeoError("INVALID_INPUT", `Days must be a whole number between 1 and ${MAX_DAYS}.`);
|
|
32278
32594
|
return value;
|
|
32279
32595
|
}
|
|
@@ -32420,12 +32736,12 @@ async function strikingDistanceWith(input, dependencies) {
|
|
|
32420
32736
|
}
|
|
32421
32737
|
//#endregion
|
|
32422
32738
|
//#region packages/core/dist/analyze/striking-distance.js
|
|
32423
|
-
const defaultDependencies$
|
|
32739
|
+
const defaultDependencies$4 = {
|
|
32424
32740
|
searchAnalytics: querySearchAnalytics,
|
|
32425
32741
|
verifyContent: verifyQueryContent,
|
|
32426
32742
|
now: () => /* @__PURE__ */ new Date()
|
|
32427
32743
|
};
|
|
32428
|
-
function strikingDistance(input, dependencies = defaultDependencies$
|
|
32744
|
+
function strikingDistance(input, dependencies = defaultDependencies$4) {
|
|
32429
32745
|
return strikingDistanceWith(input, dependencies);
|
|
32430
32746
|
}
|
|
32431
32747
|
//#endregion
|
|
@@ -32437,7 +32753,7 @@ async function fetchSearchStatusIncidents() {
|
|
|
32437
32753
|
return readJson(response, "Google Search Status", { maxBytes: 10 * 1024 * 1024 });
|
|
32438
32754
|
}
|
|
32439
32755
|
async function listSearchUpdates(input = {}) {
|
|
32440
|
-
return searchUpdatesFromIncidents(await fetchSearchStatusIncidents(), input);
|
|
32756
|
+
return searchUpdatesFromIncidents(withRankingHistory(await fetchSearchStatusIncidents()), input);
|
|
32441
32757
|
}
|
|
32442
32758
|
//#endregion
|
|
32443
32759
|
//#region packages/core/dist/analyze/traffic-anomaly-analysis.js
|
|
@@ -32496,7 +32812,7 @@ function anomalyEvidence(anomaly) {
|
|
|
32496
32812
|
if (anomaly.direction === "normal") return `${formatMetric(anomaly.metric)} changed ${changePhrase} (${anomaly.baselineMean.toLocaleString("en-GB")} -> ${anomaly.comparisonMean.toLocaleString("en-GB")} per observed day${anomaly.zScore === null ? "; the baseline had no observed variance" : `, z ${anomaly.zScore}`}).`;
|
|
32497
32813
|
return `${formatMetric(anomaly.metric)} ${direction} ${changePhrase} (${anomaly.baselineMean.toLocaleString("en-GB")} -> ${anomaly.comparisonMean.toLocaleString("en-GB")} per observed day${anomaly.zScore === null ? "; outside the flat observed baseline" : `, z ${anomaly.zScore}`}).`;
|
|
32498
32814
|
}
|
|
32499
|
-
function plural$
|
|
32815
|
+
function plural$3(count, singular, pluralLabel = `${singular}s`) {
|
|
32500
32816
|
return count === 1 ? singular : pluralLabel;
|
|
32501
32817
|
}
|
|
32502
32818
|
function strongestSignal(anomalies) {
|
|
@@ -32552,13 +32868,13 @@ function interpretUpdateCorrelation(input) {
|
|
|
32552
32868
|
summary,
|
|
32553
32869
|
evidence: [
|
|
32554
32870
|
...input.anomalies.map(anomalyEvidence),
|
|
32555
|
-
hasUpdate ? `${input.overlappingUpdates.length} official Google Ranking update ${plural$
|
|
32871
|
+
hasUpdate ? `${input.overlappingUpdates.length} official Google Ranking update ${plural$3(input.overlappingUpdates.length, "window")} overlapped the change date or comparison window: ${updateNames.join(", ")}.` : "No official Google Ranking update windows overlapped the change date or comparison window.",
|
|
32556
32872
|
largeMovement ? "Both clicks and impressions moved sharply in the same direction." : "Clicks and impressions did not both meet the sharp-movement evidence threshold.",
|
|
32557
32873
|
hasConfounders ? `Known overlapping site changes: ${confounders.map((confounder) => confounder.title).join("; ")}.` : "No overlapping site changes were supplied."
|
|
32558
32874
|
],
|
|
32559
32875
|
caveats: [
|
|
32560
32876
|
`GSC window: ${input.days} days; recent comparison window: ${input.recentDays} days.`,
|
|
32561
|
-
`Update overlap padding: ${input.paddingDays} ${plural$
|
|
32877
|
+
`Update overlap padding: ${input.paddingDays} ${plural$3(input.paddingDays, "day")} either side of the step-change date, or of the recent comparison window when no step change was found.`,
|
|
32562
32878
|
input.refresh ? "Data freshness: cache bypassed for GSC data." : "Data freshness: local GSC cache may have been used.",
|
|
32563
32879
|
"Source: official Google Search Status Dashboard Ranking incidents only; third-party volatility and unconfirmed chatter are not included.",
|
|
32564
32880
|
"An update-window overlap cannot establish causation, even when movement is large and no known site change was recorded.",
|
|
@@ -32726,7 +33042,7 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
32726
33042
|
if (step && changeDate) {
|
|
32727
33043
|
const previousDate = aggregated.rows[step.index - 1]?.keys[0];
|
|
32728
33044
|
const gap = previousDate ? calendarDays(previousDate, changeDate) - 2 : 0;
|
|
32729
|
-
stepCaveats.push(`${formatMetric(metric)} changed level on ${changeDate}, so its baseline is every observed day before that date and its comparison is every observed day from it.`, ...gap > 0 ? [`The ${metric} change date follows ${gap} missing ${plural$
|
|
33045
|
+
stepCaveats.push(`${formatMetric(metric)} changed level on ${changeDate}, so its baseline is every observed day before that date and its comparison is every observed day from it.`, ...gap > 0 ? [`The ${metric} change date follows ${gap} missing ${plural$3(gap, "day")}; the change may have started on one of them.`] : []);
|
|
32730
33046
|
return compareSegments({
|
|
32731
33047
|
site: input.site,
|
|
32732
33048
|
metric,
|
|
@@ -32757,9 +33073,9 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
32757
33073
|
const comparisonMissingDays = input.recentDays - comparisonRows.length;
|
|
32758
33074
|
const missingDays = expectedDays - aggregated.rows.length;
|
|
32759
33075
|
const caveats = [
|
|
32760
|
-
...missingDays > 0 ? [`${missingDays} requested calendar ${plural$
|
|
32761
|
-
...aggregated.invalidRows > 0 ? [`${aggregated.invalidRows} invalid provider ${plural$
|
|
32762
|
-
...aggregated.duplicateRows > 0 ? [`${aggregated.duplicateRows} duplicate date ${plural$
|
|
33076
|
+
...missingDays > 0 ? [`${missingDays} requested calendar ${plural$3(missingDays, "day")} had no returned date aggregate and ${missingDays === 1 ? "was" : "were"} not filled with zeros.`] : [],
|
|
33077
|
+
...aggregated.invalidRows > 0 ? [`${aggregated.invalidRows} invalid provider ${plural$3(aggregated.invalidRows, "row")} ${aggregated.invalidRows === 1 ? "was" : "were"} excluded.`] : [],
|
|
33078
|
+
...aggregated.duplicateRows > 0 ? [`${aggregated.duplicateRows} duplicate date ${plural$3(aggregated.duplicateRows, "row")} ${aggregated.duplicateRows === 1 ? "was" : "were"} aggregated before analysis.`] : [],
|
|
32763
33079
|
...stepCaveats
|
|
32764
33080
|
];
|
|
32765
33081
|
return {
|
|
@@ -32842,11 +33158,11 @@ async function trafficAnomalyWith(input, dependencies) {
|
|
|
32842
33158
|
}
|
|
32843
33159
|
//#endregion
|
|
32844
33160
|
//#region packages/core/dist/analyze/traffic-anomaly.js
|
|
32845
|
-
const defaultDependencies$
|
|
33161
|
+
const defaultDependencies$3 = {
|
|
32846
33162
|
searchAnalytics: querySearchAnalytics,
|
|
32847
33163
|
now: () => /* @__PURE__ */ new Date()
|
|
32848
33164
|
};
|
|
32849
|
-
function trafficAnomaly(input, dependencies = defaultDependencies$
|
|
33165
|
+
function trafficAnomaly(input, dependencies = defaultDependencies$3) {
|
|
32850
33166
|
return trafficAnomalyWith(input, dependencies);
|
|
32851
33167
|
}
|
|
32852
33168
|
async function updateCorrelation(input) {
|
|
@@ -32859,7 +33175,7 @@ async function updateCorrelation(input) {
|
|
|
32859
33175
|
const comparisonEnd = window?.end;
|
|
32860
33176
|
const updates = await listSearchUpdates({
|
|
32861
33177
|
product: "Ranking",
|
|
32862
|
-
limit:
|
|
33178
|
+
limit: Number.POSITIVE_INFINITY
|
|
32863
33179
|
});
|
|
32864
33180
|
const overlappingUpdates = comparisonStart && comparisonEnd ? findOverlappingSearchUpdates({
|
|
32865
33181
|
updates,
|
|
@@ -33204,7 +33520,7 @@ function emptyQuickWins(input) {
|
|
|
33204
33520
|
warnings: []
|
|
33205
33521
|
};
|
|
33206
33522
|
}
|
|
33207
|
-
const defaultDependencies$
|
|
33523
|
+
const defaultDependencies$2 = {
|
|
33208
33524
|
trafficAnomaly,
|
|
33209
33525
|
updateCorrelation,
|
|
33210
33526
|
segmentImpact,
|
|
@@ -33215,7 +33531,7 @@ const defaultDependencies$3 = {
|
|
|
33215
33531
|
};
|
|
33216
33532
|
async function diagnoseProperty(input, dependencies = {}) {
|
|
33217
33533
|
const providers = {
|
|
33218
|
-
...defaultDependencies$
|
|
33534
|
+
...defaultDependencies$2,
|
|
33219
33535
|
...dependencies
|
|
33220
33536
|
};
|
|
33221
33537
|
const limit = input.limit ?? 10;
|
|
@@ -33426,296 +33742,281 @@ async function diagnoseProperty(input, dependencies = {}) {
|
|
|
33426
33742
|
};
|
|
33427
33743
|
}
|
|
33428
33744
|
//#endregion
|
|
33429
|
-
//#region packages/core/dist/
|
|
33430
|
-
|
|
33431
|
-
|
|
33432
|
-
|
|
33433
|
-
|
|
33434
|
-
|
|
33435
|
-
|
|
33436
|
-
|
|
33437
|
-
|
|
33438
|
-
"for",
|
|
33439
|
-
"free",
|
|
33440
|
-
"from",
|
|
33441
|
-
"guide",
|
|
33442
|
-
"how",
|
|
33443
|
-
"in",
|
|
33444
|
-
"is",
|
|
33445
|
-
"of",
|
|
33446
|
-
"on",
|
|
33447
|
-
"online",
|
|
33448
|
-
"or",
|
|
33449
|
-
"page",
|
|
33450
|
-
"seo",
|
|
33451
|
-
"site",
|
|
33452
|
-
"the",
|
|
33453
|
-
"to",
|
|
33454
|
-
"tool",
|
|
33455
|
-
"use",
|
|
33456
|
-
"website",
|
|
33457
|
-
"what",
|
|
33458
|
-
"when",
|
|
33459
|
-
"where",
|
|
33460
|
-
"which",
|
|
33461
|
-
"with"
|
|
33462
|
-
]);
|
|
33463
|
-
function compareInternalLinkText(left, right) {
|
|
33464
|
-
return left < right ? -1 : left > right ? 1 : 0;
|
|
33465
|
-
}
|
|
33466
|
-
function normalizeInternalLinkQuery(value) {
|
|
33467
|
-
return value.normalize("NFKC").replace(/[’']/g, "'").toLowerCase().replace(/[^\p{L}\p{N}']+/gu, " ").replace(/\s+/g, " ").trim();
|
|
33468
|
-
}
|
|
33469
|
-
function normalizeInternalLinkUrl(value) {
|
|
33470
|
-
try {
|
|
33471
|
-
const url = new URL(value);
|
|
33472
|
-
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
33473
|
-
url.hash = "";
|
|
33474
|
-
return url.toString();
|
|
33475
|
-
} catch {
|
|
33476
|
-
return;
|
|
33745
|
+
//#region packages/core/dist/decision-model.js
|
|
33746
|
+
var DecisionModelError = class extends Error {
|
|
33747
|
+
kind;
|
|
33748
|
+
retryAfterMs;
|
|
33749
|
+
constructor(kind, message, retryAfterMs) {
|
|
33750
|
+
super(message);
|
|
33751
|
+
this.kind = kind;
|
|
33752
|
+
this.retryAfterMs = retryAfterMs;
|
|
33753
|
+
this.name = "DecisionModelError";
|
|
33477
33754
|
}
|
|
33478
|
-
|
|
33479
|
-
|
|
33480
|
-
|
|
33481
|
-
|
|
33482
|
-
|
|
33483
|
-
|
|
33484
|
-
|
|
33485
|
-
|
|
33486
|
-
|
|
33487
|
-
|
|
33488
|
-
|
|
33489
|
-
|
|
33490
|
-
|
|
33491
|
-
|
|
33492
|
-
|
|
33493
|
-
|
|
33494
|
-
|
|
33495
|
-
|
|
33496
|
-
|
|
33497
|
-
|
|
33498
|
-
|
|
33499
|
-
|
|
33500
|
-
|
|
33501
|
-
|
|
33502
|
-
|
|
33503
|
-
|
|
33504
|
-
|
|
33505
|
-
|
|
33506
|
-
|
|
33507
|
-
|
|
33508
|
-
|
|
33509
|
-
|
|
33510
|
-
|
|
33511
|
-
|
|
33512
|
-
|
|
33513
|
-
|
|
33514
|
-
|
|
33515
|
-
|
|
33516
|
-
|
|
33517
|
-
|
|
33518
|
-
|
|
33519
|
-
|
|
33520
|
-
|
|
33521
|
-
|
|
33522
|
-
|
|
33523
|
-
|
|
33524
|
-
|
|
33525
|
-
|
|
33526
|
-
|
|
33527
|
-
|
|
33528
|
-
|
|
33755
|
+
get retryable() {
|
|
33756
|
+
return this.kind === "rate-limit" || this.kind === "unavailable";
|
|
33757
|
+
}
|
|
33758
|
+
};
|
|
33759
|
+
const TYPESAFE_URL = "https://api.typesafe.ai/v1/systemone";
|
|
33760
|
+
const JEV_PRICE_PER_MILLION = .042;
|
|
33761
|
+
const REQUEST_TIMEOUT_MS = 6e4;
|
|
33762
|
+
const MAX_RESPONSE_BYTES = 1024 * 1024;
|
|
33763
|
+
function retryAfter(response) {
|
|
33764
|
+
const value = Number(response.headers.get("retry-after"));
|
|
33765
|
+
return Number.isFinite(value) && value > 0 ? Math.min(value, 30) * 1e3 : void 0;
|
|
33766
|
+
}
|
|
33767
|
+
function typesafeDecisionModel(input) {
|
|
33768
|
+
const call = input.fetch ?? fetch;
|
|
33769
|
+
const model = input.model ?? "jev-latest";
|
|
33770
|
+
return {
|
|
33771
|
+
provider: "typesafe",
|
|
33772
|
+
name: model,
|
|
33773
|
+
pricePerMillionInputTokens: JEV_PRICE_PER_MILLION,
|
|
33774
|
+
async ask(request, signal) {
|
|
33775
|
+
const controller = new AbortController();
|
|
33776
|
+
const abort = () => controller.abort();
|
|
33777
|
+
const timer = setTimeout(abort, REQUEST_TIMEOUT_MS);
|
|
33778
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
33779
|
+
let response;
|
|
33780
|
+
try {
|
|
33781
|
+
response = await call(TYPESAFE_URL, {
|
|
33782
|
+
method: "POST",
|
|
33783
|
+
headers: {
|
|
33784
|
+
authorization: `Bearer ${input.apiKey}`,
|
|
33785
|
+
"content-type": "application/json"
|
|
33786
|
+
},
|
|
33787
|
+
body: JSON.stringify({
|
|
33788
|
+
model,
|
|
33789
|
+
...request
|
|
33790
|
+
}),
|
|
33791
|
+
signal: controller.signal
|
|
33792
|
+
});
|
|
33793
|
+
} catch (error) {
|
|
33794
|
+
throw new DecisionModelError("unavailable", `TypeSafe could not be reached: ${error instanceof Error ? error.message : String(error)}`);
|
|
33795
|
+
} finally {
|
|
33796
|
+
clearTimeout(timer);
|
|
33797
|
+
signal?.removeEventListener("abort", abort);
|
|
33798
|
+
}
|
|
33799
|
+
if (response.status === 401 || response.status === 403) {
|
|
33800
|
+
await response.body?.cancel();
|
|
33801
|
+
throw new DecisionModelError("auth", "TypeSafe rejected the API key. Replace it at https://seoskill.dev/account#keys or in TYPESAFE_API_KEY.");
|
|
33802
|
+
}
|
|
33803
|
+
if (response.status === 429 || response.status === 529) {
|
|
33804
|
+
await response.body?.cancel();
|
|
33805
|
+
throw new DecisionModelError("rate-limit", `TypeSafe returned HTTP ${response.status}.`, retryAfter(response));
|
|
33806
|
+
}
|
|
33807
|
+
if (response.status >= 500) {
|
|
33808
|
+
await response.body?.cancel();
|
|
33809
|
+
throw new DecisionModelError("unavailable", `TypeSafe returned HTTP ${response.status}.`);
|
|
33810
|
+
}
|
|
33811
|
+
if (!response.ok) {
|
|
33812
|
+
await response.body?.cancel();
|
|
33813
|
+
throw new DecisionModelError("invalid", `TypeSafe did not accept the request (HTTP ${response.status}).`);
|
|
33814
|
+
}
|
|
33815
|
+
const body = await readJson(response, "TypeSafe", { maxBytes: MAX_RESPONSE_BYTES });
|
|
33816
|
+
return {
|
|
33817
|
+
model: body.model ?? model,
|
|
33818
|
+
answers: body.answers ?? {},
|
|
33819
|
+
inputTokens: body.usage?.input_tokens ?? 0
|
|
33820
|
+
};
|
|
33821
|
+
}
|
|
33529
33822
|
};
|
|
33530
33823
|
}
|
|
33531
|
-
|
|
33532
|
-
|
|
33533
|
-
const
|
|
33534
|
-
|
|
33535
|
-
|
|
33536
|
-
|
|
33537
|
-
|
|
33538
|
-
|
|
33539
|
-
|
|
33540
|
-
|
|
33541
|
-
|
|
33542
|
-
|
|
33543
|
-
|
|
33544
|
-
|
|
33545
|
-
|
|
33546
|
-
|
|
33547
|
-
|
|
33548
|
-
|
|
33549
|
-
|
|
33550
|
-
current.clicks += row.clicks;
|
|
33551
|
-
current.impressions += row.impressions;
|
|
33552
|
-
if (compareInternalLinkText(query, current.query) < 0) current.query = query;
|
|
33553
|
-
map.set(key, current);
|
|
33554
|
-
}
|
|
33555
|
-
function bestMatch(input) {
|
|
33556
|
-
const sourceKey = normalizeInternalLinkQuery(input.source.query);
|
|
33557
|
-
const exact = input.exactTargets.get(sourceKey);
|
|
33558
|
-
if (exact) return {
|
|
33559
|
-
sourceQuery: input.source.query,
|
|
33560
|
-
targetQuery: exact.query,
|
|
33561
|
-
kind: "exact-query",
|
|
33562
|
-
relevanceScore: 1,
|
|
33563
|
-
impressions: input.source.impressions,
|
|
33564
|
-
sharedTerms: internalLinkTokens(input.source.query)
|
|
33565
|
-
};
|
|
33566
|
-
return input.lexicalTargets.flatMap((target) => {
|
|
33567
|
-
const score = input.scoreLexical(input.source.query, target.query);
|
|
33568
|
-
return score ? [{
|
|
33569
|
-
sourceQuery: input.source.query,
|
|
33570
|
-
targetQuery: target.query,
|
|
33571
|
-
kind: "lexical-review",
|
|
33572
|
-
relevanceScore: score.score,
|
|
33573
|
-
impressions: input.source.impressions,
|
|
33574
|
-
sharedTerms: score.sharedTerms
|
|
33575
|
-
}] : [];
|
|
33576
|
-
}).sort((left, right) => right.relevanceScore - left.relevanceScore || compareInternalLinkText(left.targetQuery, right.targetQuery))[0];
|
|
33577
|
-
}
|
|
33578
|
-
function candidatesFromMatches(matchesByUrl) {
|
|
33579
|
-
return [...matchesByUrl.entries()].map(([sourceUrl, matches]) => {
|
|
33580
|
-
const ordered = [...matches].sort((left, right) => (left.kind === right.kind ? 0 : left.kind === "exact-query" ? -1 : 1) || right.impressions - left.impressions || right.relevanceScore - left.relevanceScore || compareInternalLinkText(left.sourceQuery, right.sourceQuery));
|
|
33824
|
+
const MAX_CONCURRENCY = 32;
|
|
33825
|
+
const MIN_CONCURRENCY = 2;
|
|
33826
|
+
const MAX_ATTEMPTS = 5;
|
|
33827
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
33828
|
+
var DecisionRunner = class {
|
|
33829
|
+
model;
|
|
33830
|
+
limit = MAX_CONCURRENCY;
|
|
33831
|
+
active = 0;
|
|
33832
|
+
streak = 0;
|
|
33833
|
+
waiting = [];
|
|
33834
|
+
served = "";
|
|
33835
|
+
requests = 0;
|
|
33836
|
+
failed = 0;
|
|
33837
|
+
tokens = 0;
|
|
33838
|
+
authError = null;
|
|
33839
|
+
constructor(model) {
|
|
33840
|
+
this.model = model;
|
|
33841
|
+
}
|
|
33842
|
+
usage() {
|
|
33581
33843
|
return {
|
|
33582
|
-
|
|
33583
|
-
|
|
33584
|
-
|
|
33585
|
-
|
|
33586
|
-
|
|
33587
|
-
|
|
33588
|
-
matches: ordered.slice(0, 10)
|
|
33844
|
+
provider: this.model.provider,
|
|
33845
|
+
model: this.served || this.model.name,
|
|
33846
|
+
requests: this.requests,
|
|
33847
|
+
failedRequests: this.failed,
|
|
33848
|
+
inputTokens: this.tokens,
|
|
33849
|
+
costUsd: Math.round(this.tokens * this.model.pricePerMillionInputTokens / 1e6 * 1e4) / 1e4
|
|
33589
33850
|
};
|
|
33590
|
-
}
|
|
33591
|
-
|
|
33592
|
-
|
|
33593
|
-
|
|
33594
|
-
|
|
33595
|
-
|
|
33596
|
-
|
|
33597
|
-
|
|
33598
|
-
|
|
33599
|
-
|
|
33600
|
-
|
|
33601
|
-
|
|
33602
|
-
|
|
33603
|
-
|
|
33604
|
-
|
|
33605
|
-
|
|
33606
|
-
|
|
33607
|
-
|
|
33608
|
-
|
|
33609
|
-
|
|
33610
|
-
|
|
33851
|
+
}
|
|
33852
|
+
async slot() {
|
|
33853
|
+
while (this.active >= this.limit) await new Promise((resolve) => this.waiting.push(resolve));
|
|
33854
|
+
this.active += 1;
|
|
33855
|
+
}
|
|
33856
|
+
release() {
|
|
33857
|
+
this.active -= 1;
|
|
33858
|
+
this.waiting.shift()?.();
|
|
33859
|
+
}
|
|
33860
|
+
async ask(request) {
|
|
33861
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
|
|
33862
|
+
if (this.authError) break;
|
|
33863
|
+
let delay = 0;
|
|
33864
|
+
await this.slot();
|
|
33865
|
+
try {
|
|
33866
|
+
const response = await this.model.ask(request);
|
|
33867
|
+
this.requests += 1;
|
|
33868
|
+
this.tokens += response.inputTokens;
|
|
33869
|
+
this.served = response.model;
|
|
33870
|
+
this.streak += 1;
|
|
33871
|
+
if (this.streak >= 50 && this.limit < MAX_CONCURRENCY) {
|
|
33872
|
+
this.limit += 1;
|
|
33873
|
+
this.streak = 0;
|
|
33874
|
+
this.waiting.shift()?.();
|
|
33875
|
+
}
|
|
33876
|
+
return response;
|
|
33877
|
+
} catch (error) {
|
|
33878
|
+
const failure = error instanceof DecisionModelError ? error : new DecisionModelError("unavailable", String(error));
|
|
33879
|
+
if (failure.kind === "auth") {
|
|
33880
|
+
this.authError = failure;
|
|
33881
|
+
break;
|
|
33882
|
+
}
|
|
33883
|
+
if (!failure.retryable || attempt === MAX_ATTEMPTS) break;
|
|
33884
|
+
if (failure.kind === "rate-limit") {
|
|
33885
|
+
this.limit = Math.max(MIN_CONCURRENCY, Math.floor(this.limit / 2));
|
|
33886
|
+
this.streak = 0;
|
|
33887
|
+
}
|
|
33888
|
+
delay = failure.retryAfterMs ?? 2 ** attempt * 250 + Math.floor(Math.random() * 250);
|
|
33889
|
+
} finally {
|
|
33890
|
+
this.release();
|
|
33891
|
+
}
|
|
33892
|
+
await sleep(delay);
|
|
33611
33893
|
}
|
|
33612
|
-
|
|
33894
|
+
this.failed += 1;
|
|
33895
|
+
return null;
|
|
33613
33896
|
}
|
|
33614
|
-
|
|
33615
|
-
|
|
33616
|
-
|
|
33617
|
-
|
|
33618
|
-
|
|
33619
|
-
|
|
33620
|
-
|
|
33897
|
+
};
|
|
33898
|
+
//#endregion
|
|
33899
|
+
//#region packages/core/dist/analyze/internal-links-index.js
|
|
33900
|
+
const PAGE_TERMS = 300;
|
|
33901
|
+
const VECTOR_TERMS = 100;
|
|
33902
|
+
var SiteIndex = class {
|
|
33903
|
+
ids = /* @__PURE__ */ new Map();
|
|
33904
|
+
df = [];
|
|
33905
|
+
counts = [];
|
|
33906
|
+
vectors = [];
|
|
33907
|
+
postings = [];
|
|
33908
|
+
scratch = /* @__PURE__ */ new Float64Array(0);
|
|
33909
|
+
get size() {
|
|
33910
|
+
return this.counts.length;
|
|
33911
|
+
}
|
|
33912
|
+
addPage(termCounts) {
|
|
33913
|
+
const kept = Object.entries(termCounts).sort((left, right) => right[1] - left[1] || (left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0)).slice(0, PAGE_TERMS);
|
|
33914
|
+
const ids = new Int32Array(kept.length);
|
|
33915
|
+
const counts = new Float32Array(kept.length);
|
|
33916
|
+
kept.forEach(([term, count], index) => {
|
|
33917
|
+
let id = this.ids.get(term);
|
|
33918
|
+
if (id === void 0) {
|
|
33919
|
+
id = this.df.length;
|
|
33920
|
+
this.ids.set(term, id);
|
|
33921
|
+
this.df.push(0);
|
|
33922
|
+
}
|
|
33923
|
+
this.df[id] = (this.df[id] ?? 0) + 1;
|
|
33924
|
+
ids[index] = id;
|
|
33925
|
+
counts[index] = count;
|
|
33621
33926
|
});
|
|
33622
|
-
|
|
33623
|
-
|
|
33624
|
-
|
|
33625
|
-
|
|
33626
|
-
|
|
33627
|
-
const lexicalTargets = allTargets.slice(0, LEXICAL_TARGET_LIMIT);
|
|
33628
|
-
const exactTargets = new Map(allTargets.map((target) => [normalizeInternalLinkQuery(target.query), target]));
|
|
33629
|
-
const sourceMap = /* @__PURE__ */ new Map();
|
|
33630
|
-
let sourceValidRows = 0;
|
|
33631
|
-
for (const row of input.sourceRows) {
|
|
33632
|
-
if (!validInternalLinkRow(row)) {
|
|
33633
|
-
increment("sourceInvalidRows");
|
|
33634
|
-
continue;
|
|
33635
|
-
}
|
|
33636
|
-
sourceValidRows += 1;
|
|
33637
|
-
const query = row.keys[0] ?? "";
|
|
33638
|
-
const page = normalizeInternalLinkUrl(row.keys[1] ?? "");
|
|
33639
|
-
if (!page) continue;
|
|
33640
|
-
if (aliases.has(page)) {
|
|
33641
|
-
increment("sourceTargetAliasRows");
|
|
33642
|
-
continue;
|
|
33643
|
-
}
|
|
33644
|
-
const key = `${page}\u0000${normalizeInternalLinkQuery(query)}`;
|
|
33645
|
-
const current = sourceMap.get(key) ?? {
|
|
33646
|
-
page,
|
|
33647
|
-
query,
|
|
33648
|
-
clicks: 0,
|
|
33649
|
-
impressions: 0
|
|
33650
|
-
};
|
|
33651
|
-
current.clicks += row.clicks;
|
|
33652
|
-
current.impressions += row.impressions;
|
|
33653
|
-
if (compareInternalLinkText(query, current.query) < 0) current.query = query;
|
|
33654
|
-
sourceMap.set(key, current);
|
|
33927
|
+
this.counts.push({
|
|
33928
|
+
ids,
|
|
33929
|
+
counts
|
|
33930
|
+
});
|
|
33931
|
+
return this.counts.length - 1;
|
|
33655
33932
|
}
|
|
33656
|
-
|
|
33657
|
-
|
|
33658
|
-
|
|
33659
|
-
|
|
33660
|
-
|
|
33661
|
-
|
|
33662
|
-
|
|
33933
|
+
idfOf(id) {
|
|
33934
|
+
return Math.log((this.size + 1) / ((this.df[id] ?? 1) + 1)) + 1;
|
|
33935
|
+
}
|
|
33936
|
+
idf(term) {
|
|
33937
|
+
const id = this.ids.get(term);
|
|
33938
|
+
return id === void 0 ? Math.log((this.size + 1) / 2) + 1 : this.idfOf(id);
|
|
33939
|
+
}
|
|
33940
|
+
finish() {
|
|
33941
|
+
this.vectors = this.counts.map(({ ids, counts }) => {
|
|
33942
|
+
let norm = 0;
|
|
33943
|
+
const shared = [];
|
|
33944
|
+
for (let index = 0; index < ids.length; index += 1) {
|
|
33945
|
+
const id = ids[index] ?? 0;
|
|
33946
|
+
const weight = (1 + Math.log(counts[index] ?? 1)) * this.idfOf(id);
|
|
33947
|
+
norm += weight * weight;
|
|
33948
|
+
if ((this.df[id] ?? 0) >= 2) shared.push([id, weight]);
|
|
33949
|
+
}
|
|
33950
|
+
norm = Math.sqrt(norm) || 1;
|
|
33951
|
+
shared.sort((left, right) => right[1] - left[1] || left[0] - right[0]);
|
|
33952
|
+
const top = shared.slice(0, VECTOR_TERMS);
|
|
33953
|
+
return {
|
|
33954
|
+
ids: Int32Array.from(top.map(([id]) => id)),
|
|
33955
|
+
weights: Float32Array.from(top.map(([, weight]) => weight / norm))
|
|
33956
|
+
};
|
|
33663
33957
|
});
|
|
33664
|
-
|
|
33665
|
-
|
|
33666
|
-
|
|
33667
|
-
|
|
33668
|
-
|
|
33669
|
-
|
|
33670
|
-
|
|
33671
|
-
|
|
33672
|
-
|
|
33673
|
-
|
|
33674
|
-
|
|
33675
|
-
exactTargets,
|
|
33676
|
-
lexicalTargets,
|
|
33677
|
-
scoreLexical
|
|
33958
|
+
const lists = this.df.map(() => ({
|
|
33959
|
+
pages: [],
|
|
33960
|
+
weights: []
|
|
33961
|
+
}));
|
|
33962
|
+
this.vectors.forEach((vector, page) => {
|
|
33963
|
+
vector.ids.forEach((id, index) => {
|
|
33964
|
+
const list = lists[id];
|
|
33965
|
+
if (!list) return;
|
|
33966
|
+
list.pages.push(page);
|
|
33967
|
+
list.weights.push(vector.weights[index] ?? 0);
|
|
33968
|
+
});
|
|
33678
33969
|
});
|
|
33679
|
-
|
|
33680
|
-
|
|
33681
|
-
|
|
33970
|
+
this.postings = lists.map((list) => ({
|
|
33971
|
+
pages: Int32Array.from(list.pages),
|
|
33972
|
+
weights: Float32Array.from(list.weights)
|
|
33973
|
+
}));
|
|
33974
|
+
this.scratch = new Float64Array(this.size);
|
|
33975
|
+
}
|
|
33976
|
+
similarity(left, right) {
|
|
33977
|
+
const a = this.vectors[left];
|
|
33978
|
+
const b = this.vectors[right];
|
|
33979
|
+
if (!a || !b) return 0;
|
|
33980
|
+
const weights = /* @__PURE__ */ new Map();
|
|
33981
|
+
a.ids.forEach((id, index) => {
|
|
33982
|
+
weights.set(id, a.weights[index] ?? 0);
|
|
33983
|
+
});
|
|
33984
|
+
let sum = 0;
|
|
33985
|
+
b.ids.forEach((id, index) => {
|
|
33986
|
+
sum += (weights.get(id) ?? 0) * (b.weights[index] ?? 0);
|
|
33987
|
+
});
|
|
33988
|
+
return sum;
|
|
33989
|
+
}
|
|
33990
|
+
neighbours(page, min) {
|
|
33991
|
+
const vector = this.vectors[page];
|
|
33992
|
+
if (!vector) return [];
|
|
33993
|
+
const scores = this.scratch;
|
|
33994
|
+
const touched = [];
|
|
33995
|
+
vector.ids.forEach((id, index) => {
|
|
33996
|
+
const weight = vector.weights[index] ?? 0;
|
|
33997
|
+
const list = this.postings[id];
|
|
33998
|
+
if (!list) return;
|
|
33999
|
+
for (let at = 0; at < list.pages.length; at += 1) {
|
|
34000
|
+
const other = list.pages[at] ?? 0;
|
|
34001
|
+
if (other === page) continue;
|
|
34002
|
+
if (scores[other] === 0) touched.push(other);
|
|
34003
|
+
scores[other] = (scores[other] ?? 0) + weight * (list.weights[at] ?? 0);
|
|
34004
|
+
}
|
|
34005
|
+
});
|
|
34006
|
+
const out = [];
|
|
34007
|
+
for (const other of touched) {
|
|
34008
|
+
const score = scores[other] ?? 0;
|
|
34009
|
+
if (score >= min) out.push({
|
|
34010
|
+
page: other,
|
|
34011
|
+
score
|
|
34012
|
+
});
|
|
34013
|
+
scores[other] = 0;
|
|
33682
34014
|
}
|
|
33683
|
-
|
|
33684
|
-
const matches = matchesByUrl.get(source.page) ?? [];
|
|
33685
|
-
matches.push(match);
|
|
33686
|
-
matchesByUrl.set(source.page, matches);
|
|
34015
|
+
return out.sort((left, right) => right.score - left.score || left.page - right.page);
|
|
33687
34016
|
}
|
|
33688
|
-
|
|
33689
|
-
|
|
33690
|
-
|
|
33691
|
-
targetQueries: lexicalTargets,
|
|
33692
|
-
candidates,
|
|
33693
|
-
selection: {
|
|
33694
|
-
targetSourceRows: input.targetRows.length,
|
|
33695
|
-
targetValidRows,
|
|
33696
|
-
targetInvalidRows: value("targetInvalidRows"),
|
|
33697
|
-
targetUrlMismatchRows: value("targetUrlMismatchRows"),
|
|
33698
|
-
targetLowImpressionQueries: value("targetLowImpressionQueries"),
|
|
33699
|
-
targetLowActionabilityQueries: value("targetLowActionabilityQueries"),
|
|
33700
|
-
targetBrandQueries: value("targetBrandQueries"),
|
|
33701
|
-
targetEligibleQueries: allTargets.length,
|
|
33702
|
-
selectedLexicalTargetQueries: lexicalTargets.length,
|
|
33703
|
-
sourceRows: input.sourceRows.length,
|
|
33704
|
-
sourceValidRows,
|
|
33705
|
-
sourceInvalidRows: value("sourceInvalidRows"),
|
|
33706
|
-
sourceTargetAliasRows: value("sourceTargetAliasRows"),
|
|
33707
|
-
sourceLowImpressionQueries: value("sourceLowImpressionQueries"),
|
|
33708
|
-
sourceLowActionabilityQueries: value("sourceLowActionabilityQueries"),
|
|
33709
|
-
sourceBrandQueries: value("sourceBrandQueries"),
|
|
33710
|
-
sourceUnmatchedQueries: value("sourceUnmatchedQueries"),
|
|
33711
|
-
candidateQueries,
|
|
33712
|
-
candidateUrls: candidates.length
|
|
33713
|
-
}
|
|
33714
|
-
};
|
|
33715
|
-
}
|
|
33716
|
-
const MAX_TERMS = 100;
|
|
33717
|
-
const PLACEMENT_SENTENCES = 2;
|
|
33718
|
-
const ANCHORS_PER_SENTENCE = 3;
|
|
34017
|
+
};
|
|
34018
|
+
const PAIR_SENTENCES = 4;
|
|
34019
|
+
const PAIR_ANCHORS = 8;
|
|
33719
34020
|
const MIN_SENTENCE_WORDS = 6;
|
|
33720
34021
|
const NAME_WEIGHT = 2;
|
|
33721
34022
|
const GENERIC_ANCHORS = /^(click here|here|read more|learn more|more|this|this post|this article|this page|link|website|check it out|see more|view|continue reading)$/i;
|
|
@@ -33985,65 +34286,49 @@ function pageSentences(html) {
|
|
|
33985
34286
|
flushRaw("");
|
|
33986
34287
|
return sentences;
|
|
33987
34288
|
}
|
|
33988
|
-
|
|
33989
|
-
|
|
33990
|
-
|
|
33991
|
-
|
|
33992
|
-
|
|
33993
|
-
|
|
33994
|
-
|
|
33995
|
-
|
|
33996
|
-
return Math.log((this.size + 1) / ((this.df.get(term) ?? 1) + 1)) + 1;
|
|
33997
|
-
}
|
|
33998
|
-
vector(counts) {
|
|
33999
|
-
let norm = 0;
|
|
34000
|
-
const shared = [];
|
|
34001
|
-
for (const term in counts) {
|
|
34002
|
-
const weight = (1 + Math.log(counts[term] ?? 1)) * this.idf(term);
|
|
34003
|
-
norm += weight * weight;
|
|
34004
|
-
if ((this.df.get(term) ?? 0) >= 2) shared.push([term, weight]);
|
|
34005
|
-
}
|
|
34006
|
-
norm = Math.sqrt(norm) || 1;
|
|
34007
|
-
shared.sort((left, right) => right[1] - left[1] || (left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0));
|
|
34008
|
-
return new Map(shared.slice(0, MAX_TERMS).map(([term, weight]) => [term, weight / norm]));
|
|
34009
|
-
}
|
|
34010
|
-
similarity(left, right) {
|
|
34011
|
-
let sum = 0;
|
|
34012
|
-
for (const [term, weight] of left) sum += weight * (right.get(term) ?? 0);
|
|
34013
|
-
return sum;
|
|
34289
|
+
function linkableSentences(sentences) {
|
|
34290
|
+
const seen = /* @__PURE__ */ new Set();
|
|
34291
|
+
const out = [];
|
|
34292
|
+
for (const sentence of sentences) {
|
|
34293
|
+
if (out.length >= 150) break;
|
|
34294
|
+
if (seen.has(sentence.text)) continue;
|
|
34295
|
+
seen.add(sentence.text);
|
|
34296
|
+
if (!sentence.hasLink && sentence.text.split(/\s+/).length >= MIN_SENTENCE_WORDS) out.push(sentence);
|
|
34014
34297
|
}
|
|
34015
|
-
|
|
34016
|
-
|
|
34017
|
-
|
|
34018
|
-
|
|
34019
|
-
|
|
34020
|
-
|
|
34021
|
-
|
|
34022
|
-
|
|
34023
|
-
|
|
34024
|
-
|
|
34025
|
-
|
|
34298
|
+
return out;
|
|
34299
|
+
}
|
|
34300
|
+
function sentenceMatch(sentence, key, weights) {
|
|
34301
|
+
if (!key.size) return 0;
|
|
34302
|
+
const seen = new Set(terms(sentence));
|
|
34303
|
+
let hit = 0;
|
|
34304
|
+
let total = 0;
|
|
34305
|
+
for (const [term, place] of key) {
|
|
34306
|
+
const weight = weights.idf(term) * place;
|
|
34307
|
+
total += weight;
|
|
34308
|
+
if (seen.has(term)) hit += weight;
|
|
34026
34309
|
}
|
|
34027
|
-
|
|
34028
|
-
|
|
34310
|
+
return total ? hit / total : 0;
|
|
34311
|
+
}
|
|
34312
|
+
function anchorCandidates(sentence, key, weights, limit) {
|
|
34029
34313
|
const scored = /* @__PURE__ */ new Map();
|
|
34030
34314
|
const parts = sentence.split(/\s+/).map((word) => word.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "")).filter(Boolean);
|
|
34031
|
-
for (let size =
|
|
34315
|
+
for (let size = 1; size <= 4; size += 1) for (let start = 0; start + size <= parts.length; start += 1) {
|
|
34032
34316
|
const gram = parts.slice(start, start + size);
|
|
34033
34317
|
const first = (gram[0] ?? "").toLowerCase();
|
|
34034
34318
|
const last = (gram[size - 1] ?? "").toLowerCase();
|
|
34035
34319
|
if (FUNCTION_WORDS.has(first) || FUNCTION_WORDS.has(last) || /^\d+$/.test(first) || /^\d+$/.test(last)) continue;
|
|
34036
34320
|
const stems = terms(gram.join(" "));
|
|
34037
|
-
if (stems.length < 2) continue;
|
|
34321
|
+
if (stems.length < size || size > 1 && stems.length < 2) continue;
|
|
34322
|
+
if (size === 1 && key.get(stems[0] ?? "") !== NAME_WEIGHT) continue;
|
|
34038
34323
|
const hits = stems.filter((term) => key.has(term)).length;
|
|
34039
|
-
const named = stems.reduce((sum, term) => sum + (key.get(term) ?? 0) *
|
|
34324
|
+
const named = stems.reduce((sum, term) => sum + (key.get(term) ?? 0) * weights.idf(term), 0);
|
|
34040
34325
|
if (!stems.some((term) => key.get(term) === NAME_WEIGHT)) continue;
|
|
34041
34326
|
if (Number(key.has(terms(first)[0] ?? "")) + Number(key.has(terms(last)[0] ?? "")) < 2) continue;
|
|
34042
34327
|
const text = gram.join(" ");
|
|
34043
34328
|
const lower = text.toLowerCase();
|
|
34044
34329
|
if (GENERIC_ANCHORS.test(lower)) continue;
|
|
34045
|
-
const rarity = stems.reduce((sum, term) => sum +
|
|
34046
|
-
const score = named * (1 + hits / key.size) + rarity * .05 - (size === 4 ? .2 : 0);
|
|
34330
|
+
const rarity = stems.reduce((sum, term) => sum + weights.idf(term), 0);
|
|
34331
|
+
const score = named * (1 + hits / key.size) + rarity * .05 - (size === 1 ? .5 : size === 4 ? .2 : 0);
|
|
34047
34332
|
const existing = scored.get(lower);
|
|
34048
34333
|
if (!existing || existing.score < score) scored.set(lower, {
|
|
34049
34334
|
text,
|
|
@@ -34052,27 +34337,27 @@ function anchorCandidates(sentence, key, corpus, limit) {
|
|
|
34052
34337
|
}
|
|
34053
34338
|
return [...scored.values()].sort((left, right) => right.score - left.score || (left.text < right.text ? -1 : left.text > right.text ? 1 : 0)).slice(0, limit).map((item) => item.text);
|
|
34054
34339
|
}
|
|
34055
|
-
function
|
|
34340
|
+
function pairPlacement(sentences, target, weights) {
|
|
34056
34341
|
const key = keyTerms(target);
|
|
34057
|
-
const
|
|
34058
|
-
const seen = /* @__PURE__ */ new Set();
|
|
34059
|
-
const sentences = source.sentences.filter((sentence) => {
|
|
34060
|
-
if (seen.has(sentence.text)) return false;
|
|
34061
|
-
seen.add(sentence.text);
|
|
34062
|
-
return !sentence.hasLink && sentence.text.split(/\s+/).length >= MIN_SENTENCE_WORDS;
|
|
34063
|
-
}).map((sentence, index) => ({
|
|
34342
|
+
const chosen = sentences.map((sentence, index) => ({
|
|
34064
34343
|
sentence,
|
|
34065
34344
|
index,
|
|
34066
|
-
match:
|
|
34067
|
-
})).filter((item) => item.match > 0).sort((left, right) => right.match - left.match || left.index - right.index).slice(0,
|
|
34068
|
-
|
|
34069
|
-
|
|
34070
|
-
|
|
34071
|
-
|
|
34072
|
-
|
|
34345
|
+
match: sentenceMatch(sentence.text, key, weights)
|
|
34346
|
+
})).filter((item) => item.match > 0).sort((left, right) => right.match - left.match || left.index - right.index).slice(0, PAIR_SENTENCES);
|
|
34347
|
+
const perSentence = Math.max(1, Math.floor(PAIR_ANCHORS / Math.max(1, chosen.length)));
|
|
34348
|
+
const anchors = [];
|
|
34349
|
+
for (const { sentence, index } of chosen) for (const text of anchorCandidates(sentence.text, key, weights, perSentence)) anchors.push({
|
|
34350
|
+
text,
|
|
34351
|
+
sentence: `S${index + 1}`
|
|
34352
|
+
});
|
|
34073
34353
|
return {
|
|
34074
|
-
|
|
34075
|
-
|
|
34354
|
+
sentences: chosen.map(({ sentence, index, match }) => ({
|
|
34355
|
+
id: `S${index + 1}`,
|
|
34356
|
+
text: sentence.text,
|
|
34357
|
+
section: sentence.section,
|
|
34358
|
+
match: Math.round(match * 100) / 100
|
|
34359
|
+
})),
|
|
34360
|
+
anchors: anchors.slice(0, PAIR_ANCHORS)
|
|
34076
34361
|
};
|
|
34077
34362
|
}
|
|
34078
34363
|
function placementPage(input) {
|
|
@@ -34086,542 +34371,1140 @@ function placementPage(input) {
|
|
|
34086
34371
|
};
|
|
34087
34372
|
}
|
|
34088
34373
|
//#endregion
|
|
34089
|
-
//#region packages/core/dist/analyze/internal-links-
|
|
34090
|
-
const
|
|
34091
|
-
const
|
|
34092
|
-
|
|
34093
|
-
|
|
34094
|
-
|
|
34095
|
-
|
|
34096
|
-
|
|
34097
|
-
|
|
34098
|
-
|
|
34099
|
-
|
|
34100
|
-
|
|
34101
|
-
|
|
34102
|
-
|
|
34103
|
-
|
|
34104
|
-
|
|
34105
|
-
|
|
34106
|
-
|
|
34107
|
-
|
|
34108
|
-
|
|
34109
|
-
|
|
34110
|
-
}
|
|
34111
|
-
|
|
34112
|
-
|
|
34113
|
-
|
|
34114
|
-
|
|
34115
|
-
|
|
34116
|
-
|
|
34117
|
-
|
|
34118
|
-
|
|
34119
|
-
|
|
34374
|
+
//#region packages/core/dist/analyze/internal-links-crawl.js
|
|
34375
|
+
const MAX_MAX_PAGES = 5e3;
|
|
34376
|
+
const FETCH_CONCURRENCY = 6;
|
|
34377
|
+
const MAX_CRAWL_DELAY_SECONDS = 10;
|
|
34378
|
+
const CRAWL_DELAY_BUDGET_SECONDS = 900;
|
|
34379
|
+
const REDIRECT_CHECKS = 150;
|
|
34380
|
+
const GENERIC_ANCHOR_LIMIT = 100;
|
|
34381
|
+
const EXCERPT_CHARS = 500;
|
|
34382
|
+
const ASSET = /\.(avif|bmp|css|csv|dmg|docx?|eot|gif|gz|ico|jpe?g|js|json|map|mov|mp3|mp4|ogg|otf|pdf|png|pptx?|rss|svg|tar|ttf|txt|wav|webm|webp|woff2?|xlsx?|xml|zip)$/i;
|
|
34383
|
+
const TRACKING = /^(utm_[a-z]+|gclid|fbclid|msclkid|mc_cid|mc_eid|ref)$/i;
|
|
34384
|
+
var SentenceStore = class SentenceStore {
|
|
34385
|
+
dir;
|
|
34386
|
+
handle;
|
|
34387
|
+
size = 0;
|
|
34388
|
+
constructor(dir, handle) {
|
|
34389
|
+
this.dir = dir;
|
|
34390
|
+
this.handle = handle;
|
|
34391
|
+
}
|
|
34392
|
+
static async open() {
|
|
34393
|
+
const dir = await mkdtemp(join(tmpdir(), "seo-internal-links-"));
|
|
34394
|
+
return new SentenceStore(dir, await open$1(join(dir, "sentences"), "w+"));
|
|
34395
|
+
}
|
|
34396
|
+
async write(sentences) {
|
|
34397
|
+
const bytes = Buffer.from(JSON.stringify(sentences.map((s) => [s.text, s.section])));
|
|
34398
|
+
const offset = this.size;
|
|
34399
|
+
this.size += bytes.length;
|
|
34400
|
+
await this.handle.write(bytes, 0, bytes.length, offset);
|
|
34401
|
+
return {
|
|
34402
|
+
offset,
|
|
34403
|
+
length: bytes.length
|
|
34404
|
+
};
|
|
34120
34405
|
}
|
|
34121
|
-
|
|
34122
|
-
|
|
34123
|
-
|
|
34124
|
-
|
|
34125
|
-
|
|
34126
|
-
|
|
34127
|
-
|
|
34128
|
-
|
|
34129
|
-
|
|
34130
|
-
if (!value) return void 0;
|
|
34131
|
-
try {
|
|
34132
|
-
return assertUrlMatchesGscProperty(site, value);
|
|
34133
|
-
} catch {
|
|
34134
|
-
return;
|
|
34406
|
+
async read(ref) {
|
|
34407
|
+
if (!ref.length) return [];
|
|
34408
|
+
const bytes = Buffer.alloc(ref.length);
|
|
34409
|
+
await this.handle.read(bytes, 0, ref.length, ref.offset);
|
|
34410
|
+
return JSON.parse(bytes.toString("utf8")).map(([text, section]) => ({
|
|
34411
|
+
text,
|
|
34412
|
+
section,
|
|
34413
|
+
hasLink: false
|
|
34414
|
+
}));
|
|
34135
34415
|
}
|
|
34136
|
-
|
|
34137
|
-
|
|
34138
|
-
|
|
34139
|
-
|
|
34140
|
-
|
|
34141
|
-
}
|
|
34142
|
-
function sameObservedUrl(left, right) {
|
|
34143
|
-
try {
|
|
34144
|
-
return withoutFragment(left) === withoutFragment(right);
|
|
34145
|
-
} catch {
|
|
34146
|
-
return false;
|
|
34416
|
+
async close() {
|
|
34417
|
+
await this.handle.close().catch(() => void 0);
|
|
34418
|
+
await rm(this.dir, {
|
|
34419
|
+
recursive: true,
|
|
34420
|
+
force: true
|
|
34421
|
+
});
|
|
34147
34422
|
}
|
|
34148
|
-
}
|
|
34149
|
-
|
|
34150
|
-
|
|
34423
|
+
};
|
|
34424
|
+
const bareHost = (host) => host.toLowerCase().replace(/^www\./, "");
|
|
34425
|
+
function crawlUrl(value, base) {
|
|
34151
34426
|
try {
|
|
34152
|
-
|
|
34427
|
+
const url = base ? new URL(value, base) : new URL(value);
|
|
34428
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
34429
|
+
url.hash = "";
|
|
34430
|
+
for (const key of [...url.searchParams.keys()]) if (TRACKING.test(key)) url.searchParams.delete(key);
|
|
34431
|
+
return url.toString();
|
|
34153
34432
|
} catch {
|
|
34154
34433
|
return;
|
|
34155
34434
|
}
|
|
34156
34435
|
}
|
|
34157
|
-
|
|
34158
|
-
let
|
|
34159
|
-
|
|
34160
|
-
|
|
34161
|
-
|
|
34162
|
-
|
|
34163
|
-
|
|
34164
|
-
|
|
34436
|
+
function excerptOf(sentences) {
|
|
34437
|
+
let out = "";
|
|
34438
|
+
for (const sentence of sentences) {
|
|
34439
|
+
const next = out ? `${out} ${sentence.text}` : sentence.text;
|
|
34440
|
+
if (next.length > EXCERPT_CHARS) break;
|
|
34441
|
+
out = next;
|
|
34442
|
+
}
|
|
34443
|
+
return out || (sentences[0]?.text.slice(0, EXCERPT_CHARS) ?? "");
|
|
34444
|
+
}
|
|
34445
|
+
function readPage(fetched, finalUrl) {
|
|
34446
|
+
const $ = load(fetched.html);
|
|
34447
|
+
const meta = (name) => $(`meta[name="${name}" i]`).map((_, element) => $(element).attr("content") ?? "").get();
|
|
34448
|
+
const headings = $("h1, h2, h3").map((_, element) => ({
|
|
34449
|
+
level: Number(element.tagName.slice(1)),
|
|
34450
|
+
text: $(element).text().replace(/\s+/g, " ").trim()
|
|
34451
|
+
})).get().filter((heading) => heading.text);
|
|
34452
|
+
const links = $("a[href]").map((_, element) => {
|
|
34453
|
+
const anchor = $(element);
|
|
34165
34454
|
return {
|
|
34166
|
-
|
|
34167
|
-
|
|
34168
|
-
|
|
34169
|
-
verification: "failed",
|
|
34170
|
-
technicalSignals: [],
|
|
34171
|
-
warnings: [{
|
|
34172
|
-
stage: "target-fetch",
|
|
34173
|
-
url: input.targetUrl,
|
|
34174
|
-
code: "fetch-failed",
|
|
34175
|
-
message: errorMessage(error)
|
|
34176
|
-
}]
|
|
34455
|
+
href: anchor.attr("href") ?? "",
|
|
34456
|
+
text: anchor.text().replace(/\s+/g, " ").trim(),
|
|
34457
|
+
location: anchor.closest("nav, header").length ? "navigation" : anchor.closest("footer").length ? "footer" : anchor.closest("main, article, [role=\"main\"]").length ? "main-content" : "other"
|
|
34177
34458
|
};
|
|
34178
|
-
}
|
|
34179
|
-
|
|
34459
|
+
}).get();
|
|
34460
|
+
const robots = [...meta("robots"), ...meta("googlebot")];
|
|
34461
|
+
return {
|
|
34462
|
+
title: $("title").first().text().replace(/\s+/g, " ").trim(),
|
|
34463
|
+
metaDescription: meta("description")[0]?.trim() ?? "",
|
|
34464
|
+
canonical: $("link[rel~=\"canonical\" i]").first().attr("href"),
|
|
34465
|
+
lang: $("html").attr("lang"),
|
|
34466
|
+
headings,
|
|
34467
|
+
links,
|
|
34468
|
+
nofollow: robots.some((value) => /\bnofollow\b/i.test(value)),
|
|
34469
|
+
noindex: hasMetaRobotsDirective(robots, "noindex") || hasXRobotsDirective(fetched.headers["x-robots-tag"], "noindex"),
|
|
34470
|
+
url: finalUrl
|
|
34471
|
+
};
|
|
34472
|
+
}
|
|
34473
|
+
async function crawlForLinks(input) {
|
|
34474
|
+
const warnings = [];
|
|
34475
|
+
const robots = createRobotsSession({ refresh: input.refresh });
|
|
34476
|
+
const sentences = await SentenceStore.open();
|
|
34477
|
+
const renderer = createPageRenderer({ concurrency: 2 });
|
|
34180
34478
|
try {
|
|
34181
|
-
|
|
34479
|
+
return await crawl(input, robots, sentences, renderer, warnings);
|
|
34182
34480
|
} catch (error) {
|
|
34183
|
-
|
|
34184
|
-
|
|
34185
|
-
|
|
34186
|
-
|
|
34187
|
-
status: fetched.status,
|
|
34188
|
-
aliases: [input.targetUrl],
|
|
34189
|
-
verification: "failed",
|
|
34190
|
-
technicalSignals: [],
|
|
34191
|
-
fetchDiagnostics: fetched.diagnostics,
|
|
34192
|
-
warnings: [{
|
|
34193
|
-
stage: "target-extract",
|
|
34194
|
-
url: input.targetUrl,
|
|
34195
|
-
code: "extract-failed",
|
|
34196
|
-
message: errorMessage(error)
|
|
34197
|
-
}]
|
|
34198
|
-
};
|
|
34481
|
+
await sentences.close();
|
|
34482
|
+
throw error;
|
|
34483
|
+
} finally {
|
|
34484
|
+
await renderer.close();
|
|
34199
34485
|
}
|
|
34200
|
-
|
|
34201
|
-
|
|
34202
|
-
const
|
|
34203
|
-
|
|
34204
|
-
|
|
34205
|
-
|
|
34206
|
-
|
|
34207
|
-
|
|
34208
|
-
|
|
34209
|
-
|
|
34210
|
-
|
|
34211
|
-
|
|
34212
|
-
|
|
34213
|
-
|
|
34486
|
+
}
|
|
34487
|
+
async function crawl(input, robots, sentences, renderer, warnings) {
|
|
34488
|
+
const start = crawlUrl(input.startUrl) ?? input.startUrl;
|
|
34489
|
+
let origin = new URL(start).origin;
|
|
34490
|
+
const site = bareHost(new URL(start).host);
|
|
34491
|
+
const sameSite = (url) => {
|
|
34492
|
+
try {
|
|
34493
|
+
return bareHost(new URL(url).host) === site;
|
|
34494
|
+
} catch {
|
|
34495
|
+
return false;
|
|
34496
|
+
}
|
|
34497
|
+
};
|
|
34498
|
+
const urls = [];
|
|
34499
|
+
const urlIds = /* @__PURE__ */ new Map();
|
|
34500
|
+
const idOf = (url) => {
|
|
34501
|
+
let id = urlIds.get(url);
|
|
34502
|
+
if (id === void 0) {
|
|
34503
|
+
id = urls.length;
|
|
34504
|
+
urls.push(url);
|
|
34505
|
+
urlIds.set(url, id);
|
|
34506
|
+
}
|
|
34507
|
+
return id;
|
|
34508
|
+
};
|
|
34509
|
+
const aliases = /* @__PURE__ */ new Map();
|
|
34510
|
+
const pageOfUrl = /* @__PURE__ */ new Map();
|
|
34511
|
+
const pages = [];
|
|
34512
|
+
const index = new SiteIndex();
|
|
34513
|
+
const genericAnchors = [];
|
|
34514
|
+
const skipped = {};
|
|
34515
|
+
const skip = (reason) => {
|
|
34516
|
+
skipped[reason] = (skipped[reason] ?? 0) + 1;
|
|
34517
|
+
};
|
|
34518
|
+
const rootRobots = await robots.resolve(origin, start);
|
|
34519
|
+
const delay = await robots.crawlDelay(origin);
|
|
34520
|
+
const crawlDelaySeconds = delay === void 0 ? null : Math.min(delay, MAX_CRAWL_DELAY_SECONDS);
|
|
34521
|
+
const pageLimit = crawlDelaySeconds === null ? input.maxPages : Math.min(input.maxPages, Math.max(20, Math.floor(CRAWL_DELAY_BUDGET_SECONDS / crawlDelaySeconds)));
|
|
34522
|
+
if (pageLimit < input.maxPages) warnings.push(`robots.txt asks for ${crawlDelaySeconds} seconds between requests, so the crawl stopped at ${pageLimit} pages to finish in about ${CRAWL_DELAY_BUDGET_SECONDS / 60} minutes.`);
|
|
34523
|
+
const rate = crawlDelaySeconds === null ? {
|
|
34524
|
+
concurrency: FETCH_CONCURRENCY,
|
|
34525
|
+
intervalCap: FETCH_CONCURRENCY,
|
|
34526
|
+
intervalMs: 1e3
|
|
34527
|
+
} : {
|
|
34528
|
+
concurrency: 1,
|
|
34529
|
+
intervalCap: 1,
|
|
34530
|
+
intervalMs: Math.ceil(crawlDelaySeconds * 1e3)
|
|
34531
|
+
};
|
|
34532
|
+
const options = {
|
|
34533
|
+
js: "auto",
|
|
34534
|
+
refresh: input.refresh,
|
|
34535
|
+
respectRobots: true,
|
|
34536
|
+
robotsResolver: robots.resolve,
|
|
34537
|
+
renderer,
|
|
34538
|
+
rate
|
|
34539
|
+
};
|
|
34540
|
+
const stats = (extra = {}) => ({
|
|
34541
|
+
maxPages: input.maxPages,
|
|
34542
|
+
pageLimit,
|
|
34543
|
+
urlsFound: 0,
|
|
34544
|
+
pagesCrawled: pages.length,
|
|
34545
|
+
capReached: false,
|
|
34546
|
+
skipped,
|
|
34547
|
+
crawlDelaySeconds,
|
|
34548
|
+
blockedByRobots: false,
|
|
34549
|
+
redirectsChecked: 0,
|
|
34550
|
+
...extra
|
|
34214
34551
|
});
|
|
34215
|
-
|
|
34216
|
-
|
|
34217
|
-
|
|
34218
|
-
|
|
34219
|
-
|
|
34220
|
-
|
|
34221
|
-
|
|
34222
|
-
|
|
34223
|
-
|
|
34224
|
-
|
|
34225
|
-
|
|
34226
|
-
|
|
34227
|
-
|
|
34228
|
-
|
|
34229
|
-
|
|
34230
|
-
|
|
34231
|
-
|
|
34552
|
+
if (rootRobots.allowed === false) return {
|
|
34553
|
+
origin,
|
|
34554
|
+
method: "links",
|
|
34555
|
+
pages,
|
|
34556
|
+
urls,
|
|
34557
|
+
pageOfUrl,
|
|
34558
|
+
index,
|
|
34559
|
+
sentences,
|
|
34560
|
+
genericAnchors,
|
|
34561
|
+
stats: stats({ blockedByRobots: true }),
|
|
34562
|
+
warnings: [`robots.txt does not allow this crawler to read ${start}.`]
|
|
34563
|
+
};
|
|
34564
|
+
const declared = await robots.sitemapUrls(origin);
|
|
34565
|
+
const sitemap = await sitemapSeeds({
|
|
34566
|
+
url: origin,
|
|
34567
|
+
maxPages: pageLimit * 2,
|
|
34568
|
+
declaredSitemapUrls: declared,
|
|
34232
34569
|
warnings,
|
|
34233
|
-
|
|
34570
|
+
fetchSitemapUrls
|
|
34571
|
+
});
|
|
34572
|
+
const seeds = sitemap.urls.map((url) => crawlUrl(url)).filter((url) => Boolean(url) && sameSite(url) && !ASSET.test(new URL(url).pathname));
|
|
34573
|
+
const method = seeds.length ? "sitemap" : "links";
|
|
34574
|
+
const queue = [];
|
|
34575
|
+
const queued = /* @__PURE__ */ new Set();
|
|
34576
|
+
const maxQueue = pageLimit * 5;
|
|
34577
|
+
const enqueue = (url) => {
|
|
34578
|
+
if (!url || queued.has(url) || queued.size >= maxQueue) return;
|
|
34579
|
+
if (!sameSite(url) || ASSET.test(new URL(url).pathname)) return;
|
|
34580
|
+
queued.add(url);
|
|
34581
|
+
idOf(url);
|
|
34582
|
+
queue.push(url);
|
|
34583
|
+
};
|
|
34584
|
+
enqueue(input.firstUrl ? crawlUrl(input.firstUrl) : void 0);
|
|
34585
|
+
if (method === "sitemap") for (const url of seeds) enqueue(url);
|
|
34586
|
+
else enqueue(start);
|
|
34587
|
+
let overflow = 0;
|
|
34588
|
+
const crawlOne = async (url) => {
|
|
34589
|
+
let fetched;
|
|
34590
|
+
try {
|
|
34591
|
+
fetched = await fetchPage(url, options);
|
|
34592
|
+
} catch (error) {
|
|
34593
|
+
skip(error instanceof Error && error.name === "RobotsAccessError" ? "robots" : "fetch-failed");
|
|
34594
|
+
return;
|
|
34595
|
+
}
|
|
34596
|
+
const finalUrl = crawlUrl(fetched.finalUrl) ?? url;
|
|
34597
|
+
if (finalUrl !== url) aliases.set(idOf(url), idOf(finalUrl));
|
|
34598
|
+
if (!sameSite(finalUrl)) return skip("off-site");
|
|
34599
|
+
if (fetched.status < 200 || fetched.status > 299) return skip("http-error");
|
|
34600
|
+
if (!/html/i.test(fetched.headers["content-type"] ?? "text/html")) return skip("not-html");
|
|
34601
|
+
let page;
|
|
34602
|
+
try {
|
|
34603
|
+
page = readPage(fetched, finalUrl);
|
|
34604
|
+
} catch {
|
|
34605
|
+
return skip("fetch-failed");
|
|
34606
|
+
}
|
|
34607
|
+
const canonical = page.canonical ? crawlUrl(page.canonical, finalUrl) : void 0;
|
|
34608
|
+
if (canonical && sameSite(canonical) && normalizePageUrl(canonical) !== normalizePageUrl(finalUrl)) {
|
|
34609
|
+
aliases.set(idOf(finalUrl), idOf(canonical));
|
|
34610
|
+
enqueue(canonical);
|
|
34611
|
+
return skip("canonicalised");
|
|
34612
|
+
}
|
|
34613
|
+
const finalId = idOf(finalUrl);
|
|
34614
|
+
if (pageOfUrl.has(finalId)) return skip("duplicate");
|
|
34615
|
+
if (pages.length >= pageLimit) {
|
|
34616
|
+
overflow += 1;
|
|
34617
|
+
return;
|
|
34618
|
+
}
|
|
34619
|
+
const text = placementPage({
|
|
34234
34620
|
...page,
|
|
34235
|
-
url:
|
|
34621
|
+
url: finalUrl,
|
|
34236
34622
|
html: fetched.html
|
|
34237
|
-
})
|
|
34623
|
+
});
|
|
34624
|
+
const contextual = /* @__PURE__ */ new Set();
|
|
34625
|
+
const other = /* @__PURE__ */ new Set();
|
|
34626
|
+
const id = pages.length;
|
|
34627
|
+
const hasMain = /<(main|article)[\s>]|role=["']main["']/i.test(fetched.html);
|
|
34628
|
+
for (const link of page.links) {
|
|
34629
|
+
const href = crawlUrl(link.href, finalUrl);
|
|
34630
|
+
if (!href || !sameSite(href) || ASSET.test(new URL(href).pathname)) continue;
|
|
34631
|
+
const hrefId = idOf(href);
|
|
34632
|
+
if (link.location === "main-content" || !hasMain && link.location === "other") {
|
|
34633
|
+
contextual.add(hrefId);
|
|
34634
|
+
if (GENERIC_ANCHORS.test(link.text.trim()) && genericAnchors.length < GENERIC_ANCHOR_LIMIT) genericAnchors.push({
|
|
34635
|
+
source: id,
|
|
34636
|
+
href,
|
|
34637
|
+
text: link.text.trim()
|
|
34638
|
+
});
|
|
34639
|
+
} else other.add(hrefId);
|
|
34640
|
+
}
|
|
34641
|
+
const record = {
|
|
34642
|
+
url: finalUrl,
|
|
34643
|
+
title: text.title,
|
|
34644
|
+
h1: text.h1,
|
|
34645
|
+
description: text.description,
|
|
34646
|
+
headings: text.headings,
|
|
34647
|
+
excerpt: excerptOf(text.sentences),
|
|
34648
|
+
words: text.sentences.reduce((sum, sentence) => sum + sentence.text.split(/\s+/).length, 0),
|
|
34649
|
+
lang: page.lang?.toLowerCase().split(/[-_]/)[0] || null,
|
|
34650
|
+
noindex: page.noindex,
|
|
34651
|
+
contextual: [...contextual],
|
|
34652
|
+
other: [...other],
|
|
34653
|
+
text: {
|
|
34654
|
+
offset: 0,
|
|
34655
|
+
length: 0
|
|
34656
|
+
}
|
|
34657
|
+
};
|
|
34658
|
+
pageOfUrl.set(finalId, id);
|
|
34659
|
+
pages.push(record);
|
|
34660
|
+
index.addPage(termCounts(text));
|
|
34661
|
+
record.text = await sentences.write(linkableSentences(text.sentences));
|
|
34662
|
+
if (method === "links" && !page.nofollow) for (const link of page.links) enqueue(crawlUrl(link.href, finalUrl));
|
|
34663
|
+
if (pages.length % 50 === 0) input.progress?.(`Crawled ${pages.length} pages of up to ${pageLimit}.`);
|
|
34664
|
+
};
|
|
34665
|
+
const worker = async () => {
|
|
34666
|
+
while (pages.length < pageLimit) {
|
|
34667
|
+
const url = queue.shift();
|
|
34668
|
+
if (!url) {
|
|
34669
|
+
if (active === 0) return;
|
|
34670
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
34671
|
+
continue;
|
|
34672
|
+
}
|
|
34673
|
+
active += 1;
|
|
34674
|
+
try {
|
|
34675
|
+
await crawlOne(url);
|
|
34676
|
+
} finally {
|
|
34677
|
+
active -= 1;
|
|
34678
|
+
}
|
|
34679
|
+
}
|
|
34238
34680
|
};
|
|
34239
|
-
|
|
34240
|
-
|
|
34241
|
-
|
|
34242
|
-
|
|
34243
|
-
|
|
34244
|
-
|
|
34245
|
-
|
|
34246
|
-
|
|
34247
|
-
|
|
34248
|
-
|
|
34249
|
-
|
|
34250
|
-
|
|
34681
|
+
let active = 0;
|
|
34682
|
+
if (method === "links" && queue[0] === start) {
|
|
34683
|
+
queue.shift();
|
|
34684
|
+
active += 1;
|
|
34685
|
+
await crawlOne(start).finally(() => {
|
|
34686
|
+
active -= 1;
|
|
34687
|
+
});
|
|
34688
|
+
const first = pages[0];
|
|
34689
|
+
if (first) origin = new URL(first.url).origin;
|
|
34690
|
+
}
|
|
34691
|
+
await Promise.all(Array.from({ length: rate.concurrency ?? 1 }, () => worker()));
|
|
34692
|
+
const capReached = pages.length >= pageLimit && (queue.length > 0 || overflow > 0);
|
|
34693
|
+
const linkedCounts = /* @__PURE__ */ new Map();
|
|
34694
|
+
for (const page of pages) for (const id of [...page.contextual, ...page.other]) if (!pageOfUrl.has(id) && !aliases.has(id)) linkedCounts.set(id, (linkedCounts.get(id) ?? 0) + 1);
|
|
34695
|
+
const toCheck = [...linkedCounts].sort((left, right) => right[1] - left[1] || left[0] - right[0]).slice(0, REDIRECT_CHECKS);
|
|
34696
|
+
let redirectsChecked = 0;
|
|
34697
|
+
await Promise.all(Array.from({ length: rate.concurrency ?? 1 }, async () => {
|
|
34698
|
+
while (toCheck.length) {
|
|
34699
|
+
const next = toCheck.shift();
|
|
34700
|
+
if (!next) break;
|
|
34701
|
+
const url = urls[next[0]];
|
|
34702
|
+
try {
|
|
34703
|
+
const fetched = await fetchPage(url, {
|
|
34704
|
+
...options,
|
|
34705
|
+
js: "off"
|
|
34706
|
+
});
|
|
34707
|
+
redirectsChecked += 1;
|
|
34708
|
+
const finalUrl = crawlUrl(fetched.finalUrl);
|
|
34709
|
+
if (finalUrl && finalUrl !== url) aliases.set(next[0], idOf(finalUrl));
|
|
34710
|
+
} catch {}
|
|
34251
34711
|
}
|
|
34712
|
+
}));
|
|
34713
|
+
for (let id = 0; id < urls.length; id += 1) {
|
|
34714
|
+
if (pageOfUrl.has(id)) continue;
|
|
34715
|
+
const seen = /* @__PURE__ */ new Set();
|
|
34716
|
+
let current = id;
|
|
34717
|
+
while (current !== void 0 && !seen.has(current)) {
|
|
34718
|
+
seen.add(current);
|
|
34719
|
+
const page = pageOfUrl.get(current);
|
|
34720
|
+
if (page !== void 0) {
|
|
34721
|
+
pageOfUrl.set(id, page);
|
|
34722
|
+
break;
|
|
34723
|
+
}
|
|
34724
|
+
current = aliases.get(current);
|
|
34725
|
+
}
|
|
34726
|
+
}
|
|
34727
|
+
index.finish();
|
|
34728
|
+
return {
|
|
34729
|
+
origin,
|
|
34730
|
+
method,
|
|
34731
|
+
pages,
|
|
34732
|
+
urls,
|
|
34733
|
+
pageOfUrl,
|
|
34734
|
+
index,
|
|
34735
|
+
sentences,
|
|
34736
|
+
genericAnchors,
|
|
34737
|
+
stats: stats({
|
|
34738
|
+
urlsFound: Math.max(queued.size, sitemap.discovery.urlsReturned),
|
|
34739
|
+
capReached,
|
|
34740
|
+
redirectsChecked
|
|
34741
|
+
}),
|
|
34742
|
+
warnings
|
|
34252
34743
|
};
|
|
34253
34744
|
}
|
|
34254
|
-
|
|
34255
|
-
|
|
34256
|
-
|
|
34257
|
-
|
|
34258
|
-
|
|
34259
|
-
|
|
34260
|
-
|
|
34261
|
-
|
|
34262
|
-
|
|
34263
|
-
|
|
34264
|
-
|
|
34265
|
-
|
|
34745
|
+
//#endregion
|
|
34746
|
+
//#region packages/core/dist/analyze/internal-links-judge.js
|
|
34747
|
+
const PAGE_TYPES = {
|
|
34748
|
+
pillar_guide: "A long, broad guide that covers a whole topic and would naturally link out to many narrower pages",
|
|
34749
|
+
support_article: "A focused article, post, tutorial, or explainer about one narrower subject",
|
|
34750
|
+
product_or_service: "Sells or describes a product, service, plan, pricing, demo, or signup: a page that makes money",
|
|
34751
|
+
comparison_or_review: "Compares options, reviews one, or lists the best of something to help a reader choose",
|
|
34752
|
+
glossary_or_definition: "A short page that defines one term or answers one 'what is' question",
|
|
34753
|
+
tool_or_calculator: "An interactive tool, calculator, generator, or checker the reader uses on the page",
|
|
34754
|
+
news_or_update: "A dated announcement, release note, changelog entry, or news item that will go stale",
|
|
34755
|
+
category_hub: "A listing, archive, tag, category, or index page that mostly lists other pages",
|
|
34756
|
+
home: "The site's home page",
|
|
34757
|
+
about_or_company: "About, team, contact, press, careers, or other company information",
|
|
34758
|
+
legal_or_utility: "Privacy, terms, cookies, login, search, 404, thank-you, or other utility pages readers do not seek out",
|
|
34759
|
+
other: null
|
|
34760
|
+
};
|
|
34761
|
+
const SEARCH_INTENTS = {
|
|
34762
|
+
informational: "The reader wants to learn or understand something",
|
|
34763
|
+
commercial: "The reader is comparing or evaluating options before buying",
|
|
34764
|
+
transactional: "The reader wants to buy, sign up, download, or complete an action now",
|
|
34765
|
+
navigational: "The reader wants to reach a specific page or part of the site"
|
|
34766
|
+
};
|
|
34767
|
+
const LINK_ROLES = {
|
|
34768
|
+
explains_concept: "The target explains a term or idea the source mentions",
|
|
34769
|
+
deeper_detail: "The target goes deeper into a sub-topic the source only touches on",
|
|
34770
|
+
broader_guide: "The target is the broader guide or hub the source belongs under",
|
|
34771
|
+
next_step: "The target is what a reader should read or do after this part",
|
|
34772
|
+
product_or_service: "The target is the product or service that solves the problem discussed",
|
|
34773
|
+
comparison_or_alternative: "The target compares or offers an alternative to what the source discusses",
|
|
34774
|
+
weak_or_none: "No clear reason a reader would want the target here"
|
|
34775
|
+
};
|
|
34776
|
+
const THRESHOLDS = {
|
|
34777
|
+
possible: .5,
|
|
34778
|
+
strong: .75
|
|
34779
|
+
};
|
|
34780
|
+
const FLOORS = {
|
|
34781
|
+
minSentenceMatch: .1,
|
|
34782
|
+
minSimilarityAlone: .15
|
|
34783
|
+
};
|
|
34784
|
+
function targetWeight(type) {
|
|
34785
|
+
if (type === "home" || type === "legal_or_utility" || type === "about_or_company" || type === "category_hub") return 0;
|
|
34786
|
+
return type === "news_or_update" ? .6 : 1;
|
|
34787
|
+
}
|
|
34788
|
+
function canBeSource(type) {
|
|
34789
|
+
return type !== "home" && type !== "category_hub" && type !== "legal_or_utility";
|
|
34790
|
+
}
|
|
34791
|
+
function classifyRequest(pages) {
|
|
34792
|
+
const questions = {};
|
|
34793
|
+
pages.forEach((_, index) => {
|
|
34794
|
+
questions[`type_${index}`] = {
|
|
34795
|
+
type: "choice",
|
|
34796
|
+
instructions: `What kind of page is \`pages[${index}]\`?`,
|
|
34797
|
+
criteria: PAGE_TYPES
|
|
34266
34798
|
};
|
|
34267
|
-
|
|
34268
|
-
|
|
34269
|
-
|
|
34270
|
-
|
|
34271
|
-
js: "auto",
|
|
34272
|
-
refresh: input.refresh
|
|
34273
|
-
});
|
|
34274
|
-
} catch (error) {
|
|
34275
|
-
return {
|
|
34276
|
-
exclusion: "failed",
|
|
34277
|
-
warnings: [{
|
|
34278
|
-
stage: "source-fetch",
|
|
34279
|
-
url: input.candidate.sourceUrl,
|
|
34280
|
-
code: "fetch-failed",
|
|
34281
|
-
message: errorMessage(error)
|
|
34282
|
-
}]
|
|
34799
|
+
questions[`intent_${index}`] = {
|
|
34800
|
+
type: "choice",
|
|
34801
|
+
instructions: `What does a visitor arriving at \`pages[${index}]\` from a search engine most likely want?`,
|
|
34802
|
+
criteria: SEARCH_INTENTS
|
|
34283
34803
|
};
|
|
34284
|
-
}
|
|
34285
|
-
|
|
34286
|
-
|
|
34287
|
-
|
|
34288
|
-
|
|
34289
|
-
|
|
34290
|
-
|
|
34291
|
-
|
|
34292
|
-
|
|
34293
|
-
|
|
34294
|
-
|
|
34295
|
-
|
|
34296
|
-
|
|
34804
|
+
});
|
|
34805
|
+
return {
|
|
34806
|
+
state: { pages: pages.map((page) => ({
|
|
34807
|
+
url: page.url,
|
|
34808
|
+
title: page.title,
|
|
34809
|
+
description: page.description,
|
|
34810
|
+
h1: page.h1,
|
|
34811
|
+
headings: page.headings,
|
|
34812
|
+
excerpt: page.excerpt,
|
|
34813
|
+
word_count: page.words
|
|
34814
|
+
})) },
|
|
34815
|
+
questions
|
|
34816
|
+
};
|
|
34817
|
+
}
|
|
34818
|
+
function choiceOf(answer) {
|
|
34819
|
+
return answer?.type === "choice" ? answer.choice : null;
|
|
34820
|
+
}
|
|
34821
|
+
function readClassification(answers, index) {
|
|
34822
|
+
const type = choiceOf(answers[`type_${index}`]);
|
|
34823
|
+
const intent = choiceOf(answers[`intent_${index}`]);
|
|
34824
|
+
return {
|
|
34825
|
+
type: type && Object.hasOwn(PAGE_TYPES, type) ? type : null,
|
|
34826
|
+
intent: intent && Object.hasOwn(SEARCH_INTENTS, intent) ? intent : null
|
|
34827
|
+
};
|
|
34828
|
+
}
|
|
34829
|
+
function judgeRequest(source, targets) {
|
|
34830
|
+
const sentences = /* @__PURE__ */ new Map();
|
|
34831
|
+
for (const target of targets) for (const sentence of target.placement.sentences) sentences.set(sentence.id, {
|
|
34832
|
+
id: sentence.id,
|
|
34833
|
+
under_heading: sentence.section,
|
|
34834
|
+
text: sentence.text
|
|
34835
|
+
});
|
|
34836
|
+
const questions = {};
|
|
34837
|
+
targets.forEach((target, index) => {
|
|
34838
|
+
const ref = `\`targets[${index}]\``;
|
|
34839
|
+
questions[`link_${index}`] = {
|
|
34840
|
+
type: "noul",
|
|
34841
|
+
instructions: `Should the page \`source\` add a link from within its body text to the page ${ref}?`,
|
|
34842
|
+
criteria: {
|
|
34843
|
+
true: "A reader of the source would gain something by reading the target here; the two pages cover related subjects and the link would feel natural and helpful",
|
|
34844
|
+
false: "The pages are only loosely related, the target adds nothing for this reader, or the link would feel forced or spammy"
|
|
34845
|
+
}
|
|
34846
|
+
};
|
|
34847
|
+
questions[`where_${index}`] = {
|
|
34848
|
+
type: "choice",
|
|
34849
|
+
instructions: `Which sentence in \`source.sentences\` is the most natural place to add a link to ${ref}? Choose "none" if no listed sentence leads a reader towards that page.`,
|
|
34850
|
+
criteria: {
|
|
34851
|
+
...Object.fromEntries(target.placement.sentences.map((sentence) => [sentence.id, sentence.text])),
|
|
34852
|
+
none: "No listed sentence is a natural place for this link"
|
|
34853
|
+
}
|
|
34854
|
+
};
|
|
34855
|
+
if (target.placement.anchors.length) questions[`anchor_${index}`] = {
|
|
34856
|
+
type: "choice",
|
|
34857
|
+
instructions: `Which phrase should become the clickable link text for a link from \`source\` to ${ref}? Prefer a phrase that tells the reader what the target page is about and reads naturally as a link.`,
|
|
34858
|
+
criteria: {
|
|
34859
|
+
...Object.fromEntries(target.placement.anchors.map((anchor, at) => [`A${at + 1}`, {
|
|
34860
|
+
text: anchor.text,
|
|
34861
|
+
in_sentence: anchor.sentence
|
|
34862
|
+
}])),
|
|
34863
|
+
none: "None of these phrases would make a good link"
|
|
34864
|
+
}
|
|
34865
|
+
};
|
|
34866
|
+
questions[`role_${index}`] = {
|
|
34867
|
+
type: "choice",
|
|
34868
|
+
instructions: `If \`source\` linked to ${ref}, what would the link do for the reader?`,
|
|
34869
|
+
criteria: LINK_ROLES
|
|
34297
34870
|
};
|
|
34298
|
-
}
|
|
34299
|
-
const warnings = page.warnings.map((message) => ({
|
|
34300
|
-
stage: "source-extract",
|
|
34301
|
-
url: input.candidate.sourceUrl,
|
|
34302
|
-
code: "extractor-warning",
|
|
34303
|
-
message
|
|
34304
|
-
}));
|
|
34305
|
-
if (input.target.aliases.some((alias) => sameObservedUrl(page.finalUrl, alias))) return {
|
|
34306
|
-
exclusion: "self-alias",
|
|
34307
|
-
warnings
|
|
34308
|
-
};
|
|
34309
|
-
const technicalSignals = pageTechnicalSignals({
|
|
34310
|
-
url: input.candidate.sourceUrl,
|
|
34311
|
-
page,
|
|
34312
|
-
fetchDiagnostics: fetched.diagnostics,
|
|
34313
|
-
httpStatus: fetched.status
|
|
34314
34871
|
});
|
|
34315
|
-
if (technicalSignals.length) return {
|
|
34316
|
-
exclusion: "technical",
|
|
34317
|
-
warnings
|
|
34318
|
-
};
|
|
34319
|
-
const observed = linksToAliases(page, input.target.aliases).map((link) => ({
|
|
34320
|
-
href: link.href,
|
|
34321
|
-
text: link.text,
|
|
34322
|
-
rel: link.rel,
|
|
34323
|
-
location: link.location
|
|
34324
|
-
}));
|
|
34325
|
-
const contextual = observed.filter((link) => link.location === "main-content");
|
|
34326
|
-
if (contextual.some((link) => sameObservedUrl(link.href, input.target.preferredUrl))) return {
|
|
34327
|
-
exclusion: "existing-link",
|
|
34328
|
-
warnings
|
|
34329
|
-
};
|
|
34330
|
-
const aliasContextual = contextual.length > 0;
|
|
34331
|
-
const linkStatus = aliasContextual ? "alias-contextual" : observed.length ? "non-contextual-only" : "missing";
|
|
34332
|
-
const confidence = input.candidate.bestMatchKind === "exact-query" && !page.contentExtraction.fallback ? "medium" : "low";
|
|
34333
|
-
const actionType = aliasContextual ? "review-alias-link" : "review-contextual-link";
|
|
34334
|
-
const evidenceRef = aliasContextual ? `A main-content link points to a non-preferred target alias; the preferred target is ${input.target.preferredUrl}.` : `${input.candidate.sourceUrl} matched ${input.candidate.matchedQueries} retained GSC query row${input.candidate.matchedQueries === 1 ? "" : "s"}, and no main-content link to the preferred target was observed in the fetched HTML.`;
|
|
34335
|
-
const action = aliasContextual ? `Review the existing contextual link and update it to the preferred target ${input.target.preferredUrl} if the redirect or old alias is not intentional.` : `Review whether this source page and ${input.target.preferredUrl} serve complementary intent. If the link would help a reader, add a natural main-content link without forcing exact-match anchor text.`;
|
|
34336
34872
|
return {
|
|
34337
|
-
|
|
34338
|
-
|
|
34339
|
-
|
|
34340
|
-
|
|
34341
|
-
|
|
34342
|
-
|
|
34343
|
-
pageWarnings: page.warnings,
|
|
34344
|
-
actionType,
|
|
34345
|
-
linkEvidence: {
|
|
34346
|
-
status: linkStatus,
|
|
34347
|
-
observedCount: observed.length,
|
|
34348
|
-
observedLimit: 20,
|
|
34349
|
-
limitedCount: Math.max(0, observed.length - 20),
|
|
34350
|
-
observed: observed.slice(0, 20)
|
|
34873
|
+
state: {
|
|
34874
|
+
source: {
|
|
34875
|
+
url: source.url,
|
|
34876
|
+
title: source.title,
|
|
34877
|
+
page_type: source.type ?? "unknown",
|
|
34878
|
+
sentences: [...sentences.values()].sort((left, right) => Number(left.id.slice(1)) - Number(right.id.slice(1)))
|
|
34351
34879
|
},
|
|
34352
|
-
|
|
34353
|
-
|
|
34354
|
-
|
|
34355
|
-
|
|
34356
|
-
|
|
34357
|
-
|
|
34358
|
-
|
|
34359
|
-
|
|
34360
|
-
}
|
|
34880
|
+
targets: targets.map((target) => ({
|
|
34881
|
+
url: target.url,
|
|
34882
|
+
title: target.title,
|
|
34883
|
+
description: target.description,
|
|
34884
|
+
h1: target.h1,
|
|
34885
|
+
page_type: target.type ?? "unknown",
|
|
34886
|
+
search_intent: target.intent ?? "unknown",
|
|
34887
|
+
opening: target.opening
|
|
34888
|
+
}))
|
|
34361
34889
|
},
|
|
34362
|
-
|
|
34363
|
-
|
|
34364
|
-
|
|
34365
|
-
|
|
34366
|
-
|
|
34367
|
-
|
|
34890
|
+
questions
|
|
34891
|
+
};
|
|
34892
|
+
}
|
|
34893
|
+
function floor(sentence, similarity, probability) {
|
|
34894
|
+
if (sentence && sentence.match < FLOORS.minSentenceMatch) return {
|
|
34895
|
+
state: "held-back",
|
|
34896
|
+
reason: "weak-sentence",
|
|
34897
|
+
detail: `the sentence has ${Math.round(sentence.match * 100)}% of the words that name the page; a link needs ${Math.round(FLOORS.minSentenceMatch * 100)}%`,
|
|
34898
|
+
probability
|
|
34899
|
+
};
|
|
34900
|
+
if (!sentence && similarity < FLOORS.minSimilarityAlone) return {
|
|
34901
|
+
state: "held-back",
|
|
34902
|
+
reason: "no-sentence",
|
|
34903
|
+
detail: `no sentence to place it in, and the pages are ${similarity.toFixed(2)} similar; a link without a sentence needs ${FLOORS.minSimilarityAlone}`,
|
|
34904
|
+
probability
|
|
34905
|
+
};
|
|
34906
|
+
return null;
|
|
34907
|
+
}
|
|
34908
|
+
function decideWithModel(placement, similarity, answers, index) {
|
|
34909
|
+
const link = answers[`link_${index}`];
|
|
34910
|
+
const role = choiceOf(answers[`role_${index}`]);
|
|
34911
|
+
if (link?.type !== "noul" || !role) return {
|
|
34912
|
+
state: "declined",
|
|
34913
|
+
reason: "no-answer",
|
|
34914
|
+
probability: null
|
|
34915
|
+
};
|
|
34916
|
+
const probability = Math.round(link.noul * 1e3) / 1e3;
|
|
34917
|
+
if (probability < THRESHOLDS.possible) return {
|
|
34918
|
+
state: "declined",
|
|
34919
|
+
reason: "below-threshold",
|
|
34920
|
+
probability
|
|
34921
|
+
};
|
|
34922
|
+
if (role === "weak_or_none" || !Object.hasOwn(LINK_ROLES, role)) return {
|
|
34923
|
+
state: "declined",
|
|
34924
|
+
reason: "weak-role",
|
|
34925
|
+
probability
|
|
34926
|
+
};
|
|
34927
|
+
const where = choiceOf(answers[`where_${index}`]);
|
|
34928
|
+
const sentence = placement.sentences.find((option) => option.id === where) ?? null;
|
|
34929
|
+
const held = floor(sentence, similarity, probability);
|
|
34930
|
+
if (held) return held;
|
|
34931
|
+
let anchor = null;
|
|
34932
|
+
const picked = answers[`anchor_${index}`];
|
|
34933
|
+
if (sentence && picked?.type === "choice" && picked.choice !== "none") {
|
|
34934
|
+
const inSentence = placement.anchors.map((option, at) => ({
|
|
34935
|
+
...option,
|
|
34936
|
+
key: `A${at + 1}`
|
|
34937
|
+
})).filter((option) => option.sentence === sentence.id);
|
|
34938
|
+
anchor = (inSentence.find((option) => option.key === picked.choice) ?? [...inSentence].sort((left, right) => (picked.probabilities[right.key] ?? 0) - (picked.probabilities[left.key] ?? 0) || left.key.localeCompare(right.key))[0])?.text ?? null;
|
|
34939
|
+
}
|
|
34940
|
+
return {
|
|
34941
|
+
state: "suggest",
|
|
34942
|
+
tier: probability >= THRESHOLDS.strong ? "strong" : "possible",
|
|
34943
|
+
probability,
|
|
34944
|
+
role,
|
|
34945
|
+
sentence: sentence ? {
|
|
34946
|
+
text: sentence.text,
|
|
34947
|
+
section: sentence.section,
|
|
34948
|
+
match: sentence.match
|
|
34949
|
+
} : null,
|
|
34950
|
+
anchor
|
|
34951
|
+
};
|
|
34952
|
+
}
|
|
34953
|
+
function decideWithoutModel(placement, similarity) {
|
|
34954
|
+
const sentence = placement.sentences[0] ?? null;
|
|
34955
|
+
const held = floor(sentence, similarity, null);
|
|
34956
|
+
if (held) return held;
|
|
34957
|
+
return {
|
|
34958
|
+
state: "suggest",
|
|
34959
|
+
tier: "heuristic",
|
|
34960
|
+
probability: null,
|
|
34961
|
+
role: null,
|
|
34962
|
+
sentence: sentence ? {
|
|
34963
|
+
text: sentence.text,
|
|
34964
|
+
section: sentence.section,
|
|
34965
|
+
match: sentence.match
|
|
34966
|
+
} : null,
|
|
34967
|
+
anchor: placement.anchors.find((option) => option.sentence === sentence?.id)?.text ?? null
|
|
34368
34968
|
};
|
|
34369
34969
|
}
|
|
34370
34970
|
//#endregion
|
|
34971
|
+
//#region packages/core/dist/analyze/internal-links-report.js
|
|
34972
|
+
const DAY$1 = 864e5;
|
|
34973
|
+
const ISO_DAY = /^\d{4}-\d{2}-\d{2}$/;
|
|
34974
|
+
function internalLinksReportRange(input, fallbackDays, now) {
|
|
34975
|
+
if (!input.startDate && !input.endDate) return {
|
|
34976
|
+
...finalGscDateRange(fallbackDays, now),
|
|
34977
|
+
days: fallbackDays
|
|
34978
|
+
};
|
|
34979
|
+
const { startDate, endDate } = input;
|
|
34980
|
+
if (!startDate || !endDate || !ISO_DAY.test(startDate) || !ISO_DAY.test(endDate) || startDate > endDate) throw new SeoError("INVALID_INPUT", "Send a start date and an end date together, in order, as YYYY-MM-DD.");
|
|
34981
|
+
return {
|
|
34982
|
+
startDate,
|
|
34983
|
+
endDate,
|
|
34984
|
+
days: Math.round((Date.parse(`${endDate}T00:00:00Z`) - Date.parse(`${startDate}T00:00:00Z`)) / DAY$1) + 1
|
|
34985
|
+
};
|
|
34986
|
+
}
|
|
34987
|
+
const plural$2 = (count, word) => `${count} ${word}${count === 1 ? "" : "s"}`;
|
|
34988
|
+
function internalLinksVerdict(report) {
|
|
34989
|
+
const { counts } = report;
|
|
34990
|
+
if (report.dataStatus === "blocked") return "robots.txt does not allow this crawler to read the site, so nothing was checked.";
|
|
34991
|
+
if (report.dataStatus === "empty") return "No HTML page could be crawled, so there is nothing to link.";
|
|
34992
|
+
if (report.dataStatus === "target-not-crawled") return "The target page could not be crawled as its own page (it failed, left the site, or names another page as canonical). Check it with seo page first.";
|
|
34993
|
+
if (report.dataStatus === "target-not-linkable") return `The target is ${report.target?.noindex ? "noindex" : `a ${report.target?.type?.replaceAll("_", " ")} page`}, so no links to it are suggested.`;
|
|
34994
|
+
const judged = report.judge.status === "used" || report.judge.status === "partial" ? `${counts.strong} strong and ${counts.possible} possible link${counts.strong + counts.possible === 1 ? "" : "s"} judged by ${report.judge.model}${counts.heuristic ? `, plus ${plural$2(counts.heuristic, "heuristic link")} where a request failed` : ""}` : `${plural$2(counts.heuristic, "heuristic link")} (no decision model)`;
|
|
34995
|
+
const scope = report.mode === "url" ? `to ${report.target?.url}` : `across ${plural$2(counts.pagesCrawled, "page")}`;
|
|
34996
|
+
const capped = report.crawl.capReached ? ` The crawl stopped at its limit of ${report.crawl.pageLimit} pages, so pages beyond it were not checked.` : "";
|
|
34997
|
+
return `${judged} ${scope}${counts.heldBack ? `; ${counts.heldBack} held back by the evidence floors` : ""}.${capped}`;
|
|
34998
|
+
}
|
|
34999
|
+
//#endregion
|
|
34371
35000
|
//#region packages/core/dist/analyze/internal-links.js
|
|
35001
|
+
const GSC_DAYS = 90;
|
|
34372
35002
|
const MAX_GSC_ROWS = 1e5;
|
|
34373
|
-
const
|
|
34374
|
-
const
|
|
34375
|
-
const
|
|
34376
|
-
const
|
|
34377
|
-
const
|
|
34378
|
-
|
|
34379
|
-
|
|
34380
|
-
|
|
34381
|
-
|
|
34382
|
-
|
|
34383
|
-
|
|
34384
|
-
|
|
34385
|
-
|
|
34386
|
-
const
|
|
34387
|
-
|
|
34388
|
-
|
|
34389
|
-
|
|
34390
|
-
|
|
34391
|
-
const
|
|
34392
|
-
|
|
34393
|
-
|
|
34394
|
-
|
|
34395
|
-
|
|
34396
|
-
|
|
35003
|
+
const MAX_PAGES_PER_QUERY = 25;
|
|
35004
|
+
const MAX_GSC_PAIRS = 5e5;
|
|
35005
|
+
const MIN_SIMILARITY = .08;
|
|
35006
|
+
const TARGETS_PER_SOURCE = 8;
|
|
35007
|
+
const EXTRA_GSC_TARGETS = 2;
|
|
35008
|
+
const URL_SOURCES_PLACED = 60;
|
|
35009
|
+
const URL_SOURCES_JUDGED = 24;
|
|
35010
|
+
const SOURCE_WINDOW = 64;
|
|
35011
|
+
const WEAK_INBOUND = 3;
|
|
35012
|
+
const EXAMPLE_LIMIT = 20;
|
|
35013
|
+
const LIST_LIMIT = 100;
|
|
35014
|
+
const OPENING_CHARS = 300;
|
|
35015
|
+
function siteStartUrl(site) {
|
|
35016
|
+
const value = site.trim();
|
|
35017
|
+
if (value.startsWith("sc-domain:")) return `https://${value.slice(10)}/`;
|
|
35018
|
+
return `${new URL(/^https?:\/\//i.test(value) ? value : `https://${value}`).origin}/`;
|
|
35019
|
+
}
|
|
35020
|
+
function round$1(value, places = 3) {
|
|
35021
|
+
const factor = 10 ** places;
|
|
35022
|
+
return Math.round(value * factor) / factor;
|
|
35023
|
+
}
|
|
35024
|
+
async function internalLinksReport(input, dependencies) {
|
|
35025
|
+
const deps = {
|
|
35026
|
+
crawl: crawlForLinks,
|
|
35027
|
+
now: () => /* @__PURE__ */ new Date(),
|
|
35028
|
+
...dependencies
|
|
35029
|
+
};
|
|
35030
|
+
const mode = input.targetUrl ? "url" : "site";
|
|
35031
|
+
if (mode === "url" && input.maxPages !== void 0) throw new SeoError("INVALID_INPUT", `--max-pages is for a site audit only. A page run crawls up to 500 pages.`);
|
|
35032
|
+
const maxPages = input.maxPages ?? 500;
|
|
35033
|
+
if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > 5e3) throw new SeoError("INVALID_INPUT", `--max-pages is a whole number from 1 to ${MAX_MAX_PAGES.toLocaleString("en-US")}.`);
|
|
35034
|
+
const targetUrl = input.targetUrl ? crawlUrl(input.targetUrl) : void 0;
|
|
35035
|
+
if (input.targetUrl && !targetUrl) throw new SeoError("INVALID_INPUT", "The target is an http or https URL.");
|
|
35036
|
+
const now = deps.now();
|
|
35037
|
+
const startUrl = targetUrl ? `${new URL(targetUrl).origin}/` : siteStartUrl(input.site);
|
|
35038
|
+
deps.progress?.(`Crawling ${startUrl} (up to ${maxPages} pages).`);
|
|
35039
|
+
const crawl = await deps.crawl({
|
|
35040
|
+
startUrl,
|
|
35041
|
+
maxPages,
|
|
35042
|
+
firstUrl: targetUrl,
|
|
34397
35043
|
refresh: input.refresh,
|
|
34398
|
-
|
|
35044
|
+
progress: deps.progress
|
|
34399
35045
|
});
|
|
34400
|
-
|
|
34401
|
-
|
|
34402
|
-
|
|
34403
|
-
|
|
34404
|
-
|
|
34405
|
-
|
|
34406
|
-
|
|
34407
|
-
|
|
34408
|
-
|
|
34409
|
-
|
|
34410
|
-
|
|
34411
|
-
|
|
34412
|
-
|
|
34413
|
-
|
|
34414
|
-
|
|
34415
|
-
|
|
34416
|
-
|
|
34417
|
-
|
|
34418
|
-
|
|
34419
|
-
|
|
34420
|
-
|
|
34421
|
-
})
|
|
34422
|
-
const
|
|
34423
|
-
|
|
34424
|
-
|
|
34425
|
-
|
|
34426
|
-
|
|
34427
|
-
|
|
34428
|
-
|
|
34429
|
-
|
|
34430
|
-
|
|
34431
|
-
|
|
34432
|
-
|
|
34433
|
-
|
|
34434
|
-
|
|
34435
|
-
|
|
34436
|
-
|
|
34437
|
-
|
|
34438
|
-
|
|
34439
|
-
|
|
34440
|
-
|
|
34441
|
-
|
|
35046
|
+
try {
|
|
35047
|
+
return await analyse(input, deps, crawl, {
|
|
35048
|
+
mode,
|
|
35049
|
+
targetUrl,
|
|
35050
|
+
startUrl,
|
|
35051
|
+
now
|
|
35052
|
+
});
|
|
35053
|
+
} finally {
|
|
35054
|
+
await crawl.sentences.close();
|
|
35055
|
+
}
|
|
35056
|
+
}
|
|
35057
|
+
async function analyse(input, deps, crawl, context) {
|
|
35058
|
+
const { pages, index } = crawl;
|
|
35059
|
+
const count = pages.length;
|
|
35060
|
+
const warnings = [...crawl.warnings];
|
|
35061
|
+
const lookup = /* @__PURE__ */ new Map();
|
|
35062
|
+
crawl.urls.forEach((url, id) => {
|
|
35063
|
+
const page = crawl.pageOfUrl.get(id);
|
|
35064
|
+
if (page === void 0) return;
|
|
35065
|
+
lookup.set(url, page);
|
|
35066
|
+
if (!lookup.has(normalizePageUrl(url))) lookup.set(normalizePageUrl(url), page);
|
|
35067
|
+
});
|
|
35068
|
+
const pageAt = (url) => {
|
|
35069
|
+
const clean = crawlUrl(url);
|
|
35070
|
+
if (!clean) return void 0;
|
|
35071
|
+
return lookup.get(clean) ?? lookup.get(normalizePageUrl(clean));
|
|
35072
|
+
};
|
|
35073
|
+
const target = context.targetUrl !== void 0 ? pageAt(context.targetUrl) : void 0;
|
|
35074
|
+
const contextual = pages.map(() => /* @__PURE__ */ new Set());
|
|
35075
|
+
const anywhere = pages.map(() => /* @__PURE__ */ new Set());
|
|
35076
|
+
const redirectedLinks = [];
|
|
35077
|
+
pages.forEach((page, source) => {
|
|
35078
|
+
for (const id of page.contextual) {
|
|
35079
|
+
const to = crawl.pageOfUrl.get(id);
|
|
35080
|
+
if (to === void 0 || to === source) continue;
|
|
35081
|
+
contextual[source]?.add(to);
|
|
35082
|
+
anywhere[source]?.add(to);
|
|
35083
|
+
const via = crawl.urls[id];
|
|
35084
|
+
const final = pages[to]?.url;
|
|
35085
|
+
if (normalizePageUrl(via) !== normalizePageUrl(final) && redirectedLinks.length < LIST_LIMIT) redirectedLinks.push({
|
|
35086
|
+
source: page.url,
|
|
35087
|
+
via,
|
|
35088
|
+
target: final
|
|
35089
|
+
});
|
|
35090
|
+
}
|
|
35091
|
+
for (const id of page.other) {
|
|
35092
|
+
const to = crawl.pageOfUrl.get(id);
|
|
35093
|
+
if (to !== void 0 && to !== source) anywhere[source]?.add(to);
|
|
35094
|
+
}
|
|
35095
|
+
});
|
|
35096
|
+
const inbound = new Array(count).fill(0);
|
|
35097
|
+
const inboundAnywhere = new Array(count).fill(0);
|
|
35098
|
+
contextual.forEach((set) => {
|
|
35099
|
+
for (const to of set) inbound[to] = (inbound[to] ?? 0) + 1;
|
|
35100
|
+
});
|
|
35101
|
+
anywhere.forEach((set) => {
|
|
35102
|
+
for (const to of set) inboundAnywhere[to] = (inboundAnywhere[to] ?? 0) + 1;
|
|
35103
|
+
});
|
|
35104
|
+
const existingContextualLinks = contextual.reduce((sum, set) => sum + set.size, 0);
|
|
35105
|
+
const range = internalLinksReportRange(input, GSC_DAYS, context.now);
|
|
35106
|
+
const sharedQueries = /* @__PURE__ */ new Map();
|
|
35107
|
+
const pairKey = (source, target) => source * count + target;
|
|
35108
|
+
let searchConsole = {
|
|
35109
|
+
status: "not-available",
|
|
35110
|
+
property: null,
|
|
35111
|
+
range: null,
|
|
35112
|
+
rows: 0,
|
|
35113
|
+
possiblyTruncated: false,
|
|
35114
|
+
pairsWithSharedQueries: 0,
|
|
35115
|
+
note: "unavailable" in deps.searchConsole ? deps.searchConsole.unavailable : null
|
|
35116
|
+
};
|
|
35117
|
+
if ("property" in deps.searchConsole && count > 1) {
|
|
35118
|
+
const { startDate, endDate } = range;
|
|
35119
|
+
try {
|
|
35120
|
+
deps.progress?.("Reading Search Console queries.");
|
|
35121
|
+
const result = await deps.searchConsole.query({
|
|
35122
|
+
startDate,
|
|
35123
|
+
endDate,
|
|
35124
|
+
dimensions: ["query", "page"],
|
|
35125
|
+
type: "web",
|
|
35126
|
+
dataState: "final",
|
|
35127
|
+
maxRows: MAX_GSC_ROWS
|
|
35128
|
+
});
|
|
35129
|
+
const brandTerms = deriveBrandTerms({ siteUrl: crawl.origin });
|
|
35130
|
+
const byQuery = /* @__PURE__ */ new Map();
|
|
35131
|
+
for (const row of result.rows) {
|
|
35132
|
+
const [query, url] = row.keys;
|
|
35133
|
+
if (!query || !url || row.impressions < 1) continue;
|
|
35134
|
+
if (isBrandQuery(query, brandTerms)) continue;
|
|
35135
|
+
const page = pageAt(url);
|
|
35136
|
+
if (page === void 0) continue;
|
|
35137
|
+
let set = byQuery.get(query);
|
|
35138
|
+
if (!set) {
|
|
35139
|
+
set = /* @__PURE__ */ new Set();
|
|
35140
|
+
byQuery.set(query, set);
|
|
35141
|
+
}
|
|
35142
|
+
set.add(page);
|
|
35143
|
+
}
|
|
35144
|
+
let capped = false;
|
|
35145
|
+
for (const set of byQuery.values()) {
|
|
35146
|
+
if (set.size < 2 || set.size > MAX_PAGES_PER_QUERY) continue;
|
|
35147
|
+
const list = [...set];
|
|
35148
|
+
for (const source of list) for (const to of list) {
|
|
35149
|
+
if (source === to) continue;
|
|
35150
|
+
if (target !== void 0 && to !== target) continue;
|
|
35151
|
+
const key = pairKey(source, to);
|
|
35152
|
+
if (!sharedQueries.has(key) && sharedQueries.size >= MAX_GSC_PAIRS) {
|
|
35153
|
+
capped = true;
|
|
35154
|
+
continue;
|
|
35155
|
+
}
|
|
35156
|
+
sharedQueries.set(key, (sharedQueries.get(key) ?? 0) + 1);
|
|
35157
|
+
}
|
|
35158
|
+
}
|
|
35159
|
+
searchConsole = {
|
|
35160
|
+
status: "used",
|
|
35161
|
+
property: deps.searchConsole.property,
|
|
35162
|
+
range: {
|
|
35163
|
+
startDate,
|
|
35164
|
+
endDate
|
|
35165
|
+
},
|
|
35166
|
+
rows: result.rowsFetched,
|
|
35167
|
+
possiblyTruncated: result.rowsFetched >= MAX_GSC_ROWS,
|
|
35168
|
+
pairsWithSharedQueries: sharedQueries.size,
|
|
35169
|
+
note: capped ? `Shared queries were counted for the first ${MAX_GSC_PAIRS.toLocaleString("en-US")} page pairs.` : null
|
|
35170
|
+
};
|
|
35171
|
+
} catch (error) {
|
|
35172
|
+
searchConsole = {
|
|
35173
|
+
...searchConsole,
|
|
35174
|
+
status: "failed",
|
|
35175
|
+
property: deps.searchConsole.property,
|
|
35176
|
+
note: error instanceof Error ? error.message : String(error)
|
|
35177
|
+
};
|
|
35178
|
+
}
|
|
35179
|
+
}
|
|
35180
|
+
const runner = "model" in deps.decisionModel ? new DecisionRunner(deps.decisionModel.model) : null;
|
|
35181
|
+
const types = pages.map((page) => new URL(page.url).pathname === "/" ? "home" : null);
|
|
35182
|
+
const intents = pages.map(() => null);
|
|
35183
|
+
let pagesClassified = 0;
|
|
35184
|
+
const classify = async (ids) => {
|
|
35185
|
+
if (!runner) return;
|
|
35186
|
+
const batches = [];
|
|
35187
|
+
for (let at = 0; at < ids.length; at += 5) batches.push(ids.slice(at, at + 5));
|
|
35188
|
+
deps.progress?.(`Classifying ${ids.length} pages in ${batches.length} requests.`);
|
|
35189
|
+
await Promise.all(batches.map(async (batch) => {
|
|
35190
|
+
const response = await runner.ask(classifyRequest(batch.map((id) => pages[id])));
|
|
35191
|
+
if (!response) return;
|
|
35192
|
+
batch.forEach((id, at) => {
|
|
35193
|
+
const read = readClassification(response.answers, at);
|
|
35194
|
+
if (read.type) types[id] = read.type;
|
|
35195
|
+
intents[id] = read.intent;
|
|
35196
|
+
pagesClassified += 1;
|
|
35197
|
+
});
|
|
35198
|
+
}));
|
|
34442
35199
|
};
|
|
34443
|
-
|
|
34444
|
-
|
|
34445
|
-
|
|
34446
|
-
|
|
34447
|
-
|
|
34448
|
-
|
|
34449
|
-
|
|
34450
|
-
|
|
34451
|
-
}
|
|
34452
|
-
const
|
|
34453
|
-
|
|
34454
|
-
|
|
34455
|
-
|
|
34456
|
-
|
|
34457
|
-
|
|
34458
|
-
|
|
34459
|
-
|
|
34460
|
-
|
|
34461
|
-
|
|
34462
|
-
|
|
34463
|
-
|
|
34464
|
-
|
|
34465
|
-
|
|
35200
|
+
const linkable = (source, to) => {
|
|
35201
|
+
const from = pages[source];
|
|
35202
|
+
const page = pages[to];
|
|
35203
|
+
if (!from || !page || source === to) return false;
|
|
35204
|
+
if (page.noindex || targetWeight(types[to] ?? null) === 0) return false;
|
|
35205
|
+
if (contextual[source]?.has(to)) return false;
|
|
35206
|
+
if (from.lang && page.lang && from.lang !== page.lang) return false;
|
|
35207
|
+
return canBeSource(types[source] ?? null);
|
|
35208
|
+
};
|
|
35209
|
+
const adjusted = (to, raw) => raw * ((inbound[to] ?? 0) < WEAK_INBOUND ? 1.25 : 1) * targetWeight(types[to] ?? null);
|
|
35210
|
+
let status = "complete";
|
|
35211
|
+
const candidatesBySource = /* @__PURE__ */ new Map();
|
|
35212
|
+
if (crawl.stats.blockedByRobots) status = "blocked";
|
|
35213
|
+
else if (count === 0) status = "empty";
|
|
35214
|
+
else if (context.mode === "url" && target === void 0) status = "target-not-crawled";
|
|
35215
|
+
if (status === "complete" && context.mode === "site") {
|
|
35216
|
+
await classify(pages.map((_, id) => id));
|
|
35217
|
+
const extra = /* @__PURE__ */ new Map();
|
|
35218
|
+
for (const key of sharedQueries.keys()) {
|
|
35219
|
+
const source = Math.floor(key / count);
|
|
35220
|
+
const list = extra.get(source) ?? [];
|
|
35221
|
+
list.push(key % count);
|
|
35222
|
+
extra.set(source, list);
|
|
35223
|
+
}
|
|
35224
|
+
for (let source = 0; source < count; source += 1) {
|
|
35225
|
+
if (!canBeSource(types[source] ?? null)) continue;
|
|
35226
|
+
const chosen = index.neighbours(source, MIN_SIMILARITY).filter(({ page }) => linkable(source, page)).map(({ page, score }) => ({
|
|
35227
|
+
source,
|
|
35228
|
+
target: page,
|
|
35229
|
+
raw: score,
|
|
35230
|
+
similarity: adjusted(page, score),
|
|
35231
|
+
sharedQueries: sharedQueries.get(pairKey(source, page)) ?? 0
|
|
35232
|
+
})).sort((left, right) => right.similarity - left.similarity || left.target - right.target).slice(0, TARGETS_PER_SOURCE);
|
|
35233
|
+
const taken = new Set(chosen.map((candidate) => candidate.target));
|
|
35234
|
+
const fromQueries = (extra.get(source) ?? []).filter((to) => !taken.has(to) && linkable(source, to)).map((to) => {
|
|
35235
|
+
const raw = index.similarity(source, to);
|
|
35236
|
+
return {
|
|
35237
|
+
source,
|
|
35238
|
+
target: to,
|
|
35239
|
+
raw,
|
|
35240
|
+
similarity: adjusted(to, raw),
|
|
35241
|
+
sharedQueries: sharedQueries.get(pairKey(source, to)) ?? 0
|
|
35242
|
+
};
|
|
35243
|
+
}).sort((left, right) => right.sharedQueries - left.sharedQueries || right.similarity - left.similarity || left.target - right.target).slice(0, EXTRA_GSC_TARGETS);
|
|
35244
|
+
const all = [...chosen, ...fromQueries];
|
|
35245
|
+
if (all.length) candidatesBySource.set(source, all);
|
|
35246
|
+
}
|
|
35247
|
+
}
|
|
35248
|
+
if (status === "complete" && context.mode === "url" && target !== void 0) {
|
|
35249
|
+
const near = /* @__PURE__ */ new Map();
|
|
35250
|
+
for (const { page, score } of index.neighbours(target, MIN_SIMILARITY)) near.set(page, score);
|
|
35251
|
+
for (const key of sharedQueries.keys()) {
|
|
35252
|
+
const source = Math.floor(key / count);
|
|
35253
|
+
if (!near.has(source)) near.set(source, index.similarity(source, target));
|
|
35254
|
+
}
|
|
35255
|
+
const ranked = [...near].map(([source, raw]) => ({
|
|
35256
|
+
source,
|
|
34466
35257
|
target,
|
|
34467
|
-
|
|
34468
|
-
|
|
34469
|
-
|
|
34470
|
-
|
|
34471
|
-
|
|
34472
|
-
|
|
34473
|
-
|
|
34474
|
-
|
|
34475
|
-
|
|
34476
|
-
|
|
34477
|
-
|
|
34478
|
-
|
|
34479
|
-
|
|
34480
|
-
|
|
34481
|
-
|
|
34482
|
-
const
|
|
34483
|
-
|
|
34484
|
-
|
|
34485
|
-
|
|
34486
|
-
const
|
|
34487
|
-
|
|
34488
|
-
|
|
34489
|
-
|
|
34490
|
-
|
|
34491
|
-
|
|
34492
|
-
|
|
34493
|
-
|
|
34494
|
-
|
|
34495
|
-
|
|
34496
|
-
|
|
34497
|
-
|
|
34498
|
-
|
|
34499
|
-
|
|
34500
|
-
|
|
34501
|
-
|
|
34502
|
-
|
|
34503
|
-
|
|
34504
|
-
|
|
34505
|
-
|
|
34506
|
-
|
|
34507
|
-
|
|
35258
|
+
raw,
|
|
35259
|
+
similarity: adjusted(target, raw),
|
|
35260
|
+
sharedQueries: sharedQueries.get(pairKey(source, target)) ?? 0
|
|
35261
|
+
})).sort((left, right) => Number(right.sharedQueries > 0) - Number(left.sharedQueries > 0) || right.similarity - left.similarity || left.source - right.source);
|
|
35262
|
+
await classify([target, ...ranked.slice(0, URL_SOURCES_PLACED).map((c) => c.source)]);
|
|
35263
|
+
if (targetWeight(types[target] ?? null) === 0 || pages[target]?.noindex) status = "target-not-linkable";
|
|
35264
|
+
else for (const candidate of ranked) {
|
|
35265
|
+
if (candidatesBySource.size >= URL_SOURCES_PLACED) break;
|
|
35266
|
+
if (!linkable(candidate.source, target)) continue;
|
|
35267
|
+
candidatesBySource.set(candidate.source, [{
|
|
35268
|
+
...candidate,
|
|
35269
|
+
similarity: adjusted(target, candidate.raw)
|
|
35270
|
+
}]);
|
|
35271
|
+
}
|
|
35272
|
+
}
|
|
35273
|
+
const judged = [];
|
|
35274
|
+
let pairsConsidered = 0;
|
|
35275
|
+
let pairsJudged = 0;
|
|
35276
|
+
let fellBack = 0;
|
|
35277
|
+
const sources = [...candidatesBySource.keys()].sort((left, right) => left - right);
|
|
35278
|
+
let sourcesWithPlacement = 0;
|
|
35279
|
+
for (let at = 0; at < sources.length; at += SOURCE_WINDOW) {
|
|
35280
|
+
if (context.mode === "url" && sourcesWithPlacement >= URL_SOURCES_JUDGED) break;
|
|
35281
|
+
const window = sources.slice(at, at + SOURCE_WINDOW);
|
|
35282
|
+
deps.progress?.(`Placing and judging links from pages ${at + 1} to ${at + window.length} of ${sources.length}.`);
|
|
35283
|
+
let pairs = (await Promise.all(window.map(async (source) => {
|
|
35284
|
+
const page = pages[source];
|
|
35285
|
+
if (!page) return [];
|
|
35286
|
+
const sentences = await crawl.sentences.read(page.text);
|
|
35287
|
+
const placed = [];
|
|
35288
|
+
for (const candidate of candidatesBySource.get(source) ?? []) {
|
|
35289
|
+
pairsConsidered += 1;
|
|
35290
|
+
const to = pages[candidate.target];
|
|
35291
|
+
if (!to) continue;
|
|
35292
|
+
const placement = pairPlacement(sentences, to, index);
|
|
35293
|
+
if (!placement.sentences.length) continue;
|
|
35294
|
+
placed.push({
|
|
35295
|
+
candidate,
|
|
35296
|
+
placement
|
|
35297
|
+
});
|
|
35298
|
+
}
|
|
35299
|
+
return placed;
|
|
35300
|
+
}))).flat();
|
|
35301
|
+
if (context.mode === "url") {
|
|
35302
|
+
pairs = pairs.sort((left, right) => Number(right.candidate.sharedQueries > 0) - Number(left.candidate.sharedQueries > 0) || right.candidate.similarity - left.candidate.similarity || left.candidate.source - right.candidate.source).slice(0, URL_SOURCES_JUDGED - sourcesWithPlacement);
|
|
35303
|
+
sourcesWithPlacement += pairs.length;
|
|
35304
|
+
}
|
|
35305
|
+
const bySource = /* @__PURE__ */ new Map();
|
|
35306
|
+
for (const pair of pairs) {
|
|
35307
|
+
const list = bySource.get(pair.candidate.source) ?? [];
|
|
35308
|
+
list.push(pair);
|
|
35309
|
+
bySource.set(pair.candidate.source, list);
|
|
35310
|
+
}
|
|
35311
|
+
const groups = [];
|
|
35312
|
+
for (const list of bySource.values()) for (let start = 0; start < list.length; start += 2) groups.push(list.slice(start, start + 2));
|
|
35313
|
+
await Promise.all(groups.map(async (group) => {
|
|
35314
|
+
const first = group[0];
|
|
35315
|
+
if (!first) return;
|
|
35316
|
+
const source = pages[first.candidate.source];
|
|
35317
|
+
if (!source) return;
|
|
35318
|
+
const response = runner ? await runner.ask(judgeRequest({
|
|
35319
|
+
url: source.url,
|
|
35320
|
+
title: source.title,
|
|
35321
|
+
type: types[first.candidate.source] ?? null
|
|
35322
|
+
}, group.map(({ candidate, placement }) => {
|
|
35323
|
+
const to = pages[candidate.target];
|
|
35324
|
+
return {
|
|
35325
|
+
url: to.url,
|
|
35326
|
+
title: to.title,
|
|
35327
|
+
description: to.description,
|
|
35328
|
+
h1: to.h1,
|
|
35329
|
+
type: types[candidate.target] ?? null,
|
|
35330
|
+
intent: intents[candidate.target] ?? null,
|
|
35331
|
+
opening: to.excerpt.slice(0, OPENING_CHARS),
|
|
35332
|
+
placement
|
|
35333
|
+
};
|
|
35334
|
+
}))) : null;
|
|
35335
|
+
group.forEach(({ candidate, placement }, position) => {
|
|
35336
|
+
let decision;
|
|
35337
|
+
if (response) {
|
|
35338
|
+
pairsJudged += 1;
|
|
35339
|
+
decision = decideWithModel(placement, candidate.similarity, response.answers, position);
|
|
35340
|
+
} else {
|
|
35341
|
+
if (runner) fellBack += 1;
|
|
35342
|
+
decision = decideWithoutModel(placement, candidate.similarity);
|
|
35343
|
+
}
|
|
35344
|
+
judged.push({
|
|
35345
|
+
candidate,
|
|
35346
|
+
placement,
|
|
35347
|
+
decision
|
|
35348
|
+
});
|
|
35349
|
+
});
|
|
35350
|
+
}));
|
|
35351
|
+
}
|
|
35352
|
+
const heuristicScore = (item) => item.candidate.similarity * (.5 + (item.decision.state === "suggest" ? item.decision.sentence?.match ?? 0 : 0));
|
|
35353
|
+
const suggested = judged.filter((item) => item.decision.state === "suggest").sort((left, right) => (right.decision.probability ?? -1) - (left.decision.probability ?? -1) || heuristicScore(right) - heuristicScore(left) || (pages[left.candidate.source]?.url ?? "").localeCompare(pages[right.candidate.source]?.url ?? "") || (pages[left.candidate.target]?.url ?? "").localeCompare(pages[right.candidate.target]?.url ?? ""));
|
|
35354
|
+
const perTarget = /* @__PURE__ */ new Map();
|
|
35355
|
+
const usedSentences = /* @__PURE__ */ new Set();
|
|
35356
|
+
const kept = suggested.filter((item) => {
|
|
35357
|
+
if (item.decision.tier === "heuristic" && item.decision.sentence) {
|
|
35358
|
+
const key = `${item.candidate.source}\u0000${item.decision.sentence.text}`;
|
|
35359
|
+
if (usedSentences.has(key)) return false;
|
|
35360
|
+
usedSentences.add(key);
|
|
35361
|
+
}
|
|
35362
|
+
const seen = (perTarget.get(item.candidate.target) ?? 0) + 1;
|
|
35363
|
+
perTarget.set(item.candidate.target, seen);
|
|
35364
|
+
return seen <= 8;
|
|
35365
|
+
});
|
|
35366
|
+
const items = kept.map(({ candidate, decision }) => ({
|
|
35367
|
+
source: pages[candidate.source]?.url ?? "",
|
|
35368
|
+
target: pages[candidate.target]?.url ?? "",
|
|
35369
|
+
tier: decision.tier,
|
|
35370
|
+
probability: decision.probability,
|
|
35371
|
+
role: decision.role,
|
|
35372
|
+
anchor: decision.anchor,
|
|
35373
|
+
sentence: decision.sentence?.text ?? null,
|
|
35374
|
+
section: decision.sentence?.section || null,
|
|
35375
|
+
sentenceMatch: decision.sentence?.match ?? null,
|
|
35376
|
+
similarity: round$1(candidate.similarity),
|
|
35377
|
+
sharedQueries: candidate.sharedQueries,
|
|
35378
|
+
targetInboundLinks: inbound[candidate.target] ?? 0,
|
|
35379
|
+
sourceType: types[candidate.source] ?? null,
|
|
35380
|
+
targetType: types[candidate.target] ?? null
|
|
35381
|
+
}));
|
|
35382
|
+
const suggestedInbound = new Array(count).fill(0);
|
|
35383
|
+
const suggestedOutbound = new Array(count).fill(0);
|
|
35384
|
+
for (const { candidate } of kept) {
|
|
35385
|
+
suggestedInbound[candidate.target] = (suggestedInbound[candidate.target] ?? 0) + 1;
|
|
35386
|
+
suggestedOutbound[candidate.source] = (suggestedOutbound[candidate.source] ?? 0) + 1;
|
|
35387
|
+
}
|
|
35388
|
+
const held = judged.filter((item) => item.decision.state === "held-back");
|
|
35389
|
+
const reasons = {};
|
|
35390
|
+
for (const item of held) reasons[item.decision.reason] = (reasons[item.decision.reason] ?? 0) + 1;
|
|
35391
|
+
const declined = judged.filter((item) => item.decision.state === "declined").length;
|
|
35392
|
+
const usage = runner ? runner.usage() : null;
|
|
35393
|
+
const judgeStatus = !runner ? "not-used" : runner.authError || usage && usage.requests === 0 && usage.failedRequests > 0 ? "failed" : (usage?.failedRequests ?? 0) > 0 ? "partial" : "used";
|
|
35394
|
+
const modelLabel = usage ? `${usage.model}` : null;
|
|
35395
|
+
const judgeNote = !runner ? `Jev was not used: ${"unavailable" in deps.decisionModel ? deps.decisionModel.unavailable : "no TypeSafe key"}. Links come from TF-IDF similarity and sentence matching (heuristics). Add a TypeSafe API key at https://seoskill.dev/account#keys and run seo login, or set TYPESAFE_API_KEY on this machine, to have Jev classify pages and judge each link.` : runner.authError ? `${runner.authError.message} ${fellBack} link${fellBack === 1 ? "" : "s"} fell back to heuristics.` : judgeStatus === "partial" ? `${usage?.failedRequests} request${usage?.failedRequests === 1 ? "" : "s"} to ${modelLabel} failed after retries; ${fellBack} link${fellBack === 1 ? "" : "s"} fell back to heuristics.` : `${modelLabel} classified ${pagesClassified} page${pagesClassified === 1 ? "" : "s"} and judged ${pairsJudged} link${pairsJudged === 1 ? "" : "s"} in ${usage?.requests ?? 0} requests, for $${(usage?.costUsd ?? 0).toFixed(4)}.`;
|
|
35396
|
+
if (status === "complete" && (crawl.stats.capReached || judgeStatus === "partial" || judgeStatus === "failed")) status = "partial";
|
|
35397
|
+
const counts = {
|
|
35398
|
+
verdict: "",
|
|
35399
|
+
pagesCrawled: count,
|
|
35400
|
+
existingContextualLinks,
|
|
35401
|
+
orphanPages: inboundAnywhere.filter((value, id) => value === 0 && types[id] !== "home").length,
|
|
35402
|
+
weaklyLinkedPages: inbound.filter((value, id) => value < WEAK_INBOUND && types[id] !== "home").length,
|
|
35403
|
+
pairsConsidered,
|
|
35404
|
+
suggestions: items.length,
|
|
35405
|
+
strong: items.filter((item) => item.tier === "strong").length,
|
|
35406
|
+
possible: items.filter((item) => item.tier === "possible").length,
|
|
35407
|
+
heuristic: items.filter((item) => item.tier === "heuristic").length,
|
|
35408
|
+
heldBack: held.length,
|
|
35409
|
+
declined
|
|
35410
|
+
};
|
|
35411
|
+
const targetPage = target !== void 0 ? pages[target] : void 0;
|
|
35412
|
+
const report = {
|
|
34508
35413
|
site: input.site,
|
|
34509
|
-
|
|
34510
|
-
|
|
34511
|
-
|
|
34512
|
-
|
|
34513
|
-
|
|
34514
|
-
|
|
34515
|
-
|
|
34516
|
-
|
|
34517
|
-
|
|
34518
|
-
|
|
34519
|
-
|
|
34520
|
-
|
|
34521
|
-
|
|
34522
|
-
|
|
34523
|
-
|
|
34524
|
-
calls: result.calls,
|
|
34525
|
-
possiblyTruncated: result.rowsFetched >= MAX_GSC_ROWS
|
|
34526
|
-
})),
|
|
34527
|
-
rowsFetched: targetRequests.reduce((sum, { result }) => sum + result.rowsFetched, 0),
|
|
34528
|
-
calls: targetRequests.reduce((sum, { result }) => sum + result.calls, 0),
|
|
34529
|
-
maxRowsPerRequest: MAX_GSC_ROWS,
|
|
34530
|
-
possiblyTruncated: targetTruncated
|
|
34531
|
-
},
|
|
34532
|
-
candidates: {
|
|
34533
|
-
queried: queryCandidates,
|
|
34534
|
-
rowsFetched: sourceResult.rowsFetched,
|
|
34535
|
-
calls: sourceResult.calls,
|
|
34536
|
-
maxRows: MAX_GSC_ROWS,
|
|
34537
|
-
possiblyTruncated: sourceTruncated
|
|
34538
|
-
},
|
|
34539
|
-
completeness: !queryCandidates ? "not-queried" : targetTruncated || sourceTruncated ? "possibly-truncated" : "retained-rows-only"
|
|
35414
|
+
mode: context.mode,
|
|
35415
|
+
targetUrl: context.targetUrl ?? null,
|
|
35416
|
+
generatedAt: context.now.toISOString(),
|
|
35417
|
+
dataStatus: status,
|
|
35418
|
+
crawl: {
|
|
35419
|
+
startUrl: context.startUrl,
|
|
35420
|
+
method: crawl.method,
|
|
35421
|
+
maxPages: crawl.stats.maxPages,
|
|
35422
|
+
pageLimit: crawl.stats.pageLimit,
|
|
35423
|
+
urlsFound: crawl.stats.urlsFound,
|
|
35424
|
+
pagesCrawled: count,
|
|
35425
|
+
capReached: crawl.stats.capReached,
|
|
35426
|
+
crawlDelaySeconds: crawl.stats.crawlDelaySeconds,
|
|
35427
|
+
skipped: crawl.stats.skipped,
|
|
35428
|
+
redirectsChecked: crawl.stats.redirectsChecked
|
|
34540
35429
|
},
|
|
34541
|
-
|
|
34542
|
-
|
|
34543
|
-
|
|
34544
|
-
|
|
34545
|
-
|
|
34546
|
-
|
|
34547
|
-
|
|
34548
|
-
|
|
34549
|
-
|
|
34550
|
-
|
|
34551
|
-
|
|
34552
|
-
|
|
34553
|
-
|
|
34554
|
-
|
|
34555
|
-
|
|
34556
|
-
maxGscRowsPerRequest: MAX_GSC_ROWS
|
|
34557
|
-
},
|
|
34558
|
-
target: {
|
|
34559
|
-
requestedUrl: target.requestedUrl,
|
|
34560
|
-
preferredUrl: target.preferredUrl,
|
|
34561
|
-
finalUrl: target.finalUrl,
|
|
34562
|
-
canonical: target.canonical,
|
|
34563
|
-
status: target.status,
|
|
34564
|
-
aliases: target.aliases,
|
|
34565
|
-
verification: target.verification,
|
|
34566
|
-
technicalSignals: target.technicalSignals,
|
|
34567
|
-
fetchDiagnostics: target.fetchDiagnostics,
|
|
34568
|
-
queries: analysis.targetQueries
|
|
34569
|
-
},
|
|
34570
|
-
selection,
|
|
34571
|
-
summary: {
|
|
34572
|
-
targetQueries: analysis.selection.targetEligibleQueries,
|
|
34573
|
-
candidateSources: analysis.candidates.length,
|
|
34574
|
-
attemptedSources,
|
|
34575
|
-
checkedSources,
|
|
34576
|
-
returnedSources: items.length,
|
|
34577
|
-
existingLinksObserved: existingLinkExclusions,
|
|
34578
|
-
technicalExclusions,
|
|
34579
|
-
failedChecks,
|
|
34580
|
-
uncheckedCandidates,
|
|
34581
|
-
matchedQueryImpressions: items.reduce((sum, item) => sum + item.matchedQueryImpressions, 0),
|
|
34582
|
-
brandFiltering: "excluded",
|
|
34583
|
-
verdict: summaryVerdict
|
|
35430
|
+
searchConsole,
|
|
35431
|
+
judge: {
|
|
35432
|
+
status: judgeStatus,
|
|
35433
|
+
note: judgeNote,
|
|
35434
|
+
keySource: "model" in deps.decisionModel ? deps.decisionModel.keySource : null,
|
|
35435
|
+
provider: usage?.provider ?? null,
|
|
35436
|
+
model: usage?.model ?? null,
|
|
35437
|
+
requests: usage?.requests ?? 0,
|
|
35438
|
+
failedRequests: usage?.failedRequests ?? 0,
|
|
35439
|
+
inputTokens: usage?.inputTokens ?? 0,
|
|
35440
|
+
costUsd: usage?.costUsd ?? 0,
|
|
35441
|
+
pagesClassified,
|
|
35442
|
+
pairsJudged,
|
|
35443
|
+
thresholds: { ...THRESHOLDS },
|
|
35444
|
+
floors: { ...FLOORS }
|
|
34584
35445
|
},
|
|
35446
|
+
target: target !== void 0 && targetPage ? {
|
|
35447
|
+
url: targetPage.url,
|
|
35448
|
+
title: targetPage.title,
|
|
35449
|
+
type: types[target] ?? null,
|
|
35450
|
+
intent: intents[target] ?? null,
|
|
35451
|
+
inboundLinks: inbound[target] ?? 0,
|
|
35452
|
+
inboundLinksAnywhere: inboundAnywhere[target] ?? 0,
|
|
35453
|
+
noindex: targetPage.noindex
|
|
35454
|
+
} : null,
|
|
35455
|
+
summary: counts,
|
|
34585
35456
|
items,
|
|
34586
|
-
|
|
34587
|
-
|
|
34588
|
-
|
|
34589
|
-
|
|
34590
|
-
|
|
34591
|
-
|
|
34592
|
-
|
|
34593
|
-
|
|
34594
|
-
|
|
34595
|
-
|
|
34596
|
-
|
|
34597
|
-
|
|
34598
|
-
|
|
34599
|
-
|
|
34600
|
-
|
|
35457
|
+
heldBack: {
|
|
35458
|
+
count: held.length,
|
|
35459
|
+
reasons,
|
|
35460
|
+
examples: held.slice(0, EXAMPLE_LIMIT).map((item) => ({
|
|
35461
|
+
source: pages[item.candidate.source]?.url ?? "",
|
|
35462
|
+
target: pages[item.candidate.target]?.url ?? "",
|
|
35463
|
+
reason: item.decision.reason,
|
|
35464
|
+
detail: item.decision.detail
|
|
35465
|
+
}))
|
|
35466
|
+
},
|
|
35467
|
+
pageInventory: context.mode === "site" ? pages.map((page, id) => ({
|
|
35468
|
+
url: page.url,
|
|
35469
|
+
title: page.title,
|
|
35470
|
+
type: types[id] ?? null,
|
|
35471
|
+
intent: intents[id] ?? null,
|
|
35472
|
+
words: page.words,
|
|
35473
|
+
inboundLinks: inbound[id] ?? 0,
|
|
35474
|
+
inboundLinksAnywhere: inboundAnywhere[id] ?? 0,
|
|
35475
|
+
outboundLinks: contextual[id]?.size ?? 0,
|
|
35476
|
+
suggestedInbound: suggestedInbound[id] ?? 0,
|
|
35477
|
+
suggestedOutbound: suggestedOutbound[id] ?? 0,
|
|
35478
|
+
noindex: page.noindex
|
|
35479
|
+
})).sort((left, right) => left.inboundLinks - right.inboundLinks || left.url.localeCompare(right.url)) : [],
|
|
35480
|
+
redirectedLinks,
|
|
35481
|
+
genericAnchors: crawl.genericAnchors.map((anchor) => ({
|
|
35482
|
+
source: pages[anchor.source]?.url ?? "",
|
|
35483
|
+
href: anchor.href,
|
|
35484
|
+
text: anchor.text
|
|
35485
|
+
})),
|
|
35486
|
+
warnings,
|
|
35487
|
+
caveats: [],
|
|
35488
|
+
next: null
|
|
34601
35489
|
};
|
|
34602
|
-
|
|
34603
|
-
|
|
34604
|
-
|
|
34605
|
-
const placed = found.map(({ item, text }) => {
|
|
34606
|
-
const placement = target && text ? linkPlacement(text, target, corpus) : {
|
|
34607
|
-
contentSimilarity: 0,
|
|
34608
|
-
sentences: []
|
|
34609
|
-
};
|
|
34610
|
-
const best = placement.sentences[0];
|
|
34611
|
-
const recommendation = best ? {
|
|
34612
|
-
...item.recommendation,
|
|
34613
|
-
action: `${item.recommendation.action} It fits in: "${best.text}" Anchor text could be ${best.anchors.map((anchor) => `"${anchor}"`).join(" or ")}.`
|
|
34614
|
-
} : {
|
|
34615
|
-
...item.recommendation,
|
|
34616
|
-
action: `${item.recommendation.action} No sentence on this page names the target, so a link here needs new text.`
|
|
34617
|
-
};
|
|
34618
|
-
return {
|
|
34619
|
-
...item,
|
|
34620
|
-
placement,
|
|
34621
|
-
recommendation
|
|
34622
|
-
};
|
|
35490
|
+
report.summary.verdict = internalLinksVerdict({
|
|
35491
|
+
...report,
|
|
35492
|
+
counts
|
|
34623
35493
|
});
|
|
34624
|
-
|
|
35494
|
+
report.caveats = caveats$1(report);
|
|
35495
|
+
const top = items[0];
|
|
35496
|
+
report.next = context.mode === "site" ? top ? `seo internal-links --url ${top.target}` : null : `seo page --url ${report.target?.url ?? context.targetUrl}`;
|
|
35497
|
+
return report;
|
|
35498
|
+
}
|
|
35499
|
+
function caveats$1(report) {
|
|
35500
|
+
const crawl = report.crawl;
|
|
35501
|
+
return [
|
|
35502
|
+
`Crawl: ${crawl.pagesCrawled} page${crawl.pagesCrawled === 1 ? "" : "s"} from ${crawl.method === "sitemap" ? "the sitemap" : "links from the start page"}, limit ${crawl.pageLimit}${crawl.capReached ? " (reached, so pages beyond it were not checked and orphan and weak-link counts cover crawled pages only)" : " (not reached)"}.${crawl.crawlDelaySeconds ? ` robots.txt crawl-delay of ${crawl.crawlDelaySeconds}s was honoured.` : ""}`,
|
|
35503
|
+
report.judge.note,
|
|
35504
|
+
report.searchConsole.status === "used" ? `Search Console: ${report.searchConsole.rows.toLocaleString("en-US")} query rows from ${report.searchConsole.range?.startDate} to ${report.searchConsole.range?.endDate}, brand queries left out; ${report.searchConsole.pairsWithSharedQueries} page pairs share a query.${report.searchConsole.possiblyTruncated ? " The 100,000-row cap was reached." : ""}` : `Search Console was not used${report.searchConsole.note ? `: ${report.searchConsole.note}` : ""}. Candidates come from page text only.`,
|
|
35505
|
+
`Rules in code: a link needs a model probability of ${THRESHOLDS.possible} (${THRESHOLDS.strong} is strong), a sentence with at least ${FLOORS.minSentenceMatch * 100}% of the words that name the target or pages at least ${FLOORS.minSimilarityAlone} similar, and an anchor from the chosen sentence. At most 8 links per target.`,
|
|
35506
|
+
"Similarity, sentence match and anchors are text heuristics, and a model probability is a judgement of reader value. None of them predicts rankings or traffic."
|
|
35507
|
+
];
|
|
34625
35508
|
}
|
|
34626
35509
|
//#endregion
|
|
34627
35510
|
//#region packages/core/dist/analyze/url-templates.js
|
|
@@ -36010,7 +36893,7 @@ function reportId$1(input) {
|
|
|
36010
36893
|
})).digest("hex").slice(0, 20)}`;
|
|
36011
36894
|
}
|
|
36012
36895
|
function cacheGet(id, now) {
|
|
36013
|
-
const row =
|
|
36896
|
+
const row = cacheDb().prepare("SELECT report_json FROM performance_reports WHERE id = ? AND expires_at > ?").get(id, now);
|
|
36014
36897
|
if (!row?.report_json) return void 0;
|
|
36015
36898
|
try {
|
|
36016
36899
|
const report = JSON.parse(row.report_json);
|
|
@@ -36025,10 +36908,12 @@ function cacheSet(report, now) {
|
|
|
36025
36908
|
raw: void 0
|
|
36026
36909
|
};
|
|
36027
36910
|
const reportJson = JSON.stringify(stored);
|
|
36028
|
-
|
|
36029
|
-
|
|
36030
|
-
|
|
36031
|
-
|
|
36911
|
+
writeToCache((database) => {
|
|
36912
|
+
database.prepare(`INSERT OR REPLACE INTO performance_reports
|
|
36913
|
+
(id, url, strategy, report_json, created_at, expires_at)
|
|
36914
|
+
VALUES (?, ?, ?, ?, ?, ?)`).run(report.id, report.url, report.strategy, reportJson, Date.parse(report.generatedAt), now + CACHE_TTL_MS);
|
|
36915
|
+
noteCacheWrite(Buffer.byteLength(reportJson), database);
|
|
36916
|
+
});
|
|
36032
36917
|
}
|
|
36033
36918
|
async function fallbackReport(input) {
|
|
36034
36919
|
const fetched = await fetchPage(input.url, {
|
|
@@ -37148,10 +38033,10 @@ function applyVerification(item, coverage) {
|
|
|
37148
38033
|
function verdict(report) {
|
|
37149
38034
|
if (report.dataStatus === "empty") return "Google Search Console returned no retained query/page rows for this window.";
|
|
37150
38035
|
if (report.dataStatus === "filtered") return "No pages met the position, brand, and minimum-impression criteria.";
|
|
37151
|
-
if (report.technical > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.technical, "returned page")} ${plural$
|
|
37152
|
-
if (report.failed > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.failed, "returned page")} could not be verified and ${plural$
|
|
37153
|
-
if (report.content > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.content, "returned page")} ${plural$
|
|
37154
|
-
return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.returned, "page")} ${plural$
|
|
38036
|
+
if (report.technical > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.technical, "returned page")} ${plural$4(report.technical, "has", "have")} verified technical issues to fix before content changes.`;
|
|
38037
|
+
if (report.failed > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.failed, "returned page")} could not be verified and ${plural$4(report.failed, "needs", "need")} a fetch check before content changes.`;
|
|
38038
|
+
if (report.content > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.content, "returned page")} ${plural$4(report.content, "has", "have")} verified content or search-framing gaps.`;
|
|
38039
|
+
return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.returned, "page")} ${plural$4(report.returned, "is", "are")} returned in priority order for investigation.`;
|
|
37155
38040
|
}
|
|
37156
38041
|
function reportRecommendations(items) {
|
|
37157
38042
|
const first = items[0];
|
|
@@ -38212,7 +39097,7 @@ async function updatePostmortemWorkflow(input, dependencies = {}) {
|
|
|
38212
39097
|
//#endregion
|
|
38213
39098
|
//#region packages/core/dist/clicky/api.js
|
|
38214
39099
|
const ENDPOINT$1 = "https://api.clicky.com/api/stats/4";
|
|
38215
|
-
const MAX_BYTES$
|
|
39100
|
+
const MAX_BYTES$2 = 5e6;
|
|
38216
39101
|
var ClickyError = class extends Error {
|
|
38217
39102
|
code;
|
|
38218
39103
|
constructor(code, message) {
|
|
@@ -38239,7 +39124,7 @@ async function clickyStats(site, types, options = {}) {
|
|
|
38239
39124
|
throw new ClickyError("remote", "Could not reach Clicky. Try again.");
|
|
38240
39125
|
});
|
|
38241
39126
|
if (!response.ok) throw new ClickyError("remote", `Clicky returned ${response.status}.`);
|
|
38242
|
-
const groups = await readJson(response, "Clicky", { maxBytes: MAX_BYTES$
|
|
39127
|
+
const groups = await readJson(response, "Clicky", { maxBytes: MAX_BYTES$2 });
|
|
38243
39128
|
if (!Array.isArray(groups)) throw new ClickyError("remote", "Clicky returned invalid data.");
|
|
38244
39129
|
const failed = groups.find((group) => typeof group?.error === "string");
|
|
38245
39130
|
if (failed?.error) throw /sitekey|site id|authenticat|permission/iu.test(failed.error) ? new ClickyError("auth", `Clicky rejected site ID ${site.siteId} or its sitekey.`) : new ClickyError("remote", `Clicky: ${failed.error.slice(0, 200)}`);
|
|
@@ -39995,10 +40880,69 @@ async function withDomainRating(rows, key) {
|
|
|
39995
40880
|
};
|
|
39996
40881
|
}
|
|
39997
40882
|
//#endregion
|
|
40883
|
+
//#region packages/core/dist/commands/internal-links.js
|
|
40884
|
+
const SUMMARY_ROWS = 10;
|
|
40885
|
+
const PAGE_SIZE$1 = 50;
|
|
40886
|
+
function rows(report, from, size) {
|
|
40887
|
+
return report.items.slice(from, from + size).map((item) => ({
|
|
40888
|
+
source: item.source,
|
|
40889
|
+
target: item.target,
|
|
40890
|
+
tier: item.tier,
|
|
40891
|
+
p: item.probability ?? "",
|
|
40892
|
+
role: item.role ?? "",
|
|
40893
|
+
anchor: item.anchor ?? "",
|
|
40894
|
+
sentence: item.sentence ?? "no sentence: needs new text"
|
|
40895
|
+
}));
|
|
40896
|
+
}
|
|
40897
|
+
function internalLinksText(report, options) {
|
|
40898
|
+
if (options.detail === "full") return `${report.summary.verdict}\n\n${JSON.stringify(report)}`;
|
|
40899
|
+
const origin = (() => {
|
|
40900
|
+
try {
|
|
40901
|
+
return new URL(report.items[0]?.source ?? report.target?.url ?? report.crawl.startUrl).origin;
|
|
40902
|
+
} catch {
|
|
40903
|
+
return null;
|
|
40904
|
+
}
|
|
40905
|
+
})();
|
|
40906
|
+
const columns = [
|
|
40907
|
+
"source",
|
|
40908
|
+
"target",
|
|
40909
|
+
"tier",
|
|
40910
|
+
"p",
|
|
40911
|
+
"role",
|
|
40912
|
+
"anchor",
|
|
40913
|
+
"sentence"
|
|
40914
|
+
];
|
|
40915
|
+
if (options.detail === "rows") {
|
|
40916
|
+
const page = rows(report, options.offset, PAGE_SIZE$1);
|
|
40917
|
+
const end = options.offset + page.length;
|
|
40918
|
+
return [
|
|
40919
|
+
`Internal links · ${report.mode === "url" ? report.targetUrl : report.site} · ${report.dataStatus}`,
|
|
40920
|
+
`Links ${options.offset + 1} to ${end} of ${report.items.length}:`,
|
|
40921
|
+
table(page, columns, origin, 120),
|
|
40922
|
+
end < report.items.length ? `Next page: ${options.more("rows", end)}` : ""
|
|
40923
|
+
].filter(Boolean).join("\n\n");
|
|
40924
|
+
}
|
|
40925
|
+
const s = report.summary;
|
|
40926
|
+
const judge = report.judge;
|
|
40927
|
+
const shown = rows(report, 0, SUMMARY_ROWS);
|
|
40928
|
+
const held = Object.entries(report.heldBack.reasons).map(([reason, count]) => `${reason} ${count}`).join(", ");
|
|
40929
|
+
return [
|
|
40930
|
+
`Internal links · ${report.mode === "url" ? report.targetUrl : report.site} · ${report.dataStatus}`,
|
|
40931
|
+
s.verdict,
|
|
40932
|
+
`Crawl: ${s.pagesCrawled} pages (limit ${report.crawl.pageLimit}, ${report.crawl.capReached ? "reached" : "not reached"}), ${s.existingContextualLinks} contextual links, ${s.orphanPages} orphan pages, ${s.weaklyLinkedPages} pages with fewer than 3 contextual links in.`,
|
|
40933
|
+
`Judge: ${judge.note}`,
|
|
40934
|
+
report.target ? `Target: ${report.target.type ?? "type unknown"}, ${report.target.intent ?? "intent unknown"}, ${report.target.inboundLinks} contextual links in.` : "",
|
|
40935
|
+
s.heldBack ? `Held back by the evidence floors: ${held}.` : "",
|
|
40936
|
+
shown.length ? `Links (${shown.length} of ${report.items.length}):\n${table(shown, columns, origin, 90)}` : "No links passed the rules.",
|
|
40937
|
+
`Limits:\n${report.caveats.filter((line) => line !== judge.note).map((line) => `- ${line}`).join("\n")}`,
|
|
40938
|
+
`More: ${shown.length < report.items.length ? `${options.more("rows", shown.length)} for more links, or ` : ""}${options.more("full")} for every field.${report.next ? ` Next: ${report.next}` : ""}`
|
|
40939
|
+
].filter(Boolean).join("\n\n");
|
|
40940
|
+
}
|
|
40941
|
+
//#endregion
|
|
39998
40942
|
//#region packages/core/dist/commands/page-facts.js
|
|
39999
40943
|
const MAX_PAGES = 20;
|
|
40000
|
-
const TIMEOUT_MS = 6e3;
|
|
40001
|
-
const MAX_BYTES = 1e6;
|
|
40944
|
+
const TIMEOUT_MS$1 = 6e3;
|
|
40945
|
+
const MAX_BYTES$1 = 1e6;
|
|
40002
40946
|
const clean = (value) => value?.trim().replace(/\s+/g, " ") || null;
|
|
40003
40947
|
async function boundedText(response) {
|
|
40004
40948
|
if (!response.body) return "";
|
|
@@ -40006,7 +40950,7 @@ async function boundedText(response) {
|
|
|
40006
40950
|
const decoder = new TextDecoder();
|
|
40007
40951
|
let text = "";
|
|
40008
40952
|
let total = 0;
|
|
40009
|
-
while (total < MAX_BYTES) {
|
|
40953
|
+
while (total < MAX_BYTES$1) {
|
|
40010
40954
|
const { done, value } = await reader.read();
|
|
40011
40955
|
if (done) return text + decoder.decode();
|
|
40012
40956
|
total += value.byteLength;
|
|
@@ -40086,7 +41030,7 @@ async function facts(url) {
|
|
|
40086
41030
|
"user-agent": SEO_CRAWLER_USER_AGENT,
|
|
40087
41031
|
accept: "text/html"
|
|
40088
41032
|
},
|
|
40089
|
-
signal: AbortSignal.timeout(TIMEOUT_MS)
|
|
41033
|
+
signal: AbortSignal.timeout(TIMEOUT_MS$1)
|
|
40090
41034
|
});
|
|
40091
41035
|
out.status = response.status;
|
|
40092
41036
|
const location = response.headers.get("location");
|
|
@@ -41455,6 +42399,25 @@ async function localResearchKeys() {
|
|
|
41455
42399
|
if (ahrefs?.apiKey) keys.set("x-ahrefs-key", ahrefs.apiKey);
|
|
41456
42400
|
return keys;
|
|
41457
42401
|
}
|
|
42402
|
+
const TYPESAFE_API_KEY_ENV = "TYPESAFE_API_KEY";
|
|
42403
|
+
async function localInternalLinksDependencies(options) {
|
|
42404
|
+
const searchConsole = await resolveLocalSite(options.site, { refresh: options.refresh }).then((property) => ({
|
|
42405
|
+
property,
|
|
42406
|
+
query: (request) => querySearchAnalytics(property, request, { refresh: options.refresh })
|
|
42407
|
+
}), (error) => ({ unavailable: `no Search Console property was found for this site (${error instanceof Error ? error.message : String(error)})` }));
|
|
42408
|
+
const local = process.env[TYPESAFE_API_KEY_ENV]?.trim();
|
|
42409
|
+
const saved = local ? void 0 : (await localResearchKeys().catch(() => new Headers())).get("x-typesafe-key");
|
|
42410
|
+
return {
|
|
42411
|
+
searchConsole,
|
|
42412
|
+
decisionModel: local ? {
|
|
42413
|
+
model: typesafeDecisionModel({ apiKey: local }),
|
|
42414
|
+
keySource: "environment"
|
|
42415
|
+
} : saved ? {
|
|
42416
|
+
model: typesafeDecisionModel({ apiKey: saved }),
|
|
42417
|
+
keySource: "account"
|
|
42418
|
+
} : { unavailable: "no TypeSafe key is set" }
|
|
42419
|
+
};
|
|
42420
|
+
}
|
|
41458
42421
|
async function localRefreshClicky() {
|
|
41459
42422
|
if (!cloudLinked()) return null;
|
|
41460
42423
|
await cloudCredentials({ refreshClicky: true });
|
|
@@ -41840,6 +42803,94 @@ var LocalCredentialsProvider = class {
|
|
|
41840
42803
|
};
|
|
41841
42804
|
new LocalCredentialsProvider();
|
|
41842
42805
|
//#endregion
|
|
42806
|
+
//#region packages/core/dist/update-notice.js
|
|
42807
|
+
const REGISTRY_URL = "https://registry.npmjs.org/seo/latest";
|
|
42808
|
+
const CHECK_EVERY_MS = 864e5;
|
|
42809
|
+
const TIMEOUT_MS = 2e3;
|
|
42810
|
+
const MAX_BYTES = 65536;
|
|
42811
|
+
let checking = false;
|
|
42812
|
+
function runByNpx(env, cliPath) {
|
|
42813
|
+
if (env.npm_command === "exec" || /npx/i.test(env.npm_execpath ?? "")) return true;
|
|
42814
|
+
if (!cliPath) return false;
|
|
42815
|
+
let path = cliPath;
|
|
42816
|
+
try {
|
|
42817
|
+
path = realpathSync(cliPath);
|
|
42818
|
+
} catch {}
|
|
42819
|
+
return /[\\/](?:_npx|dlx)[\\/]/.test(path);
|
|
42820
|
+
}
|
|
42821
|
+
function isNewerVersion(current, latest) {
|
|
42822
|
+
const parse = (value) => value.replace(/^v/, "").split("-")[0]?.split(".").map((part) => Number.parseInt(part, 10)) ?? [];
|
|
42823
|
+
const currentParts = parse(current);
|
|
42824
|
+
const latestParts = parse(latest);
|
|
42825
|
+
for (let index = 0; index < 3; index += 1) {
|
|
42826
|
+
const currentPart = currentParts[index] ?? 0;
|
|
42827
|
+
const latestPart = latestParts[index] ?? 0;
|
|
42828
|
+
if (!Number.isFinite(currentPart) || !Number.isFinite(latestPart)) return false;
|
|
42829
|
+
if (latestPart > currentPart) return true;
|
|
42830
|
+
if (latestPart < currentPart) return false;
|
|
42831
|
+
}
|
|
42832
|
+
return false;
|
|
42833
|
+
}
|
|
42834
|
+
function isLatestCache(value) {
|
|
42835
|
+
if (!value || typeof value !== "object") return false;
|
|
42836
|
+
const cache = value;
|
|
42837
|
+
return typeof cache.checkedAt === "number" && typeof cache.latest === "string" && /^\d+\.\d+\.\d+$/.test(cache.latest);
|
|
42838
|
+
}
|
|
42839
|
+
function checkRegistry(cacheFile, now) {
|
|
42840
|
+
if (checking) return;
|
|
42841
|
+
checking = true;
|
|
42842
|
+
const done = () => {
|
|
42843
|
+
checking = false;
|
|
42844
|
+
};
|
|
42845
|
+
const req = request(REGISTRY_URL, { headers: { accept: "application/json" } });
|
|
42846
|
+
const timer = setTimeout(() => req.destroy(), TIMEOUT_MS);
|
|
42847
|
+
timer.unref();
|
|
42848
|
+
req.on("socket", (socket) => socket.unref());
|
|
42849
|
+
req.on("error", () => void 0);
|
|
42850
|
+
req.on("close", () => {
|
|
42851
|
+
clearTimeout(timer);
|
|
42852
|
+
done();
|
|
42853
|
+
});
|
|
42854
|
+
req.on("response", (response) => {
|
|
42855
|
+
const chunks = [];
|
|
42856
|
+
let size = 0;
|
|
42857
|
+
response.on("error", () => void 0);
|
|
42858
|
+
response.on("data", (chunk) => {
|
|
42859
|
+
size += chunk.length;
|
|
42860
|
+
if (size > MAX_BYTES) req.destroy();
|
|
42861
|
+
else chunks.push(chunk);
|
|
42862
|
+
});
|
|
42863
|
+
response.on("end", () => {
|
|
42864
|
+
if (response.statusCode !== 200 || size > MAX_BYTES) return;
|
|
42865
|
+
readJson(new Response(Buffer.concat(chunks)), "The npm registry").then((body) => {
|
|
42866
|
+
const cache = {
|
|
42867
|
+
checkedAt: now,
|
|
42868
|
+
latest: body.version
|
|
42869
|
+
};
|
|
42870
|
+
if (isLatestCache(cache)) writeJsonAtomic(cacheFile, cache, 384);
|
|
42871
|
+
}).catch(() => void 0);
|
|
42872
|
+
});
|
|
42873
|
+
});
|
|
42874
|
+
req.end();
|
|
42875
|
+
}
|
|
42876
|
+
function updateNotice(options = {}) {
|
|
42877
|
+
try {
|
|
42878
|
+
const env = options.env ?? process.env;
|
|
42879
|
+
const argv = options.argv ?? process.argv;
|
|
42880
|
+
if (env.NO_UPDATE_NOTIFIER || argv.includes("--no-update-notifier") || isTelemetryCi(env)) return;
|
|
42881
|
+
const cacheFile = options.cacheFile ?? join(getSeoCliPaths().cacheDir, "latest-version.json");
|
|
42882
|
+
const now = (options.now ?? /* @__PURE__ */ new Date()).getTime();
|
|
42883
|
+
const stored = readJsonFile(cacheFile);
|
|
42884
|
+
const cache = isLatestCache(stored) ? stored : void 0;
|
|
42885
|
+
if (!cache || now - cache.checkedAt >= CHECK_EVERY_MS) checkRegistry(cacheFile, now);
|
|
42886
|
+
const current = options.version ?? SEO_VERSION;
|
|
42887
|
+
if (!cache || !isNewerVersion(current, cache.latest)) return void 0;
|
|
42888
|
+
return runByNpx(env, options.cliPath ?? argv[1]) ? `seo ${cache.latest} is out (you have ${current}). Run npx -y seo@latest; in an MCP config use seo@latest, then restart the client.` : `seo ${cache.latest} is out (you have ${current}). Update with: npm i -g seo`;
|
|
42889
|
+
} catch {
|
|
42890
|
+
return;
|
|
42891
|
+
}
|
|
42892
|
+
}
|
|
42893
|
+
//#endregion
|
|
41843
42894
|
//#region packages/mcp/src/agent-output-budget.ts
|
|
41844
42895
|
const AGENT_STRUCTURED_OUTPUT_MAX_BYTES = 96 * 1024;
|
|
41845
42896
|
function jsonBytes(value) {
|
|
@@ -42899,22 +43950,27 @@ function registerOpportunityTools(server, dependencies = {}) {
|
|
|
42899
43950
|
}
|
|
42900
43951
|
});
|
|
42901
43952
|
server.registerTool("seo_internal_links", {
|
|
42902
|
-
description: "
|
|
43953
|
+
description: "Internal links to add across a site, or to one target URL, placed in real sentences",
|
|
42903
43954
|
inputSchema: {
|
|
42904
43955
|
...mcpReportInputSchema(["site", "refresh"]),
|
|
42905
|
-
targetUrl: z$1.string().url().refine((value) => /^https?:\/\//.test(value), "Use an HTTP(S) URL."),
|
|
43956
|
+
targetUrl: z$1.string().url().refine((value) => /^https?:\/\//.test(value), "Use an HTTP(S) URL.").optional(),
|
|
43957
|
+
maxPages: z$1.number().int().min(1).max(MAX_MAX_PAGES).optional(),
|
|
42906
43958
|
startDate: z$1.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional(),
|
|
42907
43959
|
endDate: z$1.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional()
|
|
42908
43960
|
}
|
|
42909
|
-
}, async ({ site, targetUrl, startDate, endDate, refresh }) => {
|
|
43961
|
+
}, async ({ site, targetUrl, maxPages, startDate, endDate, refresh }) => {
|
|
42910
43962
|
try {
|
|
42911
43963
|
const result = await (dependencies.internalLinksReport ?? internalLinksReport)({
|
|
42912
43964
|
site,
|
|
42913
43965
|
targetUrl,
|
|
43966
|
+
maxPages,
|
|
42914
43967
|
startDate,
|
|
42915
43968
|
endDate,
|
|
42916
43969
|
refresh
|
|
42917
|
-
}
|
|
43970
|
+
}, await localInternalLinksDependencies({
|
|
43971
|
+
site: targetUrl ?? site,
|
|
43972
|
+
refresh
|
|
43973
|
+
}));
|
|
42918
43974
|
return toolSuccess(result.summary.verdict, result);
|
|
42919
43975
|
} catch (error) {
|
|
42920
43976
|
return toolError(error);
|
|
@@ -43747,11 +44803,11 @@ const REPORT_GUIDANCE = {
|
|
|
43747
44803
|
outcome: "Cross-source URL groups with clear evidence limits and a representative review queue for URL Inspection."
|
|
43748
44804
|
},
|
|
43749
44805
|
"internal-links": {
|
|
43750
|
-
name: "Internal
|
|
43751
|
-
description: "
|
|
43752
|
-
useWhen: ["
|
|
43753
|
-
avoidWhen: ["You want links added automatically.", "The
|
|
43754
|
-
outcome: "
|
|
44806
|
+
name: "Internal links",
|
|
44807
|
+
description: "Crawl a site and find the internal links it is missing, across the whole site or to one target URL, each placed in a real sentence with a real phrase as anchor text.",
|
|
44808
|
+
useWhen: ["Pages that belong together do not link to each other.", "A target page needs more contextual links from the rest of the site."],
|
|
44809
|
+
avoidWhen: ["You want links added automatically.", "The site cannot be crawled from this machine."],
|
|
44810
|
+
outcome: "Links to add with the source sentence and anchor, judged by Jev when a TypeSafe key is set, plus orphan and weakly linked pages."
|
|
43755
44811
|
},
|
|
43756
44812
|
"link-recovery": {
|
|
43757
44813
|
name: "Recover broken search-value URLs",
|
|
@@ -44381,7 +45437,7 @@ function text(value) {
|
|
|
44381
45437
|
text: value
|
|
44382
45438
|
}] };
|
|
44383
45439
|
}
|
|
44384
|
-
async function
|
|
45440
|
+
async function countedRun(command, args, telemetry, run) {
|
|
44385
45441
|
const report = telemetry ? telemetryReportFor(command, {
|
|
44386
45442
|
section: typeof args.section === "string" ? args.section : void 0,
|
|
44387
45443
|
action: typeof args.action === "string" ? args.action : void 0
|
|
@@ -44393,10 +45449,22 @@ async function counted(command, args, telemetry, run) {
|
|
|
44393
45449
|
else trackTelemetryReportComplete(report, telemetry());
|
|
44394
45450
|
return result;
|
|
44395
45451
|
} catch (error) {
|
|
44396
|
-
|
|
44397
|
-
|
|
45452
|
+
const failure = localError(error);
|
|
45453
|
+
if (report && telemetry) trackTelemetryReportFailed(report, failure, telemetry());
|
|
45454
|
+
return toolError(failure);
|
|
44398
45455
|
}
|
|
44399
45456
|
}
|
|
45457
|
+
async function counted(command, args, telemetry, run) {
|
|
45458
|
+
const result = await countedRun(command, args, telemetry, run);
|
|
45459
|
+
const notice = updateNotice();
|
|
45460
|
+
return notice ? {
|
|
45461
|
+
...result,
|
|
45462
|
+
content: [...result.content, {
|
|
45463
|
+
type: "text",
|
|
45464
|
+
text: notice
|
|
45465
|
+
}]
|
|
45466
|
+
} : result;
|
|
45467
|
+
}
|
|
44400
45468
|
function registerSharedTools(server, options = {}) {
|
|
44401
45469
|
const surface = {
|
|
44402
45470
|
deps: nodeCommandDeps({ call: mcpCall }),
|
|
@@ -44426,22 +45494,42 @@ function registerSharedTools(server, options = {}) {
|
|
|
44426
45494
|
saveReport: true
|
|
44427
45495
|
})));
|
|
44428
45496
|
server.registerTool("seo_internal_links", {
|
|
44429
|
-
title: "Internal
|
|
44430
|
-
description: "
|
|
45497
|
+
title: "Internal links",
|
|
45498
|
+
description: "Internal links to add, crawled from this machine. Send site to audit a whole site, or url to find links to one page. Each link names a real sentence on the source and a real phrase from it as anchor text. With a TypeSafe key (TYPESAFE_API_KEY or the seoskill.dev account), Jev classifies pages and judges each link; without one the result says Jev was not used. Search Console query overlap is added when this login has the property.",
|
|
44431
45499
|
inputSchema: {
|
|
44432
|
-
|
|
44433
|
-
|
|
44434
|
-
|
|
45500
|
+
site: z$1.string().min(1).optional().describe("Site to audit: a domain, URL or Search Console property."),
|
|
45501
|
+
url: z$1.url().optional().describe("One page to find links to."),
|
|
45502
|
+
max_pages: z$1.number().int().min(1).max(MAX_MAX_PAGES).optional().describe("site only: pages to crawl, 500 by default, up to 5,000."),
|
|
45503
|
+
start_date: z$1.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional().describe("YYYY-MM-DD for Search Console. Optional; send with end_date."),
|
|
45504
|
+
end_date: z$1.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional().describe("YYYY-MM-DD. Optional; send with start_date."),
|
|
45505
|
+
detail: z$1.enum([
|
|
45506
|
+
"summary",
|
|
45507
|
+
"rows",
|
|
45508
|
+
"full"
|
|
45509
|
+
]).default("summary").describe("summary first. rows: the next page of links. full: every field as JSON."),
|
|
45510
|
+
offset: z$1.number().int().min(0).default(0).describe("rows only: the first link to return.")
|
|
44435
45511
|
},
|
|
44436
45512
|
annotations: readOnly
|
|
44437
|
-
}, async ({ url, start_date, end_date }) => counted("internal-links", {}, options.telemetry, async () => {
|
|
45513
|
+
}, async ({ site, url, max_pages, start_date, end_date, detail, offset }) => counted("internal-links", {}, options.telemetry, async () => {
|
|
44438
45514
|
try {
|
|
44439
|
-
|
|
44440
|
-
|
|
45515
|
+
if (Boolean(site) === Boolean(url)) throw new SeoError("INVALID_INPUT", "Send site to audit a whole site, or url to find links to one page.");
|
|
45516
|
+
const target = url ?? site;
|
|
45517
|
+
return text(internalLinksText(await internalLinksReport({
|
|
45518
|
+
site: target,
|
|
44441
45519
|
targetUrl: url,
|
|
45520
|
+
maxPages: max_pages,
|
|
44442
45521
|
startDate: start_date,
|
|
44443
45522
|
endDate: end_date
|
|
44444
|
-
})
|
|
45523
|
+
}, await localInternalLinksDependencies({ site: target })), {
|
|
45524
|
+
detail,
|
|
45525
|
+
offset,
|
|
45526
|
+
more: (view, from) => mcpCall("internal-links", {
|
|
45527
|
+
...url ? { url } : { site },
|
|
45528
|
+
...max_pages ? { max_pages } : {},
|
|
45529
|
+
detail: view,
|
|
45530
|
+
offset: from
|
|
45531
|
+
})
|
|
45532
|
+
}));
|
|
44445
45533
|
} catch (error) {
|
|
44446
45534
|
return toolError(localError(error));
|
|
44447
45535
|
}
|
|
@@ -44489,6 +45577,7 @@ async function startMcpServer(opts = {}) {
|
|
|
44489
45577
|
}
|
|
44490
45578
|
const transport = new StdioServerTransport();
|
|
44491
45579
|
await server.connect(transport);
|
|
45580
|
+
updateNotice();
|
|
44492
45581
|
}
|
|
44493
45582
|
//#endregion
|
|
44494
45583
|
export { AGENT_STRUCTURED_OUTPUT_MAX_BYTES, REPORT_CATEGORIES, compactAgentWorkflowOutput, createServer, describeReport, executeReport, listReports, runReport, startMcpServer };
|