seo 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +12 -2
- package/README.md +1 -1
- package/dist/cli.js +81 -81
- package/dist/cli.js.map +1 -1
- package/dist/{dist-B3L_4U8E.js → dist-BvbFtizP.js} +1 -1
- package/dist/{dist-BdZyUFiw.js → dist-CsWo-wJG.js} +94 -89
- package/dist/dist-CsWo-wJG.js.map +1 -0
- package/dist/{esm-gP6ovS6b.js → esm-CM7xTOjT.js} +2 -2
- package/dist/{esm-gP6ovS6b.js.map → esm-CM7xTOjT.js.map} +1 -1
- package/dist/index.d.ts +354 -210
- package/dist/index.js +1765 -906
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +1874 -930
- package/dist/mcp.js.map +1 -1
- package/package.json +1 -1
- package/skills/seo/SKILL.md +1 -1
- package/dist/dist-BdZyUFiw.js.map +0 -1
package/dist/mcp.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-
|
|
2
|
+
import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-CM7xTOjT.js";
|
|
3
3
|
import { createHash, randomUUID, sign } from "node:crypto";
|
|
4
4
|
import { ZodOptional, z } from "zod";
|
|
5
|
-
import { arch, homedir, platform, totalmem } from "node:os";
|
|
5
|
+
import { arch, homedir, platform, tmpdir, totalmem } from "node:os";
|
|
6
6
|
import { dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
7
7
|
import "open";
|
|
8
8
|
import { chmodSync, createReadStream, existsSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
@@ -21,7 +21,7 @@ import { chromium } from "playwright-core";
|
|
|
21
21
|
import { gunzipSync } from "node:zlib";
|
|
22
22
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
23
23
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
24
|
-
import { open as open$1, readFile, stat } from "node:fs/promises";
|
|
24
|
+
import { mkdtemp, open as open$1, readFile, rm, stat } from "node:fs/promises";
|
|
25
25
|
import { TextDecoder as TextDecoder$1, promisify } from "node:util";
|
|
26
26
|
import { execFile } from "node:child_process";
|
|
27
27
|
import "pino";
|
|
@@ -13634,12 +13634,12 @@ var SeoError = class extends Error {
|
|
|
13634
13634
|
this.retryable = errorMetadata[code].retryable;
|
|
13635
13635
|
}
|
|
13636
13636
|
};
|
|
13637
|
-
function errorMessage$
|
|
13637
|
+
function errorMessage$1(error) {
|
|
13638
13638
|
return error instanceof Error ? error.message : String(error);
|
|
13639
13639
|
}
|
|
13640
13640
|
function toSeoError(error) {
|
|
13641
13641
|
if (error instanceof SeoError) return error;
|
|
13642
|
-
const message = errorMessage$
|
|
13642
|
+
const message = errorMessage$1(error);
|
|
13643
13643
|
if (/not logged in/i.test(message)) return new SeoError("AUTH_REQUIRED", message);
|
|
13644
13644
|
if (/oauth client config.*missing|shared seo google app/i.test(message)) return new SeoError("AUTH_CONFIG_REQUIRED", message);
|
|
13645
13645
|
if (/invalid_grant|refresh token.*no longer valid/i.test(message)) return new SeoError("AUTH_EXPIRED", message);
|
|
@@ -14100,6 +14100,262 @@ function decayComparisonRange(current, comparison = "previous-period") {
|
|
|
14100
14100
|
};
|
|
14101
14101
|
}
|
|
14102
14102
|
//#endregion
|
|
14103
|
+
//#region packages/core/dist/updates/ranking-history.js
|
|
14104
|
+
const RANKING_HISTORY = [
|
|
14105
|
+
{
|
|
14106
|
+
id: "XhUDXP7A67iHCD2kmbVu",
|
|
14107
|
+
begin: "2026-09-24T16:15:00+00:00",
|
|
14108
|
+
end: "2026-10-08T08:00:00+00:00",
|
|
14109
|
+
external_desc: "September 2026 spam update"
|
|
14110
|
+
},
|
|
14111
|
+
{
|
|
14112
|
+
id: "LEubPCm2octf2uMqCFKE",
|
|
14113
|
+
begin: "2026-08-18T16:27:00+00:00",
|
|
14114
|
+
end: "2026-08-21T08:49:00+00:00",
|
|
14115
|
+
external_desc: "August 2026 spam update"
|
|
14116
|
+
},
|
|
14117
|
+
{
|
|
14118
|
+
id: "YUX1peHev5a4fkxLDiUQ",
|
|
14119
|
+
begin: "2026-06-24T16:00:00+00:00",
|
|
14120
|
+
end: "2026-06-26T17:00:00+00:00",
|
|
14121
|
+
external_desc: "June 2026 spam update"
|
|
14122
|
+
},
|
|
14123
|
+
{
|
|
14124
|
+
id: "wdAXJk6LRRihEjpzEeWE",
|
|
14125
|
+
begin: "2026-05-21T15:40:00+00:00",
|
|
14126
|
+
end: "2026-06-02T12:40:00+00:00",
|
|
14127
|
+
external_desc: "May 2026 core update"
|
|
14128
|
+
},
|
|
14129
|
+
{
|
|
14130
|
+
id: "7eTbAa2jWdToLkraZj5y",
|
|
14131
|
+
begin: "2026-03-27T09:00:00+00:00",
|
|
14132
|
+
end: "2026-04-08T13:00:00+00:00",
|
|
14133
|
+
external_desc: "March 2026 core update"
|
|
14134
|
+
},
|
|
14135
|
+
{
|
|
14136
|
+
id: "VbnSXAH4SmEcxPtx4YSD",
|
|
14137
|
+
begin: "2026-03-24T19:00:00+00:00",
|
|
14138
|
+
end: "2026-03-25T14:30:00+00:00",
|
|
14139
|
+
external_desc: "March 2026 spam update"
|
|
14140
|
+
},
|
|
14141
|
+
{
|
|
14142
|
+
id: "mYbNTqV1ytDc2fA8hUz4",
|
|
14143
|
+
begin: "2026-02-05T17:00:00+00:00",
|
|
14144
|
+
end: "2026-02-27T10:00:00+00:00",
|
|
14145
|
+
external_desc: "February 2026 Discover update"
|
|
14146
|
+
},
|
|
14147
|
+
{
|
|
14148
|
+
id: "DsirqJ1gpPRgVQeccPRv",
|
|
14149
|
+
begin: "2025-12-11T17:25:00+00:00",
|
|
14150
|
+
end: "2025-12-29T19:00:00+00:00",
|
|
14151
|
+
external_desc: "December 2025 core update"
|
|
14152
|
+
},
|
|
14153
|
+
{
|
|
14154
|
+
id: "a7Aainy6E9rZsmfq82xt",
|
|
14155
|
+
begin: "2025-08-26T16:00:00+00:00",
|
|
14156
|
+
end: "2025-09-22T07:00:00+00:00",
|
|
14157
|
+
external_desc: "August 2025 spam update"
|
|
14158
|
+
},
|
|
14159
|
+
{
|
|
14160
|
+
id: "riq1AuqETW46NfBCe5NT",
|
|
14161
|
+
begin: "2025-06-30T14:30:00+00:00",
|
|
14162
|
+
end: "2025-07-17T08:00:00+00:00",
|
|
14163
|
+
external_desc: "June 2025 core update"
|
|
14164
|
+
},
|
|
14165
|
+
{
|
|
14166
|
+
id: "zpmwuSwifjDjfrVdaZUx",
|
|
14167
|
+
begin: "2025-03-13T16:00:00+00:00",
|
|
14168
|
+
end: "2025-03-27T12:35:00+00:00",
|
|
14169
|
+
external_desc: "March 2025 core update"
|
|
14170
|
+
},
|
|
14171
|
+
{
|
|
14172
|
+
id: "UUq2WSouY7PhSm8zvtD1",
|
|
14173
|
+
begin: "2024-12-19T17:00:00+00:00",
|
|
14174
|
+
end: "2024-12-26T19:00:00+00:00",
|
|
14175
|
+
external_desc: "December 2024 spam update"
|
|
14176
|
+
},
|
|
14177
|
+
{
|
|
14178
|
+
id: "V9nDKuo6nWKh2ThBALgA",
|
|
14179
|
+
begin: "2024-12-12T15:30:00+00:00",
|
|
14180
|
+
end: "2024-12-18T19:00:00+00:00",
|
|
14181
|
+
external_desc: "December 2024 core update"
|
|
14182
|
+
},
|
|
14183
|
+
{
|
|
14184
|
+
id: "G7rp11wqTaTGn6JjiPF3",
|
|
14185
|
+
begin: "2024-11-11T19:00:00+00:00",
|
|
14186
|
+
end: "2024-12-05T08:00:00+00:00",
|
|
14187
|
+
external_desc: "November 2024 core update"
|
|
14188
|
+
},
|
|
14189
|
+
{
|
|
14190
|
+
id: "42XCLGtpuCZkzqe4dc6S",
|
|
14191
|
+
begin: "2024-08-16T00:15:00+00:00",
|
|
14192
|
+
end: "2024-08-20T11:30:00+00:00",
|
|
14193
|
+
external_desc: "Ranking is experiencing an ongoing issue."
|
|
14194
|
+
},
|
|
14195
|
+
{
|
|
14196
|
+
id: "gVx6b2o78zke7GrMidGy",
|
|
14197
|
+
begin: "2024-08-15T14:58:00+00:00",
|
|
14198
|
+
end: "2024-09-03T19:00:00+00:00",
|
|
14199
|
+
external_desc: "August 2024 core update"
|
|
14200
|
+
},
|
|
14201
|
+
{
|
|
14202
|
+
id: "QdUeCQx3LRVbzQ3E7FiD",
|
|
14203
|
+
begin: "2024-06-20T15:12:00+00:00",
|
|
14204
|
+
end: "2024-06-27T16:11:00+00:00",
|
|
14205
|
+
external_desc: "June 2024 spam update"
|
|
14206
|
+
},
|
|
14207
|
+
{
|
|
14208
|
+
id: "iXz2PJfodvyjaVUeqxZE",
|
|
14209
|
+
begin: "2024-03-05T17:03:00+00:00",
|
|
14210
|
+
end: "2024-03-20T14:09:00+00:00",
|
|
14211
|
+
external_desc: "March 2024 spam update"
|
|
14212
|
+
},
|
|
14213
|
+
{
|
|
14214
|
+
id: "1jW2F89qC2NxJBWGiKxE",
|
|
14215
|
+
begin: "2024-03-05T17:02:00+00:00",
|
|
14216
|
+
end: "2024-04-19T17:00:00+00:00",
|
|
14217
|
+
external_desc: "March 2024 core update"
|
|
14218
|
+
},
|
|
14219
|
+
{
|
|
14220
|
+
id: "VqRTcmyQCwfpuYeaVNfn",
|
|
14221
|
+
begin: "2023-11-08T16:58:00+00:00",
|
|
14222
|
+
end: "2023-12-07T17:00:00+00:00",
|
|
14223
|
+
external_desc: "November 2023 reviews update"
|
|
14224
|
+
},
|
|
14225
|
+
{
|
|
14226
|
+
id: "WtY1CQgfqrr2cjxKbyUk",
|
|
14227
|
+
begin: "2023-11-02T19:09:00+00:00",
|
|
14228
|
+
end: "2023-11-28T16:30:00+00:00",
|
|
14229
|
+
external_desc: "November 2023 core update"
|
|
14230
|
+
},
|
|
14231
|
+
{
|
|
14232
|
+
id: "VKyoS53ULWzSp8BoDxqk",
|
|
14233
|
+
begin: "2023-10-05T16:00:00+00:00",
|
|
14234
|
+
end: "2023-10-19T15:00:00+00:00",
|
|
14235
|
+
external_desc: "October 2023 core update"
|
|
14236
|
+
},
|
|
14237
|
+
{
|
|
14238
|
+
id: "fCUAy6TvbDMkkLAGcwkj",
|
|
14239
|
+
begin: "2023-10-05T07:00:00+00:00",
|
|
14240
|
+
end: "2023-10-31T07:00:00+00:00",
|
|
14241
|
+
external_desc: "Ranking is experiencing an ongoing issue"
|
|
14242
|
+
},
|
|
14243
|
+
{
|
|
14244
|
+
id: "NzcEhGMDhbQEdXCS35xL",
|
|
14245
|
+
begin: "2023-10-04T16:51:00+00:00",
|
|
14246
|
+
end: "2023-10-20T05:15:00+00:00",
|
|
14247
|
+
external_desc: "October 2023 spam update"
|
|
14248
|
+
},
|
|
14249
|
+
{
|
|
14250
|
+
id: "53diuQvcEsgzqXTPBb8p",
|
|
14251
|
+
begin: "2023-09-14T20:19:00+00:00",
|
|
14252
|
+
end: "2023-09-28T07:07:00+00:00",
|
|
14253
|
+
external_desc: "September 2023 helpful content update"
|
|
14254
|
+
},
|
|
14255
|
+
{
|
|
14256
|
+
id: "nBtYtBeex4GYBbdDS2LX",
|
|
14257
|
+
begin: "2023-08-22T17:30:00+00:00",
|
|
14258
|
+
end: "2023-09-07T20:03:00+00:00",
|
|
14259
|
+
external_desc: "August 2023 core update"
|
|
14260
|
+
},
|
|
14261
|
+
{
|
|
14262
|
+
id: "5XRfC46rorevFt8yN8iR",
|
|
14263
|
+
begin: "2023-04-12T16:00:00+00:00",
|
|
14264
|
+
end: "2023-04-25T18:24:00+00:00",
|
|
14265
|
+
external_desc: "April 2023 reviews update"
|
|
14266
|
+
},
|
|
14267
|
+
{
|
|
14268
|
+
id: "Cou8Tr74r7EXNthuEsaG",
|
|
14269
|
+
begin: "2023-03-15T07:00:00+00:00",
|
|
14270
|
+
end: "2023-03-28T14:26:00+00:00",
|
|
14271
|
+
external_desc: "March 2023 core update"
|
|
14272
|
+
},
|
|
14273
|
+
{
|
|
14274
|
+
id: "NDjZ11MEUJb5i62hy5aN",
|
|
14275
|
+
begin: "2023-02-21T08:00:00+00:00",
|
|
14276
|
+
end: "2023-03-07T08:00:00+00:00",
|
|
14277
|
+
external_desc: "February 2023 product reviews update"
|
|
14278
|
+
},
|
|
14279
|
+
{
|
|
14280
|
+
id: "CxXUEK28vCKqkNocPhKR",
|
|
14281
|
+
begin: "2022-12-14T08:00:00+00:00",
|
|
14282
|
+
end: "2023-01-12T08:00:00+00:00",
|
|
14283
|
+
external_desc: "December 2022 link spam update"
|
|
14284
|
+
},
|
|
14285
|
+
{
|
|
14286
|
+
id: "XMdX5PmbMvKW4fv3CRvn",
|
|
14287
|
+
begin: "2022-12-05T08:00:00+00:00",
|
|
14288
|
+
end: "2023-01-12T08:00:00+00:00",
|
|
14289
|
+
external_desc: "December 2022 helpful content update"
|
|
14290
|
+
},
|
|
14291
|
+
{
|
|
14292
|
+
id: "SAeYuxVSo39HHotQNXHr",
|
|
14293
|
+
begin: "2022-10-19T07:00:00+00:00",
|
|
14294
|
+
end: "2022-10-21T07:00:00+00:00",
|
|
14295
|
+
external_desc: "October 2022 spam update"
|
|
14296
|
+
},
|
|
14297
|
+
{
|
|
14298
|
+
id: "ZYwuBXDhkwaoop2V5epM",
|
|
14299
|
+
begin: "2022-09-20T07:00:00+00:00",
|
|
14300
|
+
end: "2022-09-26T07:00:00+00:00",
|
|
14301
|
+
external_desc: "September 2022 product reviews update"
|
|
14302
|
+
},
|
|
14303
|
+
{
|
|
14304
|
+
id: "Y2Cg5k2xLJvfNqR58kEt",
|
|
14305
|
+
begin: "2022-09-12T07:00:00+00:00",
|
|
14306
|
+
end: "2022-09-26T07:00:00+00:00",
|
|
14307
|
+
external_desc: "September 2022 core update"
|
|
14308
|
+
},
|
|
14309
|
+
{
|
|
14310
|
+
id: "sbKc4YSptrSxcUZFDKwU",
|
|
14311
|
+
begin: "2022-08-25T07:00:00+00:00",
|
|
14312
|
+
end: "2022-09-09T07:00:00+00:00",
|
|
14313
|
+
external_desc: "August 2022 helpful content update"
|
|
14314
|
+
},
|
|
14315
|
+
{
|
|
14316
|
+
id: "Xiid524od7fpCak86sNS",
|
|
14317
|
+
begin: "2022-07-27T07:00:00+00:00",
|
|
14318
|
+
end: "2022-08-02T07:00:00+00:00",
|
|
14319
|
+
external_desc: "July 2022 product reviews update"
|
|
14320
|
+
},
|
|
14321
|
+
{
|
|
14322
|
+
id: "GHwLLtDS64a7nDng8xhD",
|
|
14323
|
+
begin: "2022-05-25T07:00:00+00:00",
|
|
14324
|
+
end: "2022-06-09T07:00:00+00:00",
|
|
14325
|
+
external_desc: "May 2022 core update"
|
|
14326
|
+
},
|
|
14327
|
+
{
|
|
14328
|
+
id: "iYWoNu4QXRsQUrzxrbLv",
|
|
14329
|
+
begin: "2022-03-23T07:00:00+00:00",
|
|
14330
|
+
end: "2022-04-06T07:00:00+00:00",
|
|
14331
|
+
external_desc: "March 2022 product reviews update"
|
|
14332
|
+
},
|
|
14333
|
+
{
|
|
14334
|
+
id: "zDCQR5mxLU3ZrQeQvwqN",
|
|
14335
|
+
begin: "2022-02-22T08:00:00+00:00",
|
|
14336
|
+
end: "2022-03-03T08:00:00+00:00",
|
|
14337
|
+
external_desc: "Page experience update for desktop"
|
|
14338
|
+
},
|
|
14339
|
+
{
|
|
14340
|
+
id: "hgwP7ei5H1uKbYSeZrBZ",
|
|
14341
|
+
begin: "2021-12-01T08:00:00+00:00",
|
|
14342
|
+
end: "2021-12-21T08:00:00+00:00",
|
|
14343
|
+
external_desc: "December 2021 product reviews update"
|
|
14344
|
+
},
|
|
14345
|
+
{
|
|
14346
|
+
id: "FncPfR7zRSZ3diemah4o",
|
|
14347
|
+
begin: "2021-11-17T08:00:00+00:00",
|
|
14348
|
+
end: "2021-11-30T08:00:00+00:00",
|
|
14349
|
+
external_desc: "November 2021 core update"
|
|
14350
|
+
},
|
|
14351
|
+
{
|
|
14352
|
+
id: "AFdXceQVsh1GEfKFvHyD",
|
|
14353
|
+
begin: "2021-11-03T07:00:00+00:00",
|
|
14354
|
+
end: "2021-11-11T08:00:00+00:00",
|
|
14355
|
+
external_desc: "November 2021 spam update"
|
|
14356
|
+
}
|
|
14357
|
+
];
|
|
14358
|
+
//#endregion
|
|
14103
14359
|
//#region packages/core/dist/updates/search-updates.js
|
|
14104
14360
|
function classifyUpdate(name, product) {
|
|
14105
14361
|
const lower = `${name} ${product}`.toLowerCase();
|
|
@@ -14127,6 +14383,18 @@ function searchUpdatesFromIncidents(incidents, input = {}) {
|
|
|
14127
14383
|
};
|
|
14128
14384
|
}).slice(0, input.limit ?? 25);
|
|
14129
14385
|
}
|
|
14386
|
+
function withRankingHistory(live) {
|
|
14387
|
+
const byId = /* @__PURE__ */ new Map();
|
|
14388
|
+
for (const saved of RANKING_HISTORY) byId.set(saved.id, {
|
|
14389
|
+
...saved,
|
|
14390
|
+
service_name: "Ranking",
|
|
14391
|
+
status_impact: "SERVICE_INFORMATION",
|
|
14392
|
+
severity: "low",
|
|
14393
|
+
uri: `incidents/${saved.id}`
|
|
14394
|
+
});
|
|
14395
|
+
for (const incident of live) byId.set(incident.id, incident);
|
|
14396
|
+
return [...byId.values()].sort((left, right) => Date.parse(right.begin) - Date.parse(left.begin) || (left.id < right.id ? -1 : 1));
|
|
14397
|
+
}
|
|
14130
14398
|
function findOverlappingSearchUpdates(input) {
|
|
14131
14399
|
const paddingMs = (input.paddingDays ?? 3) * 864e5;
|
|
14132
14400
|
const windowStart = new Date(input.startDate).getTime() - paddingMs;
|
|
@@ -14225,9 +14493,9 @@ async function googleUpdates(window) {
|
|
|
14225
14493
|
const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? readJson(response, "Google Search Status", { maxBytes: MAX_STATUS_BYTES }) : null).catch(() => null);
|
|
14226
14494
|
if (!Array.isArray(incidents)) return "Google Search Status could not be read.";
|
|
14227
14495
|
return findOverlappingSearchUpdates({
|
|
14228
|
-
updates: searchUpdatesFromIncidents(incidents, {
|
|
14496
|
+
updates: searchUpdatesFromIncidents(withRankingHistory(incidents), {
|
|
14229
14497
|
product: "Ranking",
|
|
14230
|
-
limit:
|
|
14498
|
+
limit: Number.POSITIVE_INFINITY
|
|
14231
14499
|
}),
|
|
14232
14500
|
startDate: window.start,
|
|
14233
14501
|
endDate: window.end,
|
|
@@ -14332,7 +14600,7 @@ const METRIC_COLUMNS = [
|
|
|
14332
14600
|
"score",
|
|
14333
14601
|
"priority.score"
|
|
14334
14602
|
];
|
|
14335
|
-
const SUMMARY_ROWS = 10;
|
|
14603
|
+
const SUMMARY_ROWS$1 = 10;
|
|
14336
14604
|
const CELL_MAX = 80;
|
|
14337
14605
|
const isObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
14338
14606
|
const isPrimitive = (value) => value === null || [
|
|
@@ -14491,7 +14759,7 @@ function presentReport(input) {
|
|
|
14491
14759
|
}
|
|
14492
14760
|
};
|
|
14493
14761
|
}
|
|
14494
|
-
const shown = flat.slice(0, Math.min(input.limit, SUMMARY_ROWS));
|
|
14762
|
+
const shown = flat.slice(0, Math.min(input.limit, SUMMARY_ROWS$1));
|
|
14495
14763
|
const columns = columnsFor(flat, 8);
|
|
14496
14764
|
return {
|
|
14497
14765
|
text: [
|
|
@@ -16849,11 +17117,11 @@ function analyzeAiReferralRows(input) {
|
|
|
16849
17117
|
}
|
|
16850
17118
|
//#endregion
|
|
16851
17119
|
//#region packages/core/dist/analyze/ai-referrals-query.js
|
|
16852
|
-
const PAGE_SIZE$
|
|
17120
|
+
const PAGE_SIZE$3 = 25e3;
|
|
16853
17121
|
async function fetchAiReferralRows(input) {
|
|
16854
|
-
const pageSize = input.pageSize ?? PAGE_SIZE$
|
|
17122
|
+
const pageSize = input.pageSize ?? PAGE_SIZE$3;
|
|
16855
17123
|
if (!Number.isInteger(input.maxRows) || input.maxRows < 1) throw new Error("maxRows must be a positive whole number.");
|
|
16856
|
-
if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > PAGE_SIZE$
|
|
17124
|
+
if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > PAGE_SIZE$3) throw new Error(`pageSize must be a whole number between 1 and ${PAGE_SIZE$3}.`);
|
|
16857
17125
|
const rows = [];
|
|
16858
17126
|
const warnings = /* @__PURE__ */ new Set();
|
|
16859
17127
|
const retainedKeys = /* @__PURE__ */ new Set();
|
|
@@ -18992,6 +19260,12 @@ function createRobotsSession(input = {}) {
|
|
|
18992
19260
|
};
|
|
18993
19261
|
return {
|
|
18994
19262
|
resolve: async (origin, targetUrl) => resultFromSource(await sourceFor(origin), targetUrl),
|
|
19263
|
+
crawlDelay: async (origin) => {
|
|
19264
|
+
const source = await sourceFor(origin);
|
|
19265
|
+
if (source.status === void 0 || source.status >= 300 || !source.text) return;
|
|
19266
|
+
const delay = parseRobots$1(source.robotsUrl, source.text).getCrawlDelay(SEO_CRAWLER_USER_AGENT);
|
|
19267
|
+
return delay !== void 0 && Number.isFinite(delay) && delay > 0 ? delay : void 0;
|
|
19268
|
+
},
|
|
18995
19269
|
sitemapUrls: async (origin) => {
|
|
18996
19270
|
const source = await sourceFor(origin);
|
|
18997
19271
|
if (source.status === void 0 || source.status >= 300 || !source.text) return [];
|
|
@@ -19952,7 +20226,7 @@ function finalizeUrlInspectionQuota(input) {
|
|
|
19952
20226
|
function credentialKey(input) {
|
|
19953
20227
|
return createHash("sha256").update(`${input.clientId}\n${input.accountEmail.toLowerCase()}`).digest("base64url");
|
|
19954
20228
|
}
|
|
19955
|
-
function retryAfter(response, now) {
|
|
20229
|
+
function retryAfter$1(response, now) {
|
|
19956
20230
|
const value = response.headers.get("retry-after");
|
|
19957
20231
|
if (!value) return void 0;
|
|
19958
20232
|
const seconds = Number(value);
|
|
@@ -19995,7 +20269,7 @@ async function inspectUrl(input) {
|
|
|
19995
20269
|
}
|
|
19996
20270
|
if (response.status === 429) {
|
|
19997
20271
|
const now = /* @__PURE__ */ new Date();
|
|
19998
|
-
const blockedUntil = retryAfter(response, now);
|
|
20272
|
+
const blockedUntil = retryAfter$1(response, now);
|
|
19999
20273
|
finalizeUrlInspectionQuota({
|
|
20000
20274
|
reservation,
|
|
20001
20275
|
outcome: "exhausted",
|
|
@@ -20366,7 +20640,7 @@ function estimateSerpTitleWidth(title) {
|
|
|
20366
20640
|
}
|
|
20367
20641
|
//#endregion
|
|
20368
20642
|
//#region packages/core/dist/analyze/audit-page.js
|
|
20369
|
-
const defaultDependencies$
|
|
20643
|
+
const defaultDependencies$12 = {
|
|
20370
20644
|
fetchPage,
|
|
20371
20645
|
queryPageMetrics,
|
|
20372
20646
|
now: () => /* @__PURE__ */ new Date()
|
|
@@ -20397,7 +20671,7 @@ function buildRecommendations(report) {
|
|
|
20397
20671
|
});
|
|
20398
20672
|
return recommendations;
|
|
20399
20673
|
}
|
|
20400
|
-
async function auditPage(input, dependencies = defaultDependencies$
|
|
20674
|
+
async function auditPage(input, dependencies = defaultDependencies$12) {
|
|
20401
20675
|
const fetched = await dependencies.fetchPage(input.url, {
|
|
20402
20676
|
js: input.js,
|
|
20403
20677
|
refresh: input.refresh
|
|
@@ -20516,7 +20790,7 @@ async function auditPage(input, dependencies = defaultDependencies$13) {
|
|
|
20516
20790
|
}
|
|
20517
20791
|
//#endregion
|
|
20518
20792
|
//#region packages/core/dist/analyze/content-coverage-analysis.js
|
|
20519
|
-
const STOPWORDS
|
|
20793
|
+
const STOPWORDS = /* @__PURE__ */ new Set([
|
|
20520
20794
|
"a",
|
|
20521
20795
|
"an",
|
|
20522
20796
|
"and",
|
|
@@ -20545,7 +20819,7 @@ function hasNonAscii(value) {
|
|
|
20545
20819
|
}
|
|
20546
20820
|
function queryTerms$1(query) {
|
|
20547
20821
|
const tokens = normalizeForCoverage(query).split(" ").filter((token) => token.length > 1 || hasNonAscii(token));
|
|
20548
|
-
const meaningful = tokens.filter((token) => (token.length > 2 || hasNonAscii(token)) && !STOPWORDS
|
|
20822
|
+
const meaningful = tokens.filter((token) => (token.length > 2 || hasNonAscii(token)) && !STOPWORDS.has(token));
|
|
20549
20823
|
return [...new Set(meaningful.length ? meaningful : tokens)];
|
|
20550
20824
|
}
|
|
20551
20825
|
function countPhrase$1(query, text) {
|
|
@@ -21809,7 +22083,7 @@ const actionPriority = new Map([
|
|
|
21809
22083
|
"unverified",
|
|
21810
22084
|
"covered"
|
|
21811
22085
|
].map((type, index) => [type, index]));
|
|
21812
|
-
function plural$
|
|
22086
|
+
function plural$6(count, singular, pluralLabel) {
|
|
21813
22087
|
return count === 1 ? singular : pluralLabel;
|
|
21814
22088
|
}
|
|
21815
22089
|
function actionable(item) {
|
|
@@ -21843,7 +22117,7 @@ function pageOpportunityVerdict(input) {
|
|
|
21843
22117
|
if (focus === "unverified") return "GSC visibility exists, but the page was not verified and no specific change is justified yet.";
|
|
21844
22118
|
if (focus === "covered") return "The verified page covers the sampled queries and CTR is not materially below its directional benchmark.";
|
|
21845
22119
|
const count = analysis.summary.opportunities;
|
|
21846
|
-
const prefix = `${count} ${plural$
|
|
22120
|
+
const prefix = `${count} ${plural$6(count, "actionable observation", "actionable observations")} found.`;
|
|
21847
22121
|
if (focus === "content-gap") return `${prefix} Verified body coverage is the main issue; confirm intent before expanding the page.`;
|
|
21848
22122
|
if (focus === "serp-framing") return `${prefix} The body is relevant, but its title, meta description, or H1 framing is weak.`;
|
|
21849
22123
|
if (focus === "ranking") return `${prefix} Most upside requires stronger relevance or authority; no CTR-only lift is claimed for page-two queries.`;
|
|
@@ -21858,7 +22132,7 @@ function pageOpportunityRecommendations(analysis) {
|
|
|
21858
22132
|
}
|
|
21859
22133
|
const recommendations = [`Start with "${top.query}". ${top.recommendation}`];
|
|
21860
22134
|
const technical = actions.filter((item) => item.opportunityType === "technical-check").length;
|
|
21861
|
-
if (technical) recommendations.push(`Resolve the technical evidence affecting ${technical} ${plural$
|
|
22135
|
+
if (technical) recommendations.push(`Resolve the technical evidence affecting ${technical} ${plural$6(technical, "row", "rows")} before content work.`);
|
|
21862
22136
|
const gaps = actions.filter((item) => item.opportunityType === "content-gap").length;
|
|
21863
22137
|
if (gaps >= 3) recommendations.push(`Group the ${gaps} verified content gaps into useful sections rather than adding one thin paragraph per query.`);
|
|
21864
22138
|
return recommendations;
|
|
@@ -21866,13 +22140,13 @@ function pageOpportunityRecommendations(analysis) {
|
|
|
21866
22140
|
//#endregion
|
|
21867
22141
|
//#region packages/core/dist/analyze/page-opportunities.js
|
|
21868
22142
|
const MAX_BENCHMARK_ROWS = 1e5;
|
|
21869
|
-
const defaultDependencies$
|
|
22143
|
+
const defaultDependencies$11 = {
|
|
21870
22144
|
searchAnalytics: querySearchAnalytics,
|
|
21871
22145
|
fetch: fetchPage,
|
|
21872
22146
|
extract: extractPage,
|
|
21873
22147
|
now: () => /* @__PURE__ */ new Date()
|
|
21874
22148
|
};
|
|
21875
|
-
function errorMessage
|
|
22149
|
+
function errorMessage(error) {
|
|
21876
22150
|
return error instanceof Error ? error.message : String(error);
|
|
21877
22151
|
}
|
|
21878
22152
|
function unique$3(values) {
|
|
@@ -21893,7 +22167,7 @@ async function verifyPage(input) {
|
|
|
21893
22167
|
});
|
|
21894
22168
|
warnings.push(...fetched.warnings);
|
|
21895
22169
|
} catch (error) {
|
|
21896
|
-
const reason = `Page fetch failed: ${errorMessage
|
|
22170
|
+
const reason = `Page fetch failed: ${errorMessage(error)}`;
|
|
21897
22171
|
return {
|
|
21898
22172
|
verification: {
|
|
21899
22173
|
status: "failed",
|
|
@@ -21915,7 +22189,7 @@ async function verifyPage(input) {
|
|
|
21915
22189
|
warnings: unique$3(warnings)
|
|
21916
22190
|
};
|
|
21917
22191
|
} catch (error) {
|
|
21918
|
-
const reason = `Page extraction failed: ${errorMessage
|
|
22192
|
+
const reason = `Page extraction failed: ${errorMessage(error)}`;
|
|
21919
22193
|
return {
|
|
21920
22194
|
fetched,
|
|
21921
22195
|
verification: {
|
|
@@ -21938,7 +22212,7 @@ function reportCaveats$1(input) {
|
|
|
21938
22212
|
input.benchmarkTruncated ? `The site-wide benchmark reached its ${MAX_BENCHMARK_ROWS.toLocaleString("en-US")}-row cap, so provenance is complete but the peer sample is truncated.` : ""
|
|
21939
22213
|
].filter(Boolean);
|
|
21940
22214
|
}
|
|
21941
|
-
async function pageOpportunitiesReport(input, dependencies = defaultDependencies$
|
|
22215
|
+
async function pageOpportunitiesReport(input, dependencies = defaultDependencies$11) {
|
|
21942
22216
|
const url = assertUrlMatchesGscProperty(input.site, input.url);
|
|
21943
22217
|
const days = reportDays$1(input.days);
|
|
21944
22218
|
const range = defaultDateRange(days);
|
|
@@ -26980,7 +27254,7 @@ async function loadInstalledProviderExtensions(options = {}) {
|
|
|
26980
27254
|
//#endregion
|
|
26981
27255
|
//#region packages/core/dist/provider-extensions/runtime.js
|
|
26982
27256
|
const DEFAULT_MAX_REQUESTS = 8;
|
|
26983
|
-
const MAX_RESPONSE_BYTES = 10 * 1024 * 1024;
|
|
27257
|
+
const MAX_RESPONSE_BYTES$1 = 10 * 1024 * 1024;
|
|
26984
27258
|
const MAX_REQUEST_BODY_BYTES = 64 * 1024;
|
|
26985
27259
|
const TIMEOUT_MS$1 = 15e3;
|
|
26986
27260
|
function normalizeProviderExtensionAccount(provider, account) {
|
|
@@ -27028,7 +27302,7 @@ function runtimeForProvider(provider, options = {}) {
|
|
|
27028
27302
|
...input.body ? { body: input.body } : {}
|
|
27029
27303
|
},
|
|
27030
27304
|
fetch: providerFetch,
|
|
27031
|
-
maxResponseBytes: MAX_RESPONSE_BYTES,
|
|
27305
|
+
maxResponseBytes: MAX_RESPONSE_BYTES$1,
|
|
27032
27306
|
timeoutMs: TIMEOUT_MS$1,
|
|
27033
27307
|
retry: (input.method ?? "GET") === "GET" ? "safe" : "never",
|
|
27034
27308
|
schema: z.unknown()
|
|
@@ -27117,7 +27391,7 @@ async function runAnalyticsProviderLandingPages(input) {
|
|
|
27117
27391
|
}
|
|
27118
27392
|
//#endregion
|
|
27119
27393
|
//#region packages/core/dist/analyze/workflows/analytics-value.js
|
|
27120
|
-
const defaultDependencies$
|
|
27394
|
+
const defaultDependencies$10 = {
|
|
27121
27395
|
runGa4Report,
|
|
27122
27396
|
analyticsLandingPages: runAnalyticsProviderLandingPages
|
|
27123
27397
|
};
|
|
@@ -27188,7 +27462,7 @@ function landingPageRankingPolicy(input) {
|
|
|
27188
27462
|
warnings
|
|
27189
27463
|
};
|
|
27190
27464
|
}
|
|
27191
|
-
async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
27465
|
+
async function fetchLandingPageValues(input, dependencies = defaultDependencies$10) {
|
|
27192
27466
|
const retainedRowLimit = input.limit ?? 5e3;
|
|
27193
27467
|
const connection = input.connection ?? (input.propertyId ? {
|
|
27194
27468
|
provider: "google",
|
|
@@ -27207,7 +27481,7 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
27207
27481
|
}
|
|
27208
27482
|
};
|
|
27209
27483
|
if (connection.provider === "extension") try {
|
|
27210
|
-
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$
|
|
27484
|
+
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$10.analyticsLandingPages)({
|
|
27211
27485
|
providerId: connection.providerId,
|
|
27212
27486
|
account: connection.account,
|
|
27213
27487
|
startDate: input.startDate,
|
|
@@ -27247,7 +27521,7 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
27247
27521
|
clicky: async (selected) => {
|
|
27248
27522
|
if (selected.provider !== "clicky") throw new Error("Clicky analytics connection mismatch.");
|
|
27249
27523
|
try {
|
|
27250
|
-
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$
|
|
27524
|
+
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$10.analyticsLandingPages)({
|
|
27251
27525
|
providerId: "clicky",
|
|
27252
27526
|
account: { siteId: selected.siteId },
|
|
27253
27527
|
startDate: input.startDate,
|
|
@@ -27286,7 +27560,7 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
27286
27560
|
google: async (selected) => {
|
|
27287
27561
|
if (selected.provider !== "google") throw new Error("Google Analytics connection mismatch.");
|
|
27288
27562
|
try {
|
|
27289
|
-
const result = await (dependencies.runGa4Report ?? defaultDependencies$
|
|
27563
|
+
const result = await (dependencies.runGa4Report ?? defaultDependencies$10.runGa4Report)(selected.propertyId, {
|
|
27290
27564
|
dateRanges: [{
|
|
27291
27565
|
startDate: input.startDate,
|
|
27292
27566
|
endDate: input.endDate
|
|
@@ -28067,7 +28341,7 @@ const NAVIGATIONAL_STEM_THRESHOLDS = {
|
|
|
28067
28341
|
function compact(value) {
|
|
28068
28342
|
return normalizeBrandText(value).replace(/\s+/g, "");
|
|
28069
28343
|
}
|
|
28070
|
-
function round$
|
|
28344
|
+
function round$2(value, digits) {
|
|
28071
28345
|
return Number(value.toFixed(digits));
|
|
28072
28346
|
}
|
|
28073
28347
|
function stemQueryEvidence(rows, stem) {
|
|
@@ -28094,8 +28368,8 @@ function stemQueryEvidence(rows, stem) {
|
|
|
28094
28368
|
query,
|
|
28095
28369
|
clicks,
|
|
28096
28370
|
impressions,
|
|
28097
|
-
ctr: round$
|
|
28098
|
-
position: round$
|
|
28371
|
+
ctr: round$2(clicks / impressions, 4),
|
|
28372
|
+
position: round$2(weightedPosition / impressions, 2),
|
|
28099
28373
|
sourceRows
|
|
28100
28374
|
};
|
|
28101
28375
|
}
|
|
@@ -28590,9 +28864,9 @@ function analyzeCtrUnderperformersFromRows(input) {
|
|
|
28590
28864
|
//#endregion
|
|
28591
28865
|
//#region packages/core/dist/analyze/ctr-underperformers-report.js
|
|
28592
28866
|
const MAX_GSC_ROWS$3 = 1e5;
|
|
28593
|
-
const DEFAULT_DAYS$
|
|
28867
|
+
const DEFAULT_DAYS$3 = 28;
|
|
28594
28868
|
const MAX_DAYS$2 = 548;
|
|
28595
|
-
function plural$
|
|
28869
|
+
function plural$5(count, singular, pluralLabel = `${singular}s`) {
|
|
28596
28870
|
return count === 1 ? singular : pluralLabel;
|
|
28597
28871
|
}
|
|
28598
28872
|
async function ctrUnderperformersReportWith(input, dependencies) {
|
|
@@ -28613,7 +28887,7 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
28613
28887
|
const explicitRange = explicitDateRange(input, MAX_DAYS$2);
|
|
28614
28888
|
const days = explicitRange?.days ?? integerOption$1({
|
|
28615
28889
|
value: input.days,
|
|
28616
|
-
fallback: DEFAULT_DAYS$
|
|
28890
|
+
fallback: DEFAULT_DAYS$3,
|
|
28617
28891
|
minimum: 1,
|
|
28618
28892
|
maximum: MAX_DAYS$2,
|
|
28619
28893
|
label: "Days"
|
|
@@ -28695,11 +28969,11 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
28695
28969
|
minImpressions,
|
|
28696
28970
|
limit,
|
|
28697
28971
|
brandFiltering: input.includeBrand ? "included" : "excluded",
|
|
28698
|
-
verdict: top ? `${selection.eligibleUnderperformers} CTR ${plural$
|
|
28972
|
+
verdict: top ? `${selection.eligibleUnderperformers} CTR ${plural$5(selection.eligibleUnderperformers, "underperformer")} found in retained rows, with a calculated heuristic shortfall of ${totalClickShortfall.toFixed(0)} clicks for this window. Start review with "${top.query}" because it has the largest gap.` : dataStatus === "partial" ? "No material CTR underperformers remained in the validated retained rows, but partial evidence prevents an all-clear." : `No high-impression rows at average position below 4 in the retained rows are materially below the heuristic CTR benchmark.`
|
|
28699
28973
|
},
|
|
28700
28974
|
items,
|
|
28701
28975
|
caveats: [
|
|
28702
|
-
`Date window: ${range.startDate} to ${range.endDate} (${days} ${plural$
|
|
28976
|
+
`Date window: ${range.startDate} to ${range.endDate} (${days} ${plural$5(days, "day")}), using final GSC data where available.`,
|
|
28703
28977
|
`Brand queries: ${input.includeBrand ? "included" : "excluded"}. Brand terms used: ${brand.terms.join(", ") || "none"} (${brand.source === "provided" ? "provided" : "host forms, plus the bare domain stem only when Search Console shows the site is its navigational answer"}).`,
|
|
28704
28978
|
`Only rows at average position below 4 with at least ${minImpressions} impressions were checked. Rows at position 4 to 10 belong to quick-wins, which uses the same site CTR curve.`,
|
|
28705
28979
|
minImpressionsCaveat({
|
|
@@ -28707,13 +28981,13 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
28707
28981
|
belowMinimumRows: selection.belowMinimumRows
|
|
28708
28982
|
}),
|
|
28709
28983
|
"Expected CTR uses the site CTR curve from all page-one rows (75th percentile of qualified URL CTRs per position) when enough peer data exists, otherwise the fallback position curve.",
|
|
28710
|
-
...selection.fragmentRows ? [`${selection.fragmentRows} source ${plural$
|
|
28711
|
-
...selection.flaggedRows ? [`${selection.flaggedRows} eligible ${plural$
|
|
28984
|
+
...selection.fragmentRows ? [`${selection.fragmentRows} source ${plural$5(selection.fragmentRows, "row")} with a #fragment URL ${selection.fragmentRows === 1 ? "was" : "were"} merged into the base URL.`] : [],
|
|
28985
|
+
...selection.flaggedRows ? [`${selection.flaggedRows} eligible ${plural$5(selection.flaggedRows, "row")} carry heuristic query flags (likely-automated or navigational-other) and rank after unflagged rows.`] : [],
|
|
28712
28986
|
"CTR benchmarks and calculated click shortfalls are directional heuristics, not forecasts or proof that clicks are available. Validate the search intent and live SERP before editing.",
|
|
28713
|
-
...selection.invalidRows ? [`${selection.invalidRows} invalid provider ${plural$
|
|
28714
|
-
...selection.duplicateRows ? [`${selection.duplicateRows} repeated query/page ${plural$
|
|
28987
|
+
...selection.invalidRows ? [`${selection.invalidRows} invalid provider ${plural$5(selection.invalidRows, "row")} ${selection.invalidRows === 1 ? "was" : "were"} excluded, so this report is partial.`] : [],
|
|
28988
|
+
...selection.duplicateRows ? [`${selection.duplicateRows} repeated query/page ${plural$5(selection.duplicateRows, "row")} ${selection.duplicateRows === 1 ? "was" : "were"} aggregated before CTR and average position were calculated.`] : [],
|
|
28715
28989
|
...possiblyTruncated ? [`The retained-row limit of ${MAX_GSC_ROWS$3} was reached, so lower-ranked source rows may be absent.`] : [],
|
|
28716
|
-
...selection.limitedUnderperformers ? [`${selection.limitedUnderperformers} eligible ${plural$
|
|
28990
|
+
...selection.limitedUnderperformers ? [`${selection.limitedUnderperformers} eligible ${plural$5(selection.limitedUnderperformers, "underperformer")} ${selection.limitedUnderperformers === 1 ? "was" : "were"} omitted by the output limit.`] : []
|
|
28717
28991
|
],
|
|
28718
28992
|
recommendations: top ? [
|
|
28719
28993
|
`Review the live SERP and displayed snippet for "${top.query}" first. Test the title or meta description only if the observed framing does not match the query intent.`,
|
|
@@ -28724,20 +28998,20 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
28724
28998
|
}
|
|
28725
28999
|
//#endregion
|
|
28726
29000
|
//#region packages/core/dist/analyze/ctr-underperformers.js
|
|
28727
|
-
const defaultDependencies$
|
|
29001
|
+
const defaultDependencies$9 = {
|
|
28728
29002
|
searchAnalytics: querySearchAnalytics,
|
|
28729
29003
|
now: () => /* @__PURE__ */ new Date()
|
|
28730
29004
|
};
|
|
28731
|
-
function ctrUnderperformersReport(input, dependencies = defaultDependencies$
|
|
29005
|
+
function ctrUnderperformersReport(input, dependencies = defaultDependencies$9) {
|
|
28732
29006
|
return ctrUnderperformersReportWith(input, dependencies);
|
|
28733
29007
|
}
|
|
28734
29008
|
//#endregion
|
|
28735
29009
|
//#region packages/core/dist/phrasing.js
|
|
28736
|
-
function plural$
|
|
29010
|
+
function plural$4(count, singular, pluralLabel = `${singular}s`) {
|
|
28737
29011
|
return count === 1 ? singular : pluralLabel;
|
|
28738
29012
|
}
|
|
28739
29013
|
function countLabel(count, singular, pluralLabel = `${singular}s`) {
|
|
28740
|
-
return `${count.toLocaleString("en-GB")} ${plural$
|
|
29014
|
+
return `${count.toLocaleString("en-GB")} ${plural$4(count, singular, pluralLabel)}`;
|
|
28741
29015
|
}
|
|
28742
29016
|
//#endregion
|
|
28743
29017
|
//#region packages/core/dist/analyze/diagnosis-priorities.js
|
|
@@ -29185,11 +29459,11 @@ function unavailableSegmentImpactReport(input) {
|
|
|
29185
29459
|
//#endregion
|
|
29186
29460
|
//#region packages/core/dist/analyze/segment-impact.js
|
|
29187
29461
|
const DEFAULT_MAX_ROWS = 1e5;
|
|
29188
|
-
const defaultDependencies$
|
|
29462
|
+
const defaultDependencies$8 = {
|
|
29189
29463
|
searchAnalytics: querySearchAnalytics,
|
|
29190
29464
|
now: () => /* @__PURE__ */ new Date()
|
|
29191
29465
|
};
|
|
29192
|
-
async function segmentImpact(input, dependencies = defaultDependencies$
|
|
29466
|
+
async function segmentImpact(input, dependencies = defaultDependencies$8) {
|
|
29193
29467
|
validateSegmentSite(input.site);
|
|
29194
29468
|
const now = dependencies.now();
|
|
29195
29469
|
const ranges = resolveSegmentRanges({
|
|
@@ -31042,11 +31316,11 @@ async function cannibalReportWith(input, dependencies) {
|
|
|
31042
31316
|
}
|
|
31043
31317
|
//#endregion
|
|
31044
31318
|
//#region packages/core/dist/analyze/site-diagnostics/cannibal.js
|
|
31045
|
-
const defaultDependencies$
|
|
31319
|
+
const defaultDependencies$7 = {
|
|
31046
31320
|
searchAnalytics: querySearchAnalytics,
|
|
31047
31321
|
now: () => /* @__PURE__ */ new Date()
|
|
31048
31322
|
};
|
|
31049
|
-
async function cannibalReport(input, dependencies = defaultDependencies$
|
|
31323
|
+
async function cannibalReport(input, dependencies = defaultDependencies$7) {
|
|
31050
31324
|
if (input.researchFiles) {
|
|
31051
31325
|
const conflicts = [
|
|
31052
31326
|
["days", input.days],
|
|
@@ -31504,11 +31778,11 @@ async function decayingReportWith(input, dependencies) {
|
|
|
31504
31778
|
}
|
|
31505
31779
|
//#endregion
|
|
31506
31780
|
//#region packages/core/dist/analyze/site-diagnostics/decaying.js
|
|
31507
|
-
const defaultDependencies$
|
|
31781
|
+
const defaultDependencies$6 = {
|
|
31508
31782
|
searchAnalytics: querySearchAnalytics,
|
|
31509
31783
|
now: () => /* @__PURE__ */ new Date()
|
|
31510
31784
|
};
|
|
31511
|
-
function decayingReport(input, dependencies = defaultDependencies$
|
|
31785
|
+
function decayingReport(input, dependencies = defaultDependencies$6) {
|
|
31512
31786
|
return decayingReportWith(input, dependencies);
|
|
31513
31787
|
}
|
|
31514
31788
|
//#endregion
|
|
@@ -31818,7 +32092,7 @@ function quickWinTemplateSummaries(items) {
|
|
|
31818
32092
|
}
|
|
31819
32093
|
//#endregion
|
|
31820
32094
|
//#region packages/core/dist/analyze/site-diagnostics/quick-wins-report.js
|
|
31821
|
-
const DEFAULT_DAYS$
|
|
32095
|
+
const DEFAULT_DAYS$2 = 28;
|
|
31822
32096
|
const MAX_DAYS$1 = 548;
|
|
31823
32097
|
const DEFAULT_VERIFY_LIMIT$1 = 5;
|
|
31824
32098
|
const MAX_VERIFY_LIMIT = 100;
|
|
@@ -31840,7 +32114,7 @@ async function quickWinsReportWith(input, dependencies) {
|
|
|
31840
32114
|
const explicitRange = explicitDateRange(input, MAX_DAYS$1);
|
|
31841
32115
|
const days = explicitRange?.days ?? integerOption$1({
|
|
31842
32116
|
value: input.days,
|
|
31843
|
-
fallback: DEFAULT_DAYS$
|
|
32117
|
+
fallback: DEFAULT_DAYS$2,
|
|
31844
32118
|
minimum: 1,
|
|
31845
32119
|
maximum: MAX_DAYS$1,
|
|
31846
32120
|
label: "Days"
|
|
@@ -32005,13 +32279,13 @@ async function quickWinsReportWith(input, dependencies) {
|
|
|
32005
32279
|
}
|
|
32006
32280
|
//#endregion
|
|
32007
32281
|
//#region packages/core/dist/analyze/site-diagnostics/quick-wins.js
|
|
32008
|
-
const defaultDependencies$
|
|
32282
|
+
const defaultDependencies$5 = {
|
|
32009
32283
|
searchAnalytics: querySearchAnalytics,
|
|
32010
32284
|
fetch: fetchPage,
|
|
32011
32285
|
extract: extractPage,
|
|
32012
32286
|
now: () => /* @__PURE__ */ new Date()
|
|
32013
32287
|
};
|
|
32014
|
-
function quickWinsReport(input, dependencies = defaultDependencies$
|
|
32288
|
+
function quickWinsReport(input, dependencies = defaultDependencies$5) {
|
|
32015
32289
|
return quickWinsReportWith(input, dependencies);
|
|
32016
32290
|
}
|
|
32017
32291
|
//#endregion
|
|
@@ -32264,7 +32538,7 @@ function analyzeStrikingDistanceRows(input) {
|
|
|
32264
32538
|
}
|
|
32265
32539
|
//#endregion
|
|
32266
32540
|
//#region packages/core/dist/analyze/striking-distance-report.js
|
|
32267
|
-
const DEFAULT_DAYS$
|
|
32541
|
+
const DEFAULT_DAYS$1 = 28;
|
|
32268
32542
|
const MAX_DAYS = 548;
|
|
32269
32543
|
const DEFAULT_VERIFY_LIMIT = 5;
|
|
32270
32544
|
const MAX_SOURCE_ROWS$1 = 1e5;
|
|
@@ -32273,7 +32547,7 @@ function boundedInteger$2(value, fallback, minimum, maximum) {
|
|
|
32273
32547
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
32274
32548
|
}
|
|
32275
32549
|
function reportDays(value) {
|
|
32276
|
-
if (value === void 0) return DEFAULT_DAYS$
|
|
32550
|
+
if (value === void 0) return DEFAULT_DAYS$1;
|
|
32277
32551
|
if (!Number.isInteger(value) || value < 1 || value > MAX_DAYS) throw new SeoError("INVALID_INPUT", `Days must be a whole number between 1 and ${MAX_DAYS}.`);
|
|
32278
32552
|
return value;
|
|
32279
32553
|
}
|
|
@@ -32420,12 +32694,12 @@ async function strikingDistanceWith(input, dependencies) {
|
|
|
32420
32694
|
}
|
|
32421
32695
|
//#endregion
|
|
32422
32696
|
//#region packages/core/dist/analyze/striking-distance.js
|
|
32423
|
-
const defaultDependencies$
|
|
32697
|
+
const defaultDependencies$4 = {
|
|
32424
32698
|
searchAnalytics: querySearchAnalytics,
|
|
32425
32699
|
verifyContent: verifyQueryContent,
|
|
32426
32700
|
now: () => /* @__PURE__ */ new Date()
|
|
32427
32701
|
};
|
|
32428
|
-
function strikingDistance(input, dependencies = defaultDependencies$
|
|
32702
|
+
function strikingDistance(input, dependencies = defaultDependencies$4) {
|
|
32429
32703
|
return strikingDistanceWith(input, dependencies);
|
|
32430
32704
|
}
|
|
32431
32705
|
//#endregion
|
|
@@ -32437,7 +32711,7 @@ async function fetchSearchStatusIncidents() {
|
|
|
32437
32711
|
return readJson(response, "Google Search Status", { maxBytes: 10 * 1024 * 1024 });
|
|
32438
32712
|
}
|
|
32439
32713
|
async function listSearchUpdates(input = {}) {
|
|
32440
|
-
return searchUpdatesFromIncidents(await fetchSearchStatusIncidents(), input);
|
|
32714
|
+
return searchUpdatesFromIncidents(withRankingHistory(await fetchSearchStatusIncidents()), input);
|
|
32441
32715
|
}
|
|
32442
32716
|
//#endregion
|
|
32443
32717
|
//#region packages/core/dist/analyze/traffic-anomaly-analysis.js
|
|
@@ -32496,7 +32770,7 @@ function anomalyEvidence(anomaly) {
|
|
|
32496
32770
|
if (anomaly.direction === "normal") return `${formatMetric(anomaly.metric)} changed ${changePhrase} (${anomaly.baselineMean.toLocaleString("en-GB")} -> ${anomaly.comparisonMean.toLocaleString("en-GB")} per observed day${anomaly.zScore === null ? "; the baseline had no observed variance" : `, z ${anomaly.zScore}`}).`;
|
|
32497
32771
|
return `${formatMetric(anomaly.metric)} ${direction} ${changePhrase} (${anomaly.baselineMean.toLocaleString("en-GB")} -> ${anomaly.comparisonMean.toLocaleString("en-GB")} per observed day${anomaly.zScore === null ? "; outside the flat observed baseline" : `, z ${anomaly.zScore}`}).`;
|
|
32498
32772
|
}
|
|
32499
|
-
function plural$
|
|
32773
|
+
function plural$3(count, singular, pluralLabel = `${singular}s`) {
|
|
32500
32774
|
return count === 1 ? singular : pluralLabel;
|
|
32501
32775
|
}
|
|
32502
32776
|
function strongestSignal(anomalies) {
|
|
@@ -32552,13 +32826,13 @@ function interpretUpdateCorrelation(input) {
|
|
|
32552
32826
|
summary,
|
|
32553
32827
|
evidence: [
|
|
32554
32828
|
...input.anomalies.map(anomalyEvidence),
|
|
32555
|
-
hasUpdate ? `${input.overlappingUpdates.length} official Google Ranking update ${plural$
|
|
32829
|
+
hasUpdate ? `${input.overlappingUpdates.length} official Google Ranking update ${plural$3(input.overlappingUpdates.length, "window")} overlapped the change date or comparison window: ${updateNames.join(", ")}.` : "No official Google Ranking update windows overlapped the change date or comparison window.",
|
|
32556
32830
|
largeMovement ? "Both clicks and impressions moved sharply in the same direction." : "Clicks and impressions did not both meet the sharp-movement evidence threshold.",
|
|
32557
32831
|
hasConfounders ? `Known overlapping site changes: ${confounders.map((confounder) => confounder.title).join("; ")}.` : "No overlapping site changes were supplied."
|
|
32558
32832
|
],
|
|
32559
32833
|
caveats: [
|
|
32560
32834
|
`GSC window: ${input.days} days; recent comparison window: ${input.recentDays} days.`,
|
|
32561
|
-
`Update overlap padding: ${input.paddingDays} ${plural$
|
|
32835
|
+
`Update overlap padding: ${input.paddingDays} ${plural$3(input.paddingDays, "day")} either side of the step-change date, or of the recent comparison window when no step change was found.`,
|
|
32562
32836
|
input.refresh ? "Data freshness: cache bypassed for GSC data." : "Data freshness: local GSC cache may have been used.",
|
|
32563
32837
|
"Source: official Google Search Status Dashboard Ranking incidents only; third-party volatility and unconfirmed chatter are not included.",
|
|
32564
32838
|
"An update-window overlap cannot establish causation, even when movement is large and no known site change was recorded.",
|
|
@@ -32726,7 +33000,7 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
32726
33000
|
if (step && changeDate) {
|
|
32727
33001
|
const previousDate = aggregated.rows[step.index - 1]?.keys[0];
|
|
32728
33002
|
const gap = previousDate ? calendarDays(previousDate, changeDate) - 2 : 0;
|
|
32729
|
-
stepCaveats.push(`${formatMetric(metric)} changed level on ${changeDate}, so its baseline is every observed day before that date and its comparison is every observed day from it.`, ...gap > 0 ? [`The ${metric} change date follows ${gap} missing ${plural$
|
|
33003
|
+
stepCaveats.push(`${formatMetric(metric)} changed level on ${changeDate}, so its baseline is every observed day before that date and its comparison is every observed day from it.`, ...gap > 0 ? [`The ${metric} change date follows ${gap} missing ${plural$3(gap, "day")}; the change may have started on one of them.`] : []);
|
|
32730
33004
|
return compareSegments({
|
|
32731
33005
|
site: input.site,
|
|
32732
33006
|
metric,
|
|
@@ -32757,9 +33031,9 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
32757
33031
|
const comparisonMissingDays = input.recentDays - comparisonRows.length;
|
|
32758
33032
|
const missingDays = expectedDays - aggregated.rows.length;
|
|
32759
33033
|
const caveats = [
|
|
32760
|
-
...missingDays > 0 ? [`${missingDays} requested calendar ${plural$
|
|
32761
|
-
...aggregated.invalidRows > 0 ? [`${aggregated.invalidRows} invalid provider ${plural$
|
|
32762
|
-
...aggregated.duplicateRows > 0 ? [`${aggregated.duplicateRows} duplicate date ${plural$
|
|
33034
|
+
...missingDays > 0 ? [`${missingDays} requested calendar ${plural$3(missingDays, "day")} had no returned date aggregate and ${missingDays === 1 ? "was" : "were"} not filled with zeros.`] : [],
|
|
33035
|
+
...aggregated.invalidRows > 0 ? [`${aggregated.invalidRows} invalid provider ${plural$3(aggregated.invalidRows, "row")} ${aggregated.invalidRows === 1 ? "was" : "were"} excluded.`] : [],
|
|
33036
|
+
...aggregated.duplicateRows > 0 ? [`${aggregated.duplicateRows} duplicate date ${plural$3(aggregated.duplicateRows, "row")} ${aggregated.duplicateRows === 1 ? "was" : "were"} aggregated before analysis.`] : [],
|
|
32763
33037
|
...stepCaveats
|
|
32764
33038
|
];
|
|
32765
33039
|
return {
|
|
@@ -32842,11 +33116,11 @@ async function trafficAnomalyWith(input, dependencies) {
|
|
|
32842
33116
|
}
|
|
32843
33117
|
//#endregion
|
|
32844
33118
|
//#region packages/core/dist/analyze/traffic-anomaly.js
|
|
32845
|
-
const defaultDependencies$
|
|
33119
|
+
const defaultDependencies$3 = {
|
|
32846
33120
|
searchAnalytics: querySearchAnalytics,
|
|
32847
33121
|
now: () => /* @__PURE__ */ new Date()
|
|
32848
33122
|
};
|
|
32849
|
-
function trafficAnomaly(input, dependencies = defaultDependencies$
|
|
33123
|
+
function trafficAnomaly(input, dependencies = defaultDependencies$3) {
|
|
32850
33124
|
return trafficAnomalyWith(input, dependencies);
|
|
32851
33125
|
}
|
|
32852
33126
|
async function updateCorrelation(input) {
|
|
@@ -32859,7 +33133,7 @@ async function updateCorrelation(input) {
|
|
|
32859
33133
|
const comparisonEnd = window?.end;
|
|
32860
33134
|
const updates = await listSearchUpdates({
|
|
32861
33135
|
product: "Ranking",
|
|
32862
|
-
limit:
|
|
33136
|
+
limit: Number.POSITIVE_INFINITY
|
|
32863
33137
|
});
|
|
32864
33138
|
const overlappingUpdates = comparisonStart && comparisonEnd ? findOverlappingSearchUpdates({
|
|
32865
33139
|
updates,
|
|
@@ -33204,7 +33478,7 @@ function emptyQuickWins(input) {
|
|
|
33204
33478
|
warnings: []
|
|
33205
33479
|
};
|
|
33206
33480
|
}
|
|
33207
|
-
const defaultDependencies$
|
|
33481
|
+
const defaultDependencies$2 = {
|
|
33208
33482
|
trafficAnomaly,
|
|
33209
33483
|
updateCorrelation,
|
|
33210
33484
|
segmentImpact,
|
|
@@ -33215,7 +33489,7 @@ const defaultDependencies$3 = {
|
|
|
33215
33489
|
};
|
|
33216
33490
|
async function diagnoseProperty(input, dependencies = {}) {
|
|
33217
33491
|
const providers = {
|
|
33218
|
-
...defaultDependencies$
|
|
33492
|
+
...defaultDependencies$2,
|
|
33219
33493
|
...dependencies
|
|
33220
33494
|
};
|
|
33221
33495
|
const limit = input.limit ?? 10;
|
|
@@ -33426,296 +33700,281 @@ async function diagnoseProperty(input, dependencies = {}) {
|
|
|
33426
33700
|
};
|
|
33427
33701
|
}
|
|
33428
33702
|
//#endregion
|
|
33429
|
-
//#region packages/core/dist/
|
|
33430
|
-
|
|
33431
|
-
|
|
33432
|
-
|
|
33433
|
-
|
|
33434
|
-
|
|
33435
|
-
|
|
33436
|
-
|
|
33437
|
-
|
|
33438
|
-
"for",
|
|
33439
|
-
"free",
|
|
33440
|
-
"from",
|
|
33441
|
-
"guide",
|
|
33442
|
-
"how",
|
|
33443
|
-
"in",
|
|
33444
|
-
"is",
|
|
33445
|
-
"of",
|
|
33446
|
-
"on",
|
|
33447
|
-
"online",
|
|
33448
|
-
"or",
|
|
33449
|
-
"page",
|
|
33450
|
-
"seo",
|
|
33451
|
-
"site",
|
|
33452
|
-
"the",
|
|
33453
|
-
"to",
|
|
33454
|
-
"tool",
|
|
33455
|
-
"use",
|
|
33456
|
-
"website",
|
|
33457
|
-
"what",
|
|
33458
|
-
"when",
|
|
33459
|
-
"where",
|
|
33460
|
-
"which",
|
|
33461
|
-
"with"
|
|
33462
|
-
]);
|
|
33463
|
-
function compareInternalLinkText(left, right) {
|
|
33464
|
-
return left < right ? -1 : left > right ? 1 : 0;
|
|
33465
|
-
}
|
|
33466
|
-
function normalizeInternalLinkQuery(value) {
|
|
33467
|
-
return value.normalize("NFKC").replace(/[’']/g, "'").toLowerCase().replace(/[^\p{L}\p{N}']+/gu, " ").replace(/\s+/g, " ").trim();
|
|
33468
|
-
}
|
|
33469
|
-
function normalizeInternalLinkUrl(value) {
|
|
33470
|
-
try {
|
|
33471
|
-
const url = new URL(value);
|
|
33472
|
-
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
33473
|
-
url.hash = "";
|
|
33474
|
-
return url.toString();
|
|
33475
|
-
} catch {
|
|
33476
|
-
return;
|
|
33703
|
+
//#region packages/core/dist/decision-model.js
|
|
33704
|
+
var DecisionModelError = class extends Error {
|
|
33705
|
+
kind;
|
|
33706
|
+
retryAfterMs;
|
|
33707
|
+
constructor(kind, message, retryAfterMs) {
|
|
33708
|
+
super(message);
|
|
33709
|
+
this.kind = kind;
|
|
33710
|
+
this.retryAfterMs = retryAfterMs;
|
|
33711
|
+
this.name = "DecisionModelError";
|
|
33477
33712
|
}
|
|
33478
|
-
|
|
33479
|
-
|
|
33480
|
-
|
|
33481
|
-
|
|
33482
|
-
|
|
33483
|
-
|
|
33484
|
-
|
|
33485
|
-
|
|
33486
|
-
|
|
33487
|
-
|
|
33488
|
-
|
|
33489
|
-
|
|
33490
|
-
|
|
33491
|
-
|
|
33492
|
-
|
|
33493
|
-
|
|
33494
|
-
|
|
33495
|
-
|
|
33496
|
-
|
|
33497
|
-
|
|
33498
|
-
|
|
33499
|
-
|
|
33500
|
-
|
|
33501
|
-
|
|
33502
|
-
|
|
33503
|
-
|
|
33504
|
-
|
|
33505
|
-
|
|
33506
|
-
|
|
33507
|
-
|
|
33508
|
-
|
|
33509
|
-
|
|
33510
|
-
|
|
33511
|
-
|
|
33512
|
-
|
|
33513
|
-
|
|
33514
|
-
|
|
33515
|
-
|
|
33516
|
-
|
|
33517
|
-
|
|
33518
|
-
|
|
33519
|
-
|
|
33520
|
-
|
|
33521
|
-
|
|
33522
|
-
|
|
33523
|
-
|
|
33524
|
-
|
|
33525
|
-
|
|
33526
|
-
|
|
33527
|
-
|
|
33528
|
-
|
|
33713
|
+
get retryable() {
|
|
33714
|
+
return this.kind === "rate-limit" || this.kind === "unavailable";
|
|
33715
|
+
}
|
|
33716
|
+
};
|
|
33717
|
+
const TYPESAFE_URL = "https://api.typesafe.ai/v1/systemone";
|
|
33718
|
+
const JEV_PRICE_PER_MILLION = .042;
|
|
33719
|
+
const REQUEST_TIMEOUT_MS = 6e4;
|
|
33720
|
+
const MAX_RESPONSE_BYTES = 1024 * 1024;
|
|
33721
|
+
function retryAfter(response) {
|
|
33722
|
+
const value = Number(response.headers.get("retry-after"));
|
|
33723
|
+
return Number.isFinite(value) && value > 0 ? Math.min(value, 30) * 1e3 : void 0;
|
|
33724
|
+
}
|
|
33725
|
+
function typesafeDecisionModel(input) {
|
|
33726
|
+
const call = input.fetch ?? fetch;
|
|
33727
|
+
const model = input.model ?? "jev-latest";
|
|
33728
|
+
return {
|
|
33729
|
+
provider: "typesafe",
|
|
33730
|
+
name: model,
|
|
33731
|
+
pricePerMillionInputTokens: JEV_PRICE_PER_MILLION,
|
|
33732
|
+
async ask(request, signal) {
|
|
33733
|
+
const controller = new AbortController();
|
|
33734
|
+
const abort = () => controller.abort();
|
|
33735
|
+
const timer = setTimeout(abort, REQUEST_TIMEOUT_MS);
|
|
33736
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
33737
|
+
let response;
|
|
33738
|
+
try {
|
|
33739
|
+
response = await call(TYPESAFE_URL, {
|
|
33740
|
+
method: "POST",
|
|
33741
|
+
headers: {
|
|
33742
|
+
authorization: `Bearer ${input.apiKey}`,
|
|
33743
|
+
"content-type": "application/json"
|
|
33744
|
+
},
|
|
33745
|
+
body: JSON.stringify({
|
|
33746
|
+
model,
|
|
33747
|
+
...request
|
|
33748
|
+
}),
|
|
33749
|
+
signal: controller.signal
|
|
33750
|
+
});
|
|
33751
|
+
} catch (error) {
|
|
33752
|
+
throw new DecisionModelError("unavailable", `TypeSafe could not be reached: ${error instanceof Error ? error.message : String(error)}`);
|
|
33753
|
+
} finally {
|
|
33754
|
+
clearTimeout(timer);
|
|
33755
|
+
signal?.removeEventListener("abort", abort);
|
|
33756
|
+
}
|
|
33757
|
+
if (response.status === 401 || response.status === 403) {
|
|
33758
|
+
await response.body?.cancel();
|
|
33759
|
+
throw new DecisionModelError("auth", "TypeSafe rejected the API key. Replace it at https://seoskill.dev/account#keys or in TYPESAFE_API_KEY.");
|
|
33760
|
+
}
|
|
33761
|
+
if (response.status === 429 || response.status === 529) {
|
|
33762
|
+
await response.body?.cancel();
|
|
33763
|
+
throw new DecisionModelError("rate-limit", `TypeSafe returned HTTP ${response.status}.`, retryAfter(response));
|
|
33764
|
+
}
|
|
33765
|
+
if (response.status >= 500) {
|
|
33766
|
+
await response.body?.cancel();
|
|
33767
|
+
throw new DecisionModelError("unavailable", `TypeSafe returned HTTP ${response.status}.`);
|
|
33768
|
+
}
|
|
33769
|
+
if (!response.ok) {
|
|
33770
|
+
await response.body?.cancel();
|
|
33771
|
+
throw new DecisionModelError("invalid", `TypeSafe did not accept the request (HTTP ${response.status}).`);
|
|
33772
|
+
}
|
|
33773
|
+
const body = await readJson(response, "TypeSafe", { maxBytes: MAX_RESPONSE_BYTES });
|
|
33774
|
+
return {
|
|
33775
|
+
model: body.model ?? model,
|
|
33776
|
+
answers: body.answers ?? {},
|
|
33777
|
+
inputTokens: body.usage?.input_tokens ?? 0
|
|
33778
|
+
};
|
|
33779
|
+
}
|
|
33529
33780
|
};
|
|
33530
33781
|
}
|
|
33531
|
-
|
|
33532
|
-
|
|
33533
|
-
const
|
|
33534
|
-
|
|
33535
|
-
|
|
33536
|
-
|
|
33537
|
-
|
|
33538
|
-
|
|
33539
|
-
|
|
33540
|
-
|
|
33541
|
-
|
|
33542
|
-
|
|
33543
|
-
|
|
33544
|
-
|
|
33545
|
-
|
|
33546
|
-
|
|
33547
|
-
|
|
33548
|
-
|
|
33549
|
-
|
|
33550
|
-
current.clicks += row.clicks;
|
|
33551
|
-
current.impressions += row.impressions;
|
|
33552
|
-
if (compareInternalLinkText(query, current.query) < 0) current.query = query;
|
|
33553
|
-
map.set(key, current);
|
|
33554
|
-
}
|
|
33555
|
-
function bestMatch(input) {
|
|
33556
|
-
const sourceKey = normalizeInternalLinkQuery(input.source.query);
|
|
33557
|
-
const exact = input.exactTargets.get(sourceKey);
|
|
33558
|
-
if (exact) return {
|
|
33559
|
-
sourceQuery: input.source.query,
|
|
33560
|
-
targetQuery: exact.query,
|
|
33561
|
-
kind: "exact-query",
|
|
33562
|
-
relevanceScore: 1,
|
|
33563
|
-
impressions: input.source.impressions,
|
|
33564
|
-
sharedTerms: internalLinkTokens(input.source.query)
|
|
33565
|
-
};
|
|
33566
|
-
return input.lexicalTargets.flatMap((target) => {
|
|
33567
|
-
const score = input.scoreLexical(input.source.query, target.query);
|
|
33568
|
-
return score ? [{
|
|
33569
|
-
sourceQuery: input.source.query,
|
|
33570
|
-
targetQuery: target.query,
|
|
33571
|
-
kind: "lexical-review",
|
|
33572
|
-
relevanceScore: score.score,
|
|
33573
|
-
impressions: input.source.impressions,
|
|
33574
|
-
sharedTerms: score.sharedTerms
|
|
33575
|
-
}] : [];
|
|
33576
|
-
}).sort((left, right) => right.relevanceScore - left.relevanceScore || compareInternalLinkText(left.targetQuery, right.targetQuery))[0];
|
|
33577
|
-
}
|
|
33578
|
-
function candidatesFromMatches(matchesByUrl) {
|
|
33579
|
-
return [...matchesByUrl.entries()].map(([sourceUrl, matches]) => {
|
|
33580
|
-
const ordered = [...matches].sort((left, right) => (left.kind === right.kind ? 0 : left.kind === "exact-query" ? -1 : 1) || right.impressions - left.impressions || right.relevanceScore - left.relevanceScore || compareInternalLinkText(left.sourceQuery, right.sourceQuery));
|
|
33782
|
+
const MAX_CONCURRENCY = 32;
|
|
33783
|
+
const MIN_CONCURRENCY = 2;
|
|
33784
|
+
const MAX_ATTEMPTS = 5;
|
|
33785
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
33786
|
+
var DecisionRunner = class {
|
|
33787
|
+
model;
|
|
33788
|
+
limit = MAX_CONCURRENCY;
|
|
33789
|
+
active = 0;
|
|
33790
|
+
streak = 0;
|
|
33791
|
+
waiting = [];
|
|
33792
|
+
served = "";
|
|
33793
|
+
requests = 0;
|
|
33794
|
+
failed = 0;
|
|
33795
|
+
tokens = 0;
|
|
33796
|
+
authError = null;
|
|
33797
|
+
constructor(model) {
|
|
33798
|
+
this.model = model;
|
|
33799
|
+
}
|
|
33800
|
+
usage() {
|
|
33581
33801
|
return {
|
|
33582
|
-
|
|
33583
|
-
|
|
33584
|
-
|
|
33585
|
-
|
|
33586
|
-
|
|
33587
|
-
|
|
33588
|
-
matches: ordered.slice(0, 10)
|
|
33802
|
+
provider: this.model.provider,
|
|
33803
|
+
model: this.served || this.model.name,
|
|
33804
|
+
requests: this.requests,
|
|
33805
|
+
failedRequests: this.failed,
|
|
33806
|
+
inputTokens: this.tokens,
|
|
33807
|
+
costUsd: Math.round(this.tokens * this.model.pricePerMillionInputTokens / 1e6 * 1e4) / 1e4
|
|
33589
33808
|
};
|
|
33590
|
-
}
|
|
33591
|
-
|
|
33592
|
-
|
|
33593
|
-
|
|
33594
|
-
|
|
33595
|
-
|
|
33596
|
-
|
|
33597
|
-
|
|
33598
|
-
|
|
33599
|
-
|
|
33600
|
-
|
|
33601
|
-
|
|
33602
|
-
|
|
33603
|
-
|
|
33604
|
-
|
|
33605
|
-
|
|
33606
|
-
|
|
33607
|
-
|
|
33608
|
-
|
|
33609
|
-
|
|
33610
|
-
|
|
33809
|
+
}
|
|
33810
|
+
async slot() {
|
|
33811
|
+
while (this.active >= this.limit) await new Promise((resolve) => this.waiting.push(resolve));
|
|
33812
|
+
this.active += 1;
|
|
33813
|
+
}
|
|
33814
|
+
release() {
|
|
33815
|
+
this.active -= 1;
|
|
33816
|
+
this.waiting.shift()?.();
|
|
33817
|
+
}
|
|
33818
|
+
async ask(request) {
|
|
33819
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
|
|
33820
|
+
if (this.authError) break;
|
|
33821
|
+
let delay = 0;
|
|
33822
|
+
await this.slot();
|
|
33823
|
+
try {
|
|
33824
|
+
const response = await this.model.ask(request);
|
|
33825
|
+
this.requests += 1;
|
|
33826
|
+
this.tokens += response.inputTokens;
|
|
33827
|
+
this.served = response.model;
|
|
33828
|
+
this.streak += 1;
|
|
33829
|
+
if (this.streak >= 50 && this.limit < MAX_CONCURRENCY) {
|
|
33830
|
+
this.limit += 1;
|
|
33831
|
+
this.streak = 0;
|
|
33832
|
+
this.waiting.shift()?.();
|
|
33833
|
+
}
|
|
33834
|
+
return response;
|
|
33835
|
+
} catch (error) {
|
|
33836
|
+
const failure = error instanceof DecisionModelError ? error : new DecisionModelError("unavailable", String(error));
|
|
33837
|
+
if (failure.kind === "auth") {
|
|
33838
|
+
this.authError = failure;
|
|
33839
|
+
break;
|
|
33840
|
+
}
|
|
33841
|
+
if (!failure.retryable || attempt === MAX_ATTEMPTS) break;
|
|
33842
|
+
if (failure.kind === "rate-limit") {
|
|
33843
|
+
this.limit = Math.max(MIN_CONCURRENCY, Math.floor(this.limit / 2));
|
|
33844
|
+
this.streak = 0;
|
|
33845
|
+
}
|
|
33846
|
+
delay = failure.retryAfterMs ?? 2 ** attempt * 250 + Math.floor(Math.random() * 250);
|
|
33847
|
+
} finally {
|
|
33848
|
+
this.release();
|
|
33849
|
+
}
|
|
33850
|
+
await sleep(delay);
|
|
33611
33851
|
}
|
|
33612
|
-
|
|
33852
|
+
this.failed += 1;
|
|
33853
|
+
return null;
|
|
33613
33854
|
}
|
|
33614
|
-
|
|
33615
|
-
|
|
33616
|
-
|
|
33617
|
-
|
|
33618
|
-
|
|
33619
|
-
|
|
33620
|
-
|
|
33855
|
+
};
|
|
33856
|
+
//#endregion
|
|
33857
|
+
//#region packages/core/dist/analyze/internal-links-index.js
|
|
33858
|
+
const PAGE_TERMS = 300;
|
|
33859
|
+
const VECTOR_TERMS = 100;
|
|
33860
|
+
var SiteIndex = class {
|
|
33861
|
+
ids = /* @__PURE__ */ new Map();
|
|
33862
|
+
df = [];
|
|
33863
|
+
counts = [];
|
|
33864
|
+
vectors = [];
|
|
33865
|
+
postings = [];
|
|
33866
|
+
scratch = /* @__PURE__ */ new Float64Array(0);
|
|
33867
|
+
get size() {
|
|
33868
|
+
return this.counts.length;
|
|
33869
|
+
}
|
|
33870
|
+
addPage(termCounts) {
|
|
33871
|
+
const kept = Object.entries(termCounts).sort((left, right) => right[1] - left[1] || (left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0)).slice(0, PAGE_TERMS);
|
|
33872
|
+
const ids = new Int32Array(kept.length);
|
|
33873
|
+
const counts = new Float32Array(kept.length);
|
|
33874
|
+
kept.forEach(([term, count], index) => {
|
|
33875
|
+
let id = this.ids.get(term);
|
|
33876
|
+
if (id === void 0) {
|
|
33877
|
+
id = this.df.length;
|
|
33878
|
+
this.ids.set(term, id);
|
|
33879
|
+
this.df.push(0);
|
|
33880
|
+
}
|
|
33881
|
+
this.df[id] = (this.df[id] ?? 0) + 1;
|
|
33882
|
+
ids[index] = id;
|
|
33883
|
+
counts[index] = count;
|
|
33621
33884
|
});
|
|
33622
|
-
|
|
33623
|
-
|
|
33624
|
-
|
|
33625
|
-
|
|
33626
|
-
|
|
33627
|
-
const lexicalTargets = allTargets.slice(0, LEXICAL_TARGET_LIMIT);
|
|
33628
|
-
const exactTargets = new Map(allTargets.map((target) => [normalizeInternalLinkQuery(target.query), target]));
|
|
33629
|
-
const sourceMap = /* @__PURE__ */ new Map();
|
|
33630
|
-
let sourceValidRows = 0;
|
|
33631
|
-
for (const row of input.sourceRows) {
|
|
33632
|
-
if (!validInternalLinkRow(row)) {
|
|
33633
|
-
increment("sourceInvalidRows");
|
|
33634
|
-
continue;
|
|
33635
|
-
}
|
|
33636
|
-
sourceValidRows += 1;
|
|
33637
|
-
const query = row.keys[0] ?? "";
|
|
33638
|
-
const page = normalizeInternalLinkUrl(row.keys[1] ?? "");
|
|
33639
|
-
if (!page) continue;
|
|
33640
|
-
if (aliases.has(page)) {
|
|
33641
|
-
increment("sourceTargetAliasRows");
|
|
33642
|
-
continue;
|
|
33643
|
-
}
|
|
33644
|
-
const key = `${page}\u0000${normalizeInternalLinkQuery(query)}`;
|
|
33645
|
-
const current = sourceMap.get(key) ?? {
|
|
33646
|
-
page,
|
|
33647
|
-
query,
|
|
33648
|
-
clicks: 0,
|
|
33649
|
-
impressions: 0
|
|
33650
|
-
};
|
|
33651
|
-
current.clicks += row.clicks;
|
|
33652
|
-
current.impressions += row.impressions;
|
|
33653
|
-
if (compareInternalLinkText(query, current.query) < 0) current.query = query;
|
|
33654
|
-
sourceMap.set(key, current);
|
|
33885
|
+
this.counts.push({
|
|
33886
|
+
ids,
|
|
33887
|
+
counts
|
|
33888
|
+
});
|
|
33889
|
+
return this.counts.length - 1;
|
|
33655
33890
|
}
|
|
33656
|
-
|
|
33657
|
-
|
|
33658
|
-
|
|
33659
|
-
|
|
33660
|
-
|
|
33661
|
-
|
|
33662
|
-
|
|
33891
|
+
idfOf(id) {
|
|
33892
|
+
return Math.log((this.size + 1) / ((this.df[id] ?? 1) + 1)) + 1;
|
|
33893
|
+
}
|
|
33894
|
+
idf(term) {
|
|
33895
|
+
const id = this.ids.get(term);
|
|
33896
|
+
return id === void 0 ? Math.log((this.size + 1) / 2) + 1 : this.idfOf(id);
|
|
33897
|
+
}
|
|
33898
|
+
finish() {
|
|
33899
|
+
this.vectors = this.counts.map(({ ids, counts }) => {
|
|
33900
|
+
let norm = 0;
|
|
33901
|
+
const shared = [];
|
|
33902
|
+
for (let index = 0; index < ids.length; index += 1) {
|
|
33903
|
+
const id = ids[index] ?? 0;
|
|
33904
|
+
const weight = (1 + Math.log(counts[index] ?? 1)) * this.idfOf(id);
|
|
33905
|
+
norm += weight * weight;
|
|
33906
|
+
if ((this.df[id] ?? 0) >= 2) shared.push([id, weight]);
|
|
33907
|
+
}
|
|
33908
|
+
norm = Math.sqrt(norm) || 1;
|
|
33909
|
+
shared.sort((left, right) => right[1] - left[1] || left[0] - right[0]);
|
|
33910
|
+
const top = shared.slice(0, VECTOR_TERMS);
|
|
33911
|
+
return {
|
|
33912
|
+
ids: Int32Array.from(top.map(([id]) => id)),
|
|
33913
|
+
weights: Float32Array.from(top.map(([, weight]) => weight / norm))
|
|
33914
|
+
};
|
|
33663
33915
|
});
|
|
33664
|
-
|
|
33665
|
-
|
|
33666
|
-
|
|
33667
|
-
|
|
33668
|
-
|
|
33669
|
-
|
|
33670
|
-
|
|
33671
|
-
|
|
33672
|
-
|
|
33673
|
-
|
|
33674
|
-
|
|
33675
|
-
exactTargets,
|
|
33676
|
-
lexicalTargets,
|
|
33677
|
-
scoreLexical
|
|
33916
|
+
const lists = this.df.map(() => ({
|
|
33917
|
+
pages: [],
|
|
33918
|
+
weights: []
|
|
33919
|
+
}));
|
|
33920
|
+
this.vectors.forEach((vector, page) => {
|
|
33921
|
+
vector.ids.forEach((id, index) => {
|
|
33922
|
+
const list = lists[id];
|
|
33923
|
+
if (!list) return;
|
|
33924
|
+
list.pages.push(page);
|
|
33925
|
+
list.weights.push(vector.weights[index] ?? 0);
|
|
33926
|
+
});
|
|
33678
33927
|
});
|
|
33679
|
-
|
|
33680
|
-
|
|
33681
|
-
|
|
33928
|
+
this.postings = lists.map((list) => ({
|
|
33929
|
+
pages: Int32Array.from(list.pages),
|
|
33930
|
+
weights: Float32Array.from(list.weights)
|
|
33931
|
+
}));
|
|
33932
|
+
this.scratch = new Float64Array(this.size);
|
|
33933
|
+
}
|
|
33934
|
+
similarity(left, right) {
|
|
33935
|
+
const a = this.vectors[left];
|
|
33936
|
+
const b = this.vectors[right];
|
|
33937
|
+
if (!a || !b) return 0;
|
|
33938
|
+
const weights = /* @__PURE__ */ new Map();
|
|
33939
|
+
a.ids.forEach((id, index) => {
|
|
33940
|
+
weights.set(id, a.weights[index] ?? 0);
|
|
33941
|
+
});
|
|
33942
|
+
let sum = 0;
|
|
33943
|
+
b.ids.forEach((id, index) => {
|
|
33944
|
+
sum += (weights.get(id) ?? 0) * (b.weights[index] ?? 0);
|
|
33945
|
+
});
|
|
33946
|
+
return sum;
|
|
33947
|
+
}
|
|
33948
|
+
neighbours(page, min) {
|
|
33949
|
+
const vector = this.vectors[page];
|
|
33950
|
+
if (!vector) return [];
|
|
33951
|
+
const scores = this.scratch;
|
|
33952
|
+
const touched = [];
|
|
33953
|
+
vector.ids.forEach((id, index) => {
|
|
33954
|
+
const weight = vector.weights[index] ?? 0;
|
|
33955
|
+
const list = this.postings[id];
|
|
33956
|
+
if (!list) return;
|
|
33957
|
+
for (let at = 0; at < list.pages.length; at += 1) {
|
|
33958
|
+
const other = list.pages[at] ?? 0;
|
|
33959
|
+
if (other === page) continue;
|
|
33960
|
+
if (scores[other] === 0) touched.push(other);
|
|
33961
|
+
scores[other] = (scores[other] ?? 0) + weight * (list.weights[at] ?? 0);
|
|
33962
|
+
}
|
|
33963
|
+
});
|
|
33964
|
+
const out = [];
|
|
33965
|
+
for (const other of touched) {
|
|
33966
|
+
const score = scores[other] ?? 0;
|
|
33967
|
+
if (score >= min) out.push({
|
|
33968
|
+
page: other,
|
|
33969
|
+
score
|
|
33970
|
+
});
|
|
33971
|
+
scores[other] = 0;
|
|
33682
33972
|
}
|
|
33683
|
-
|
|
33684
|
-
const matches = matchesByUrl.get(source.page) ?? [];
|
|
33685
|
-
matches.push(match);
|
|
33686
|
-
matchesByUrl.set(source.page, matches);
|
|
33973
|
+
return out.sort((left, right) => right.score - left.score || left.page - right.page);
|
|
33687
33974
|
}
|
|
33688
|
-
|
|
33689
|
-
|
|
33690
|
-
|
|
33691
|
-
targetQueries: lexicalTargets,
|
|
33692
|
-
candidates,
|
|
33693
|
-
selection: {
|
|
33694
|
-
targetSourceRows: input.targetRows.length,
|
|
33695
|
-
targetValidRows,
|
|
33696
|
-
targetInvalidRows: value("targetInvalidRows"),
|
|
33697
|
-
targetUrlMismatchRows: value("targetUrlMismatchRows"),
|
|
33698
|
-
targetLowImpressionQueries: value("targetLowImpressionQueries"),
|
|
33699
|
-
targetLowActionabilityQueries: value("targetLowActionabilityQueries"),
|
|
33700
|
-
targetBrandQueries: value("targetBrandQueries"),
|
|
33701
|
-
targetEligibleQueries: allTargets.length,
|
|
33702
|
-
selectedLexicalTargetQueries: lexicalTargets.length,
|
|
33703
|
-
sourceRows: input.sourceRows.length,
|
|
33704
|
-
sourceValidRows,
|
|
33705
|
-
sourceInvalidRows: value("sourceInvalidRows"),
|
|
33706
|
-
sourceTargetAliasRows: value("sourceTargetAliasRows"),
|
|
33707
|
-
sourceLowImpressionQueries: value("sourceLowImpressionQueries"),
|
|
33708
|
-
sourceLowActionabilityQueries: value("sourceLowActionabilityQueries"),
|
|
33709
|
-
sourceBrandQueries: value("sourceBrandQueries"),
|
|
33710
|
-
sourceUnmatchedQueries: value("sourceUnmatchedQueries"),
|
|
33711
|
-
candidateQueries,
|
|
33712
|
-
candidateUrls: candidates.length
|
|
33713
|
-
}
|
|
33714
|
-
};
|
|
33715
|
-
}
|
|
33716
|
-
const MAX_TERMS = 100;
|
|
33717
|
-
const PLACEMENT_SENTENCES = 2;
|
|
33718
|
-
const ANCHORS_PER_SENTENCE = 3;
|
|
33975
|
+
};
|
|
33976
|
+
const PAIR_SENTENCES = 4;
|
|
33977
|
+
const PAIR_ANCHORS = 8;
|
|
33719
33978
|
const MIN_SENTENCE_WORDS = 6;
|
|
33720
33979
|
const NAME_WEIGHT = 2;
|
|
33721
33980
|
const GENERIC_ANCHORS = /^(click here|here|read more|learn more|more|this|this post|this article|this page|link|website|check it out|see more|view|continue reading)$/i;
|
|
@@ -33985,65 +34244,49 @@ function pageSentences(html) {
|
|
|
33985
34244
|
flushRaw("");
|
|
33986
34245
|
return sentences;
|
|
33987
34246
|
}
|
|
33988
|
-
|
|
33989
|
-
|
|
33990
|
-
|
|
33991
|
-
|
|
33992
|
-
|
|
33993
|
-
|
|
33994
|
-
|
|
33995
|
-
|
|
33996
|
-
return Math.log((this.size + 1) / ((this.df.get(term) ?? 1) + 1)) + 1;
|
|
33997
|
-
}
|
|
33998
|
-
vector(counts) {
|
|
33999
|
-
let norm = 0;
|
|
34000
|
-
const shared = [];
|
|
34001
|
-
for (const term in counts) {
|
|
34002
|
-
const weight = (1 + Math.log(counts[term] ?? 1)) * this.idf(term);
|
|
34003
|
-
norm += weight * weight;
|
|
34004
|
-
if ((this.df.get(term) ?? 0) >= 2) shared.push([term, weight]);
|
|
34005
|
-
}
|
|
34006
|
-
norm = Math.sqrt(norm) || 1;
|
|
34007
|
-
shared.sort((left, right) => right[1] - left[1] || (left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0));
|
|
34008
|
-
return new Map(shared.slice(0, MAX_TERMS).map(([term, weight]) => [term, weight / norm]));
|
|
34009
|
-
}
|
|
34010
|
-
similarity(left, right) {
|
|
34011
|
-
let sum = 0;
|
|
34012
|
-
for (const [term, weight] of left) sum += weight * (right.get(term) ?? 0);
|
|
34013
|
-
return sum;
|
|
34247
|
+
function linkableSentences(sentences) {
|
|
34248
|
+
const seen = /* @__PURE__ */ new Set();
|
|
34249
|
+
const out = [];
|
|
34250
|
+
for (const sentence of sentences) {
|
|
34251
|
+
if (out.length >= 150) break;
|
|
34252
|
+
if (seen.has(sentence.text)) continue;
|
|
34253
|
+
seen.add(sentence.text);
|
|
34254
|
+
if (!sentence.hasLink && sentence.text.split(/\s+/).length >= MIN_SENTENCE_WORDS) out.push(sentence);
|
|
34014
34255
|
}
|
|
34015
|
-
|
|
34016
|
-
|
|
34017
|
-
|
|
34018
|
-
|
|
34019
|
-
|
|
34020
|
-
|
|
34021
|
-
|
|
34022
|
-
|
|
34023
|
-
|
|
34024
|
-
|
|
34025
|
-
|
|
34256
|
+
return out;
|
|
34257
|
+
}
|
|
34258
|
+
function sentenceMatch(sentence, key, weights) {
|
|
34259
|
+
if (!key.size) return 0;
|
|
34260
|
+
const seen = new Set(terms(sentence));
|
|
34261
|
+
let hit = 0;
|
|
34262
|
+
let total = 0;
|
|
34263
|
+
for (const [term, place] of key) {
|
|
34264
|
+
const weight = weights.idf(term) * place;
|
|
34265
|
+
total += weight;
|
|
34266
|
+
if (seen.has(term)) hit += weight;
|
|
34026
34267
|
}
|
|
34027
|
-
|
|
34028
|
-
|
|
34268
|
+
return total ? hit / total : 0;
|
|
34269
|
+
}
|
|
34270
|
+
function anchorCandidates(sentence, key, weights, limit) {
|
|
34029
34271
|
const scored = /* @__PURE__ */ new Map();
|
|
34030
34272
|
const parts = sentence.split(/\s+/).map((word) => word.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "")).filter(Boolean);
|
|
34031
|
-
for (let size =
|
|
34273
|
+
for (let size = 1; size <= 4; size += 1) for (let start = 0; start + size <= parts.length; start += 1) {
|
|
34032
34274
|
const gram = parts.slice(start, start + size);
|
|
34033
34275
|
const first = (gram[0] ?? "").toLowerCase();
|
|
34034
34276
|
const last = (gram[size - 1] ?? "").toLowerCase();
|
|
34035
34277
|
if (FUNCTION_WORDS.has(first) || FUNCTION_WORDS.has(last) || /^\d+$/.test(first) || /^\d+$/.test(last)) continue;
|
|
34036
34278
|
const stems = terms(gram.join(" "));
|
|
34037
|
-
if (stems.length < 2) continue;
|
|
34279
|
+
if (stems.length < size || size > 1 && stems.length < 2) continue;
|
|
34280
|
+
if (size === 1 && key.get(stems[0] ?? "") !== NAME_WEIGHT) continue;
|
|
34038
34281
|
const hits = stems.filter((term) => key.has(term)).length;
|
|
34039
|
-
const named = stems.reduce((sum, term) => sum + (key.get(term) ?? 0) *
|
|
34282
|
+
const named = stems.reduce((sum, term) => sum + (key.get(term) ?? 0) * weights.idf(term), 0);
|
|
34040
34283
|
if (!stems.some((term) => key.get(term) === NAME_WEIGHT)) continue;
|
|
34041
34284
|
if (Number(key.has(terms(first)[0] ?? "")) + Number(key.has(terms(last)[0] ?? "")) < 2) continue;
|
|
34042
34285
|
const text = gram.join(" ");
|
|
34043
34286
|
const lower = text.toLowerCase();
|
|
34044
34287
|
if (GENERIC_ANCHORS.test(lower)) continue;
|
|
34045
|
-
const rarity = stems.reduce((sum, term) => sum +
|
|
34046
|
-
const score = named * (1 + hits / key.size) + rarity * .05 - (size === 4 ? .2 : 0);
|
|
34288
|
+
const rarity = stems.reduce((sum, term) => sum + weights.idf(term), 0);
|
|
34289
|
+
const score = named * (1 + hits / key.size) + rarity * .05 - (size === 1 ? .5 : size === 4 ? .2 : 0);
|
|
34047
34290
|
const existing = scored.get(lower);
|
|
34048
34291
|
if (!existing || existing.score < score) scored.set(lower, {
|
|
34049
34292
|
text,
|
|
@@ -34052,27 +34295,27 @@ function anchorCandidates(sentence, key, corpus, limit) {
|
|
|
34052
34295
|
}
|
|
34053
34296
|
return [...scored.values()].sort((left, right) => right.score - left.score || (left.text < right.text ? -1 : left.text > right.text ? 1 : 0)).slice(0, limit).map((item) => item.text);
|
|
34054
34297
|
}
|
|
34055
|
-
function
|
|
34298
|
+
function pairPlacement(sentences, target, weights) {
|
|
34056
34299
|
const key = keyTerms(target);
|
|
34057
|
-
const
|
|
34058
|
-
const seen = /* @__PURE__ */ new Set();
|
|
34059
|
-
const sentences = source.sentences.filter((sentence) => {
|
|
34060
|
-
if (seen.has(sentence.text)) return false;
|
|
34061
|
-
seen.add(sentence.text);
|
|
34062
|
-
return !sentence.hasLink && sentence.text.split(/\s+/).length >= MIN_SENTENCE_WORDS;
|
|
34063
|
-
}).map((sentence, index) => ({
|
|
34300
|
+
const chosen = sentences.map((sentence, index) => ({
|
|
34064
34301
|
sentence,
|
|
34065
34302
|
index,
|
|
34066
|
-
match:
|
|
34067
|
-
})).filter((item) => item.match > 0).sort((left, right) => right.match - left.match || left.index - right.index).slice(0,
|
|
34068
|
-
|
|
34069
|
-
|
|
34070
|
-
|
|
34071
|
-
|
|
34072
|
-
|
|
34303
|
+
match: sentenceMatch(sentence.text, key, weights)
|
|
34304
|
+
})).filter((item) => item.match > 0).sort((left, right) => right.match - left.match || left.index - right.index).slice(0, PAIR_SENTENCES);
|
|
34305
|
+
const perSentence = Math.max(1, Math.floor(PAIR_ANCHORS / Math.max(1, chosen.length)));
|
|
34306
|
+
const anchors = [];
|
|
34307
|
+
for (const { sentence, index } of chosen) for (const text of anchorCandidates(sentence.text, key, weights, perSentence)) anchors.push({
|
|
34308
|
+
text,
|
|
34309
|
+
sentence: `S${index + 1}`
|
|
34310
|
+
});
|
|
34073
34311
|
return {
|
|
34074
|
-
|
|
34075
|
-
|
|
34312
|
+
sentences: chosen.map(({ sentence, index, match }) => ({
|
|
34313
|
+
id: `S${index + 1}`,
|
|
34314
|
+
text: sentence.text,
|
|
34315
|
+
section: sentence.section,
|
|
34316
|
+
match: Math.round(match * 100) / 100
|
|
34317
|
+
})),
|
|
34318
|
+
anchors: anchors.slice(0, PAIR_ANCHORS)
|
|
34076
34319
|
};
|
|
34077
34320
|
}
|
|
34078
34321
|
function placementPage(input) {
|
|
@@ -34086,542 +34329,1140 @@ function placementPage(input) {
|
|
|
34086
34329
|
};
|
|
34087
34330
|
}
|
|
34088
34331
|
//#endregion
|
|
34089
|
-
//#region packages/core/dist/analyze/internal-links-
|
|
34090
|
-
const
|
|
34091
|
-
const
|
|
34092
|
-
|
|
34093
|
-
|
|
34094
|
-
|
|
34095
|
-
|
|
34096
|
-
|
|
34097
|
-
|
|
34098
|
-
|
|
34099
|
-
|
|
34100
|
-
|
|
34101
|
-
|
|
34102
|
-
|
|
34103
|
-
|
|
34104
|
-
|
|
34105
|
-
|
|
34106
|
-
|
|
34107
|
-
|
|
34108
|
-
|
|
34109
|
-
|
|
34110
|
-
}
|
|
34111
|
-
|
|
34112
|
-
|
|
34113
|
-
|
|
34114
|
-
|
|
34115
|
-
|
|
34116
|
-
|
|
34117
|
-
|
|
34118
|
-
|
|
34119
|
-
|
|
34332
|
+
//#region packages/core/dist/analyze/internal-links-crawl.js
|
|
34333
|
+
const MAX_MAX_PAGES = 5e3;
|
|
34334
|
+
const FETCH_CONCURRENCY = 6;
|
|
34335
|
+
const MAX_CRAWL_DELAY_SECONDS = 10;
|
|
34336
|
+
const CRAWL_DELAY_BUDGET_SECONDS = 900;
|
|
34337
|
+
const REDIRECT_CHECKS = 150;
|
|
34338
|
+
const GENERIC_ANCHOR_LIMIT = 100;
|
|
34339
|
+
const EXCERPT_CHARS = 500;
|
|
34340
|
+
const ASSET = /\.(avif|bmp|css|csv|dmg|docx?|eot|gif|gz|ico|jpe?g|js|json|map|mov|mp3|mp4|ogg|otf|pdf|png|pptx?|rss|svg|tar|ttf|txt|wav|webm|webp|woff2?|xlsx?|xml|zip)$/i;
|
|
34341
|
+
const TRACKING = /^(utm_[a-z]+|gclid|fbclid|msclkid|mc_cid|mc_eid|ref)$/i;
|
|
34342
|
+
var SentenceStore = class SentenceStore {
|
|
34343
|
+
dir;
|
|
34344
|
+
handle;
|
|
34345
|
+
size = 0;
|
|
34346
|
+
constructor(dir, handle) {
|
|
34347
|
+
this.dir = dir;
|
|
34348
|
+
this.handle = handle;
|
|
34349
|
+
}
|
|
34350
|
+
static async open() {
|
|
34351
|
+
const dir = await mkdtemp(join(tmpdir(), "seo-internal-links-"));
|
|
34352
|
+
return new SentenceStore(dir, await open$1(join(dir, "sentences"), "w+"));
|
|
34353
|
+
}
|
|
34354
|
+
async write(sentences) {
|
|
34355
|
+
const bytes = Buffer.from(JSON.stringify(sentences.map((s) => [s.text, s.section])));
|
|
34356
|
+
const offset = this.size;
|
|
34357
|
+
this.size += bytes.length;
|
|
34358
|
+
await this.handle.write(bytes, 0, bytes.length, offset);
|
|
34359
|
+
return {
|
|
34360
|
+
offset,
|
|
34361
|
+
length: bytes.length
|
|
34362
|
+
};
|
|
34120
34363
|
}
|
|
34121
|
-
|
|
34122
|
-
|
|
34123
|
-
|
|
34124
|
-
|
|
34125
|
-
|
|
34126
|
-
|
|
34127
|
-
|
|
34128
|
-
|
|
34129
|
-
|
|
34130
|
-
if (!value) return void 0;
|
|
34131
|
-
try {
|
|
34132
|
-
return assertUrlMatchesGscProperty(site, value);
|
|
34133
|
-
} catch {
|
|
34134
|
-
return;
|
|
34364
|
+
async read(ref) {
|
|
34365
|
+
if (!ref.length) return [];
|
|
34366
|
+
const bytes = Buffer.alloc(ref.length);
|
|
34367
|
+
await this.handle.read(bytes, 0, ref.length, ref.offset);
|
|
34368
|
+
return JSON.parse(bytes.toString("utf8")).map(([text, section]) => ({
|
|
34369
|
+
text,
|
|
34370
|
+
section,
|
|
34371
|
+
hasLink: false
|
|
34372
|
+
}));
|
|
34135
34373
|
}
|
|
34136
|
-
|
|
34137
|
-
|
|
34138
|
-
|
|
34139
|
-
|
|
34140
|
-
|
|
34141
|
-
}
|
|
34142
|
-
function sameObservedUrl(left, right) {
|
|
34143
|
-
try {
|
|
34144
|
-
return withoutFragment(left) === withoutFragment(right);
|
|
34145
|
-
} catch {
|
|
34146
|
-
return false;
|
|
34374
|
+
async close() {
|
|
34375
|
+
await this.handle.close().catch(() => void 0);
|
|
34376
|
+
await rm(this.dir, {
|
|
34377
|
+
recursive: true,
|
|
34378
|
+
force: true
|
|
34379
|
+
});
|
|
34147
34380
|
}
|
|
34148
|
-
}
|
|
34149
|
-
|
|
34150
|
-
|
|
34381
|
+
};
|
|
34382
|
+
const bareHost = (host) => host.toLowerCase().replace(/^www\./, "");
|
|
34383
|
+
function crawlUrl(value, base) {
|
|
34151
34384
|
try {
|
|
34152
|
-
|
|
34385
|
+
const url = base ? new URL(value, base) : new URL(value);
|
|
34386
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
34387
|
+
url.hash = "";
|
|
34388
|
+
for (const key of [...url.searchParams.keys()]) if (TRACKING.test(key)) url.searchParams.delete(key);
|
|
34389
|
+
return url.toString();
|
|
34153
34390
|
} catch {
|
|
34154
34391
|
return;
|
|
34155
34392
|
}
|
|
34156
34393
|
}
|
|
34157
|
-
|
|
34158
|
-
let
|
|
34159
|
-
|
|
34160
|
-
|
|
34161
|
-
|
|
34162
|
-
|
|
34163
|
-
|
|
34164
|
-
|
|
34394
|
+
function excerptOf(sentences) {
|
|
34395
|
+
let out = "";
|
|
34396
|
+
for (const sentence of sentences) {
|
|
34397
|
+
const next = out ? `${out} ${sentence.text}` : sentence.text;
|
|
34398
|
+
if (next.length > EXCERPT_CHARS) break;
|
|
34399
|
+
out = next;
|
|
34400
|
+
}
|
|
34401
|
+
return out || (sentences[0]?.text.slice(0, EXCERPT_CHARS) ?? "");
|
|
34402
|
+
}
|
|
34403
|
+
function readPage(fetched, finalUrl) {
|
|
34404
|
+
const $ = load(fetched.html);
|
|
34405
|
+
const meta = (name) => $(`meta[name="${name}" i]`).map((_, element) => $(element).attr("content") ?? "").get();
|
|
34406
|
+
const headings = $("h1, h2, h3").map((_, element) => ({
|
|
34407
|
+
level: Number(element.tagName.slice(1)),
|
|
34408
|
+
text: $(element).text().replace(/\s+/g, " ").trim()
|
|
34409
|
+
})).get().filter((heading) => heading.text);
|
|
34410
|
+
const links = $("a[href]").map((_, element) => {
|
|
34411
|
+
const anchor = $(element);
|
|
34165
34412
|
return {
|
|
34166
|
-
|
|
34167
|
-
|
|
34168
|
-
|
|
34169
|
-
verification: "failed",
|
|
34170
|
-
technicalSignals: [],
|
|
34171
|
-
warnings: [{
|
|
34172
|
-
stage: "target-fetch",
|
|
34173
|
-
url: input.targetUrl,
|
|
34174
|
-
code: "fetch-failed",
|
|
34175
|
-
message: errorMessage(error)
|
|
34176
|
-
}]
|
|
34413
|
+
href: anchor.attr("href") ?? "",
|
|
34414
|
+
text: anchor.text().replace(/\s+/g, " ").trim(),
|
|
34415
|
+
location: anchor.closest("nav, header").length ? "navigation" : anchor.closest("footer").length ? "footer" : anchor.closest("main, article, [role=\"main\"]").length ? "main-content" : "other"
|
|
34177
34416
|
};
|
|
34178
|
-
}
|
|
34179
|
-
|
|
34417
|
+
}).get();
|
|
34418
|
+
const robots = [...meta("robots"), ...meta("googlebot")];
|
|
34419
|
+
return {
|
|
34420
|
+
title: $("title").first().text().replace(/\s+/g, " ").trim(),
|
|
34421
|
+
metaDescription: meta("description")[0]?.trim() ?? "",
|
|
34422
|
+
canonical: $("link[rel~=\"canonical\" i]").first().attr("href"),
|
|
34423
|
+
lang: $("html").attr("lang"),
|
|
34424
|
+
headings,
|
|
34425
|
+
links,
|
|
34426
|
+
nofollow: robots.some((value) => /\bnofollow\b/i.test(value)),
|
|
34427
|
+
noindex: hasMetaRobotsDirective(robots, "noindex") || hasXRobotsDirective(fetched.headers["x-robots-tag"], "noindex"),
|
|
34428
|
+
url: finalUrl
|
|
34429
|
+
};
|
|
34430
|
+
}
|
|
34431
|
+
async function crawlForLinks(input) {
|
|
34432
|
+
const warnings = [];
|
|
34433
|
+
const robots = createRobotsSession({ refresh: input.refresh });
|
|
34434
|
+
const sentences = await SentenceStore.open();
|
|
34435
|
+
const renderer = createPageRenderer({ concurrency: 2 });
|
|
34180
34436
|
try {
|
|
34181
|
-
|
|
34437
|
+
return await crawl(input, robots, sentences, renderer, warnings);
|
|
34182
34438
|
} catch (error) {
|
|
34183
|
-
|
|
34184
|
-
|
|
34185
|
-
|
|
34186
|
-
|
|
34187
|
-
status: fetched.status,
|
|
34188
|
-
aliases: [input.targetUrl],
|
|
34189
|
-
verification: "failed",
|
|
34190
|
-
technicalSignals: [],
|
|
34191
|
-
fetchDiagnostics: fetched.diagnostics,
|
|
34192
|
-
warnings: [{
|
|
34193
|
-
stage: "target-extract",
|
|
34194
|
-
url: input.targetUrl,
|
|
34195
|
-
code: "extract-failed",
|
|
34196
|
-
message: errorMessage(error)
|
|
34197
|
-
}]
|
|
34198
|
-
};
|
|
34439
|
+
await sentences.close();
|
|
34440
|
+
throw error;
|
|
34441
|
+
} finally {
|
|
34442
|
+
await renderer.close();
|
|
34199
34443
|
}
|
|
34200
|
-
|
|
34201
|
-
|
|
34202
|
-
const
|
|
34203
|
-
|
|
34204
|
-
|
|
34205
|
-
|
|
34206
|
-
|
|
34207
|
-
|
|
34208
|
-
|
|
34209
|
-
|
|
34210
|
-
|
|
34211
|
-
|
|
34212
|
-
|
|
34213
|
-
|
|
34444
|
+
}
|
|
34445
|
+
async function crawl(input, robots, sentences, renderer, warnings) {
|
|
34446
|
+
const start = crawlUrl(input.startUrl) ?? input.startUrl;
|
|
34447
|
+
let origin = new URL(start).origin;
|
|
34448
|
+
const site = bareHost(new URL(start).host);
|
|
34449
|
+
const sameSite = (url) => {
|
|
34450
|
+
try {
|
|
34451
|
+
return bareHost(new URL(url).host) === site;
|
|
34452
|
+
} catch {
|
|
34453
|
+
return false;
|
|
34454
|
+
}
|
|
34455
|
+
};
|
|
34456
|
+
const urls = [];
|
|
34457
|
+
const urlIds = /* @__PURE__ */ new Map();
|
|
34458
|
+
const idOf = (url) => {
|
|
34459
|
+
let id = urlIds.get(url);
|
|
34460
|
+
if (id === void 0) {
|
|
34461
|
+
id = urls.length;
|
|
34462
|
+
urls.push(url);
|
|
34463
|
+
urlIds.set(url, id);
|
|
34464
|
+
}
|
|
34465
|
+
return id;
|
|
34466
|
+
};
|
|
34467
|
+
const aliases = /* @__PURE__ */ new Map();
|
|
34468
|
+
const pageOfUrl = /* @__PURE__ */ new Map();
|
|
34469
|
+
const pages = [];
|
|
34470
|
+
const index = new SiteIndex();
|
|
34471
|
+
const genericAnchors = [];
|
|
34472
|
+
const skipped = {};
|
|
34473
|
+
const skip = (reason) => {
|
|
34474
|
+
skipped[reason] = (skipped[reason] ?? 0) + 1;
|
|
34475
|
+
};
|
|
34476
|
+
const rootRobots = await robots.resolve(origin, start);
|
|
34477
|
+
const delay = await robots.crawlDelay(origin);
|
|
34478
|
+
const crawlDelaySeconds = delay === void 0 ? null : Math.min(delay, MAX_CRAWL_DELAY_SECONDS);
|
|
34479
|
+
const pageLimit = crawlDelaySeconds === null ? input.maxPages : Math.min(input.maxPages, Math.max(20, Math.floor(CRAWL_DELAY_BUDGET_SECONDS / crawlDelaySeconds)));
|
|
34480
|
+
if (pageLimit < input.maxPages) warnings.push(`robots.txt asks for ${crawlDelaySeconds} seconds between requests, so the crawl stopped at ${pageLimit} pages to finish in about ${CRAWL_DELAY_BUDGET_SECONDS / 60} minutes.`);
|
|
34481
|
+
const rate = crawlDelaySeconds === null ? {
|
|
34482
|
+
concurrency: FETCH_CONCURRENCY,
|
|
34483
|
+
intervalCap: FETCH_CONCURRENCY,
|
|
34484
|
+
intervalMs: 1e3
|
|
34485
|
+
} : {
|
|
34486
|
+
concurrency: 1,
|
|
34487
|
+
intervalCap: 1,
|
|
34488
|
+
intervalMs: Math.ceil(crawlDelaySeconds * 1e3)
|
|
34489
|
+
};
|
|
34490
|
+
const options = {
|
|
34491
|
+
js: "auto",
|
|
34492
|
+
refresh: input.refresh,
|
|
34493
|
+
respectRobots: true,
|
|
34494
|
+
robotsResolver: robots.resolve,
|
|
34495
|
+
renderer,
|
|
34496
|
+
rate
|
|
34497
|
+
};
|
|
34498
|
+
const stats = (extra = {}) => ({
|
|
34499
|
+
maxPages: input.maxPages,
|
|
34500
|
+
pageLimit,
|
|
34501
|
+
urlsFound: 0,
|
|
34502
|
+
pagesCrawled: pages.length,
|
|
34503
|
+
capReached: false,
|
|
34504
|
+
skipped,
|
|
34505
|
+
crawlDelaySeconds,
|
|
34506
|
+
blockedByRobots: false,
|
|
34507
|
+
redirectsChecked: 0,
|
|
34508
|
+
...extra
|
|
34214
34509
|
});
|
|
34215
|
-
|
|
34216
|
-
|
|
34217
|
-
|
|
34218
|
-
|
|
34219
|
-
|
|
34220
|
-
|
|
34221
|
-
|
|
34222
|
-
|
|
34223
|
-
|
|
34224
|
-
|
|
34225
|
-
|
|
34226
|
-
|
|
34227
|
-
|
|
34228
|
-
|
|
34229
|
-
|
|
34230
|
-
|
|
34231
|
-
|
|
34510
|
+
if (rootRobots.allowed === false) return {
|
|
34511
|
+
origin,
|
|
34512
|
+
method: "links",
|
|
34513
|
+
pages,
|
|
34514
|
+
urls,
|
|
34515
|
+
pageOfUrl,
|
|
34516
|
+
index,
|
|
34517
|
+
sentences,
|
|
34518
|
+
genericAnchors,
|
|
34519
|
+
stats: stats({ blockedByRobots: true }),
|
|
34520
|
+
warnings: [`robots.txt does not allow this crawler to read ${start}.`]
|
|
34521
|
+
};
|
|
34522
|
+
const declared = await robots.sitemapUrls(origin);
|
|
34523
|
+
const sitemap = await sitemapSeeds({
|
|
34524
|
+
url: origin,
|
|
34525
|
+
maxPages: pageLimit * 2,
|
|
34526
|
+
declaredSitemapUrls: declared,
|
|
34232
34527
|
warnings,
|
|
34233
|
-
|
|
34528
|
+
fetchSitemapUrls
|
|
34529
|
+
});
|
|
34530
|
+
const seeds = sitemap.urls.map((url) => crawlUrl(url)).filter((url) => Boolean(url) && sameSite(url) && !ASSET.test(new URL(url).pathname));
|
|
34531
|
+
const method = seeds.length ? "sitemap" : "links";
|
|
34532
|
+
const queue = [];
|
|
34533
|
+
const queued = /* @__PURE__ */ new Set();
|
|
34534
|
+
const maxQueue = pageLimit * 5;
|
|
34535
|
+
const enqueue = (url) => {
|
|
34536
|
+
if (!url || queued.has(url) || queued.size >= maxQueue) return;
|
|
34537
|
+
if (!sameSite(url) || ASSET.test(new URL(url).pathname)) return;
|
|
34538
|
+
queued.add(url);
|
|
34539
|
+
idOf(url);
|
|
34540
|
+
queue.push(url);
|
|
34541
|
+
};
|
|
34542
|
+
enqueue(input.firstUrl ? crawlUrl(input.firstUrl) : void 0);
|
|
34543
|
+
if (method === "sitemap") for (const url of seeds) enqueue(url);
|
|
34544
|
+
else enqueue(start);
|
|
34545
|
+
let overflow = 0;
|
|
34546
|
+
const crawlOne = async (url) => {
|
|
34547
|
+
let fetched;
|
|
34548
|
+
try {
|
|
34549
|
+
fetched = await fetchPage(url, options);
|
|
34550
|
+
} catch (error) {
|
|
34551
|
+
skip(error instanceof Error && error.name === "RobotsAccessError" ? "robots" : "fetch-failed");
|
|
34552
|
+
return;
|
|
34553
|
+
}
|
|
34554
|
+
const finalUrl = crawlUrl(fetched.finalUrl) ?? url;
|
|
34555
|
+
if (finalUrl !== url) aliases.set(idOf(url), idOf(finalUrl));
|
|
34556
|
+
if (!sameSite(finalUrl)) return skip("off-site");
|
|
34557
|
+
if (fetched.status < 200 || fetched.status > 299) return skip("http-error");
|
|
34558
|
+
if (!/html/i.test(fetched.headers["content-type"] ?? "text/html")) return skip("not-html");
|
|
34559
|
+
let page;
|
|
34560
|
+
try {
|
|
34561
|
+
page = readPage(fetched, finalUrl);
|
|
34562
|
+
} catch {
|
|
34563
|
+
return skip("fetch-failed");
|
|
34564
|
+
}
|
|
34565
|
+
const canonical = page.canonical ? crawlUrl(page.canonical, finalUrl) : void 0;
|
|
34566
|
+
if (canonical && sameSite(canonical) && normalizePageUrl(canonical) !== normalizePageUrl(finalUrl)) {
|
|
34567
|
+
aliases.set(idOf(finalUrl), idOf(canonical));
|
|
34568
|
+
enqueue(canonical);
|
|
34569
|
+
return skip("canonicalised");
|
|
34570
|
+
}
|
|
34571
|
+
const finalId = idOf(finalUrl);
|
|
34572
|
+
if (pageOfUrl.has(finalId)) return skip("duplicate");
|
|
34573
|
+
if (pages.length >= pageLimit) {
|
|
34574
|
+
overflow += 1;
|
|
34575
|
+
return;
|
|
34576
|
+
}
|
|
34577
|
+
const text = placementPage({
|
|
34234
34578
|
...page,
|
|
34235
|
-
url:
|
|
34579
|
+
url: finalUrl,
|
|
34236
34580
|
html: fetched.html
|
|
34237
|
-
})
|
|
34581
|
+
});
|
|
34582
|
+
const contextual = /* @__PURE__ */ new Set();
|
|
34583
|
+
const other = /* @__PURE__ */ new Set();
|
|
34584
|
+
const id = pages.length;
|
|
34585
|
+
const hasMain = /<(main|article)[\s>]|role=["']main["']/i.test(fetched.html);
|
|
34586
|
+
for (const link of page.links) {
|
|
34587
|
+
const href = crawlUrl(link.href, finalUrl);
|
|
34588
|
+
if (!href || !sameSite(href) || ASSET.test(new URL(href).pathname)) continue;
|
|
34589
|
+
const hrefId = idOf(href);
|
|
34590
|
+
if (link.location === "main-content" || !hasMain && link.location === "other") {
|
|
34591
|
+
contextual.add(hrefId);
|
|
34592
|
+
if (GENERIC_ANCHORS.test(link.text.trim()) && genericAnchors.length < GENERIC_ANCHOR_LIMIT) genericAnchors.push({
|
|
34593
|
+
source: id,
|
|
34594
|
+
href,
|
|
34595
|
+
text: link.text.trim()
|
|
34596
|
+
});
|
|
34597
|
+
} else other.add(hrefId);
|
|
34598
|
+
}
|
|
34599
|
+
const record = {
|
|
34600
|
+
url: finalUrl,
|
|
34601
|
+
title: text.title,
|
|
34602
|
+
h1: text.h1,
|
|
34603
|
+
description: text.description,
|
|
34604
|
+
headings: text.headings,
|
|
34605
|
+
excerpt: excerptOf(text.sentences),
|
|
34606
|
+
words: text.sentences.reduce((sum, sentence) => sum + sentence.text.split(/\s+/).length, 0),
|
|
34607
|
+
lang: page.lang?.toLowerCase().split(/[-_]/)[0] || null,
|
|
34608
|
+
noindex: page.noindex,
|
|
34609
|
+
contextual: [...contextual],
|
|
34610
|
+
other: [...other],
|
|
34611
|
+
text: {
|
|
34612
|
+
offset: 0,
|
|
34613
|
+
length: 0
|
|
34614
|
+
}
|
|
34615
|
+
};
|
|
34616
|
+
pageOfUrl.set(finalId, id);
|
|
34617
|
+
pages.push(record);
|
|
34618
|
+
index.addPage(termCounts(text));
|
|
34619
|
+
record.text = await sentences.write(linkableSentences(text.sentences));
|
|
34620
|
+
if (method === "links" && !page.nofollow) for (const link of page.links) enqueue(crawlUrl(link.href, finalUrl));
|
|
34621
|
+
if (pages.length % 50 === 0) input.progress?.(`Crawled ${pages.length} pages of up to ${pageLimit}.`);
|
|
34622
|
+
};
|
|
34623
|
+
const worker = async () => {
|
|
34624
|
+
while (pages.length < pageLimit) {
|
|
34625
|
+
const url = queue.shift();
|
|
34626
|
+
if (!url) {
|
|
34627
|
+
if (active === 0) return;
|
|
34628
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
34629
|
+
continue;
|
|
34630
|
+
}
|
|
34631
|
+
active += 1;
|
|
34632
|
+
try {
|
|
34633
|
+
await crawlOne(url);
|
|
34634
|
+
} finally {
|
|
34635
|
+
active -= 1;
|
|
34636
|
+
}
|
|
34637
|
+
}
|
|
34238
34638
|
};
|
|
34239
|
-
|
|
34240
|
-
|
|
34241
|
-
|
|
34242
|
-
|
|
34243
|
-
|
|
34244
|
-
|
|
34245
|
-
|
|
34246
|
-
|
|
34247
|
-
|
|
34248
|
-
|
|
34249
|
-
|
|
34250
|
-
|
|
34639
|
+
let active = 0;
|
|
34640
|
+
if (method === "links" && queue[0] === start) {
|
|
34641
|
+
queue.shift();
|
|
34642
|
+
active += 1;
|
|
34643
|
+
await crawlOne(start).finally(() => {
|
|
34644
|
+
active -= 1;
|
|
34645
|
+
});
|
|
34646
|
+
const first = pages[0];
|
|
34647
|
+
if (first) origin = new URL(first.url).origin;
|
|
34648
|
+
}
|
|
34649
|
+
await Promise.all(Array.from({ length: rate.concurrency ?? 1 }, () => worker()));
|
|
34650
|
+
const capReached = pages.length >= pageLimit && (queue.length > 0 || overflow > 0);
|
|
34651
|
+
const linkedCounts = /* @__PURE__ */ new Map();
|
|
34652
|
+
for (const page of pages) for (const id of [...page.contextual, ...page.other]) if (!pageOfUrl.has(id) && !aliases.has(id)) linkedCounts.set(id, (linkedCounts.get(id) ?? 0) + 1);
|
|
34653
|
+
const toCheck = [...linkedCounts].sort((left, right) => right[1] - left[1] || left[0] - right[0]).slice(0, REDIRECT_CHECKS);
|
|
34654
|
+
let redirectsChecked = 0;
|
|
34655
|
+
await Promise.all(Array.from({ length: rate.concurrency ?? 1 }, async () => {
|
|
34656
|
+
while (toCheck.length) {
|
|
34657
|
+
const next = toCheck.shift();
|
|
34658
|
+
if (!next) break;
|
|
34659
|
+
const url = urls[next[0]];
|
|
34660
|
+
try {
|
|
34661
|
+
const fetched = await fetchPage(url, {
|
|
34662
|
+
...options,
|
|
34663
|
+
js: "off"
|
|
34664
|
+
});
|
|
34665
|
+
redirectsChecked += 1;
|
|
34666
|
+
const finalUrl = crawlUrl(fetched.finalUrl);
|
|
34667
|
+
if (finalUrl && finalUrl !== url) aliases.set(next[0], idOf(finalUrl));
|
|
34668
|
+
} catch {}
|
|
34251
34669
|
}
|
|
34670
|
+
}));
|
|
34671
|
+
for (let id = 0; id < urls.length; id += 1) {
|
|
34672
|
+
if (pageOfUrl.has(id)) continue;
|
|
34673
|
+
const seen = /* @__PURE__ */ new Set();
|
|
34674
|
+
let current = id;
|
|
34675
|
+
while (current !== void 0 && !seen.has(current)) {
|
|
34676
|
+
seen.add(current);
|
|
34677
|
+
const page = pageOfUrl.get(current);
|
|
34678
|
+
if (page !== void 0) {
|
|
34679
|
+
pageOfUrl.set(id, page);
|
|
34680
|
+
break;
|
|
34681
|
+
}
|
|
34682
|
+
current = aliases.get(current);
|
|
34683
|
+
}
|
|
34684
|
+
}
|
|
34685
|
+
index.finish();
|
|
34686
|
+
return {
|
|
34687
|
+
origin,
|
|
34688
|
+
method,
|
|
34689
|
+
pages,
|
|
34690
|
+
urls,
|
|
34691
|
+
pageOfUrl,
|
|
34692
|
+
index,
|
|
34693
|
+
sentences,
|
|
34694
|
+
genericAnchors,
|
|
34695
|
+
stats: stats({
|
|
34696
|
+
urlsFound: Math.max(queued.size, sitemap.discovery.urlsReturned),
|
|
34697
|
+
capReached,
|
|
34698
|
+
redirectsChecked
|
|
34699
|
+
}),
|
|
34700
|
+
warnings
|
|
34252
34701
|
};
|
|
34253
34702
|
}
|
|
34254
|
-
|
|
34255
|
-
|
|
34256
|
-
|
|
34257
|
-
|
|
34258
|
-
|
|
34259
|
-
|
|
34260
|
-
|
|
34261
|
-
|
|
34262
|
-
|
|
34263
|
-
|
|
34264
|
-
|
|
34265
|
-
|
|
34703
|
+
//#endregion
|
|
34704
|
+
//#region packages/core/dist/analyze/internal-links-judge.js
|
|
34705
|
+
const PAGE_TYPES = {
|
|
34706
|
+
pillar_guide: "A long, broad guide that covers a whole topic and would naturally link out to many narrower pages",
|
|
34707
|
+
support_article: "A focused article, post, tutorial, or explainer about one narrower subject",
|
|
34708
|
+
product_or_service: "Sells or describes a product, service, plan, pricing, demo, or signup: a page that makes money",
|
|
34709
|
+
comparison_or_review: "Compares options, reviews one, or lists the best of something to help a reader choose",
|
|
34710
|
+
glossary_or_definition: "A short page that defines one term or answers one 'what is' question",
|
|
34711
|
+
tool_or_calculator: "An interactive tool, calculator, generator, or checker the reader uses on the page",
|
|
34712
|
+
news_or_update: "A dated announcement, release note, changelog entry, or news item that will go stale",
|
|
34713
|
+
category_hub: "A listing, archive, tag, category, or index page that mostly lists other pages",
|
|
34714
|
+
home: "The site's home page",
|
|
34715
|
+
about_or_company: "About, team, contact, press, careers, or other company information",
|
|
34716
|
+
legal_or_utility: "Privacy, terms, cookies, login, search, 404, thank-you, or other utility pages readers do not seek out",
|
|
34717
|
+
other: null
|
|
34718
|
+
};
|
|
34719
|
+
const SEARCH_INTENTS = {
|
|
34720
|
+
informational: "The reader wants to learn or understand something",
|
|
34721
|
+
commercial: "The reader is comparing or evaluating options before buying",
|
|
34722
|
+
transactional: "The reader wants to buy, sign up, download, or complete an action now",
|
|
34723
|
+
navigational: "The reader wants to reach a specific page or part of the site"
|
|
34724
|
+
};
|
|
34725
|
+
const LINK_ROLES = {
|
|
34726
|
+
explains_concept: "The target explains a term or idea the source mentions",
|
|
34727
|
+
deeper_detail: "The target goes deeper into a sub-topic the source only touches on",
|
|
34728
|
+
broader_guide: "The target is the broader guide or hub the source belongs under",
|
|
34729
|
+
next_step: "The target is what a reader should read or do after this part",
|
|
34730
|
+
product_or_service: "The target is the product or service that solves the problem discussed",
|
|
34731
|
+
comparison_or_alternative: "The target compares or offers an alternative to what the source discusses",
|
|
34732
|
+
weak_or_none: "No clear reason a reader would want the target here"
|
|
34733
|
+
};
|
|
34734
|
+
const THRESHOLDS = {
|
|
34735
|
+
possible: .5,
|
|
34736
|
+
strong: .75
|
|
34737
|
+
};
|
|
34738
|
+
const FLOORS = {
|
|
34739
|
+
minSentenceMatch: .1,
|
|
34740
|
+
minSimilarityAlone: .15
|
|
34741
|
+
};
|
|
34742
|
+
function targetWeight(type) {
|
|
34743
|
+
if (type === "home" || type === "legal_or_utility" || type === "about_or_company" || type === "category_hub") return 0;
|
|
34744
|
+
return type === "news_or_update" ? .6 : 1;
|
|
34745
|
+
}
|
|
34746
|
+
function canBeSource(type) {
|
|
34747
|
+
return type !== "home" && type !== "category_hub" && type !== "legal_or_utility";
|
|
34748
|
+
}
|
|
34749
|
+
function classifyRequest(pages) {
|
|
34750
|
+
const questions = {};
|
|
34751
|
+
pages.forEach((_, index) => {
|
|
34752
|
+
questions[`type_${index}`] = {
|
|
34753
|
+
type: "choice",
|
|
34754
|
+
instructions: `What kind of page is \`pages[${index}]\`?`,
|
|
34755
|
+
criteria: PAGE_TYPES
|
|
34266
34756
|
};
|
|
34267
|
-
|
|
34268
|
-
|
|
34269
|
-
|
|
34270
|
-
|
|
34271
|
-
js: "auto",
|
|
34272
|
-
refresh: input.refresh
|
|
34273
|
-
});
|
|
34274
|
-
} catch (error) {
|
|
34275
|
-
return {
|
|
34276
|
-
exclusion: "failed",
|
|
34277
|
-
warnings: [{
|
|
34278
|
-
stage: "source-fetch",
|
|
34279
|
-
url: input.candidate.sourceUrl,
|
|
34280
|
-
code: "fetch-failed",
|
|
34281
|
-
message: errorMessage(error)
|
|
34282
|
-
}]
|
|
34757
|
+
questions[`intent_${index}`] = {
|
|
34758
|
+
type: "choice",
|
|
34759
|
+
instructions: `What does a visitor arriving at \`pages[${index}]\` from a search engine most likely want?`,
|
|
34760
|
+
criteria: SEARCH_INTENTS
|
|
34283
34761
|
};
|
|
34284
|
-
}
|
|
34285
|
-
|
|
34286
|
-
|
|
34287
|
-
|
|
34288
|
-
|
|
34289
|
-
|
|
34290
|
-
|
|
34291
|
-
|
|
34292
|
-
|
|
34293
|
-
|
|
34294
|
-
|
|
34295
|
-
|
|
34296
|
-
|
|
34762
|
+
});
|
|
34763
|
+
return {
|
|
34764
|
+
state: { pages: pages.map((page) => ({
|
|
34765
|
+
url: page.url,
|
|
34766
|
+
title: page.title,
|
|
34767
|
+
description: page.description,
|
|
34768
|
+
h1: page.h1,
|
|
34769
|
+
headings: page.headings,
|
|
34770
|
+
excerpt: page.excerpt,
|
|
34771
|
+
word_count: page.words
|
|
34772
|
+
})) },
|
|
34773
|
+
questions
|
|
34774
|
+
};
|
|
34775
|
+
}
|
|
34776
|
+
function choiceOf(answer) {
|
|
34777
|
+
return answer?.type === "choice" ? answer.choice : null;
|
|
34778
|
+
}
|
|
34779
|
+
function readClassification(answers, index) {
|
|
34780
|
+
const type = choiceOf(answers[`type_${index}`]);
|
|
34781
|
+
const intent = choiceOf(answers[`intent_${index}`]);
|
|
34782
|
+
return {
|
|
34783
|
+
type: type && Object.hasOwn(PAGE_TYPES, type) ? type : null,
|
|
34784
|
+
intent: intent && Object.hasOwn(SEARCH_INTENTS, intent) ? intent : null
|
|
34785
|
+
};
|
|
34786
|
+
}
|
|
34787
|
+
function judgeRequest(source, targets) {
|
|
34788
|
+
const sentences = /* @__PURE__ */ new Map();
|
|
34789
|
+
for (const target of targets) for (const sentence of target.placement.sentences) sentences.set(sentence.id, {
|
|
34790
|
+
id: sentence.id,
|
|
34791
|
+
under_heading: sentence.section,
|
|
34792
|
+
text: sentence.text
|
|
34793
|
+
});
|
|
34794
|
+
const questions = {};
|
|
34795
|
+
targets.forEach((target, index) => {
|
|
34796
|
+
const ref = `\`targets[${index}]\``;
|
|
34797
|
+
questions[`link_${index}`] = {
|
|
34798
|
+
type: "noul",
|
|
34799
|
+
instructions: `Should the page \`source\` add a link from within its body text to the page ${ref}?`,
|
|
34800
|
+
criteria: {
|
|
34801
|
+
true: "A reader of the source would gain something by reading the target here; the two pages cover related subjects and the link would feel natural and helpful",
|
|
34802
|
+
false: "The pages are only loosely related, the target adds nothing for this reader, or the link would feel forced or spammy"
|
|
34803
|
+
}
|
|
34804
|
+
};
|
|
34805
|
+
questions[`where_${index}`] = {
|
|
34806
|
+
type: "choice",
|
|
34807
|
+
instructions: `Which sentence in \`source.sentences\` is the most natural place to add a link to ${ref}? Choose "none" if no listed sentence leads a reader towards that page.`,
|
|
34808
|
+
criteria: {
|
|
34809
|
+
...Object.fromEntries(target.placement.sentences.map((sentence) => [sentence.id, sentence.text])),
|
|
34810
|
+
none: "No listed sentence is a natural place for this link"
|
|
34811
|
+
}
|
|
34812
|
+
};
|
|
34813
|
+
if (target.placement.anchors.length) questions[`anchor_${index}`] = {
|
|
34814
|
+
type: "choice",
|
|
34815
|
+
instructions: `Which phrase should become the clickable link text for a link from \`source\` to ${ref}? Prefer a phrase that tells the reader what the target page is about and reads naturally as a link.`,
|
|
34816
|
+
criteria: {
|
|
34817
|
+
...Object.fromEntries(target.placement.anchors.map((anchor, at) => [`A${at + 1}`, {
|
|
34818
|
+
text: anchor.text,
|
|
34819
|
+
in_sentence: anchor.sentence
|
|
34820
|
+
}])),
|
|
34821
|
+
none: "None of these phrases would make a good link"
|
|
34822
|
+
}
|
|
34823
|
+
};
|
|
34824
|
+
questions[`role_${index}`] = {
|
|
34825
|
+
type: "choice",
|
|
34826
|
+
instructions: `If \`source\` linked to ${ref}, what would the link do for the reader?`,
|
|
34827
|
+
criteria: LINK_ROLES
|
|
34297
34828
|
};
|
|
34298
|
-
}
|
|
34299
|
-
const warnings = page.warnings.map((message) => ({
|
|
34300
|
-
stage: "source-extract",
|
|
34301
|
-
url: input.candidate.sourceUrl,
|
|
34302
|
-
code: "extractor-warning",
|
|
34303
|
-
message
|
|
34304
|
-
}));
|
|
34305
|
-
if (input.target.aliases.some((alias) => sameObservedUrl(page.finalUrl, alias))) return {
|
|
34306
|
-
exclusion: "self-alias",
|
|
34307
|
-
warnings
|
|
34308
|
-
};
|
|
34309
|
-
const technicalSignals = pageTechnicalSignals({
|
|
34310
|
-
url: input.candidate.sourceUrl,
|
|
34311
|
-
page,
|
|
34312
|
-
fetchDiagnostics: fetched.diagnostics,
|
|
34313
|
-
httpStatus: fetched.status
|
|
34314
34829
|
});
|
|
34315
|
-
if (technicalSignals.length) return {
|
|
34316
|
-
exclusion: "technical",
|
|
34317
|
-
warnings
|
|
34318
|
-
};
|
|
34319
|
-
const observed = linksToAliases(page, input.target.aliases).map((link) => ({
|
|
34320
|
-
href: link.href,
|
|
34321
|
-
text: link.text,
|
|
34322
|
-
rel: link.rel,
|
|
34323
|
-
location: link.location
|
|
34324
|
-
}));
|
|
34325
|
-
const contextual = observed.filter((link) => link.location === "main-content");
|
|
34326
|
-
if (contextual.some((link) => sameObservedUrl(link.href, input.target.preferredUrl))) return {
|
|
34327
|
-
exclusion: "existing-link",
|
|
34328
|
-
warnings
|
|
34329
|
-
};
|
|
34330
|
-
const aliasContextual = contextual.length > 0;
|
|
34331
|
-
const linkStatus = aliasContextual ? "alias-contextual" : observed.length ? "non-contextual-only" : "missing";
|
|
34332
|
-
const confidence = input.candidate.bestMatchKind === "exact-query" && !page.contentExtraction.fallback ? "medium" : "low";
|
|
34333
|
-
const actionType = aliasContextual ? "review-alias-link" : "review-contextual-link";
|
|
34334
|
-
const evidenceRef = aliasContextual ? `A main-content link points to a non-preferred target alias; the preferred target is ${input.target.preferredUrl}.` : `${input.candidate.sourceUrl} matched ${input.candidate.matchedQueries} retained GSC query row${input.candidate.matchedQueries === 1 ? "" : "s"}, and no main-content link to the preferred target was observed in the fetched HTML.`;
|
|
34335
|
-
const action = aliasContextual ? `Review the existing contextual link and update it to the preferred target ${input.target.preferredUrl} if the redirect or old alias is not intentional.` : `Review whether this source page and ${input.target.preferredUrl} serve complementary intent. If the link would help a reader, add a natural main-content link without forcing exact-match anchor text.`;
|
|
34336
34830
|
return {
|
|
34337
|
-
|
|
34338
|
-
|
|
34339
|
-
|
|
34340
|
-
|
|
34341
|
-
|
|
34342
|
-
|
|
34343
|
-
pageWarnings: page.warnings,
|
|
34344
|
-
actionType,
|
|
34345
|
-
linkEvidence: {
|
|
34346
|
-
status: linkStatus,
|
|
34347
|
-
observedCount: observed.length,
|
|
34348
|
-
observedLimit: 20,
|
|
34349
|
-
limitedCount: Math.max(0, observed.length - 20),
|
|
34350
|
-
observed: observed.slice(0, 20)
|
|
34831
|
+
state: {
|
|
34832
|
+
source: {
|
|
34833
|
+
url: source.url,
|
|
34834
|
+
title: source.title,
|
|
34835
|
+
page_type: source.type ?? "unknown",
|
|
34836
|
+
sentences: [...sentences.values()].sort((left, right) => Number(left.id.slice(1)) - Number(right.id.slice(1)))
|
|
34351
34837
|
},
|
|
34352
|
-
|
|
34353
|
-
|
|
34354
|
-
|
|
34355
|
-
|
|
34356
|
-
|
|
34357
|
-
|
|
34358
|
-
|
|
34359
|
-
|
|
34360
|
-
}
|
|
34838
|
+
targets: targets.map((target) => ({
|
|
34839
|
+
url: target.url,
|
|
34840
|
+
title: target.title,
|
|
34841
|
+
description: target.description,
|
|
34842
|
+
h1: target.h1,
|
|
34843
|
+
page_type: target.type ?? "unknown",
|
|
34844
|
+
search_intent: target.intent ?? "unknown",
|
|
34845
|
+
opening: target.opening
|
|
34846
|
+
}))
|
|
34361
34847
|
},
|
|
34362
|
-
|
|
34363
|
-
|
|
34364
|
-
|
|
34365
|
-
|
|
34366
|
-
|
|
34367
|
-
|
|
34848
|
+
questions
|
|
34849
|
+
};
|
|
34850
|
+
}
|
|
34851
|
+
function floor(sentence, similarity, probability) {
|
|
34852
|
+
if (sentence && sentence.match < FLOORS.minSentenceMatch) return {
|
|
34853
|
+
state: "held-back",
|
|
34854
|
+
reason: "weak-sentence",
|
|
34855
|
+
detail: `the sentence has ${Math.round(sentence.match * 100)}% of the words that name the page; a link needs ${Math.round(FLOORS.minSentenceMatch * 100)}%`,
|
|
34856
|
+
probability
|
|
34857
|
+
};
|
|
34858
|
+
if (!sentence && similarity < FLOORS.minSimilarityAlone) return {
|
|
34859
|
+
state: "held-back",
|
|
34860
|
+
reason: "no-sentence",
|
|
34861
|
+
detail: `no sentence to place it in, and the pages are ${similarity.toFixed(2)} similar; a link without a sentence needs ${FLOORS.minSimilarityAlone}`,
|
|
34862
|
+
probability
|
|
34863
|
+
};
|
|
34864
|
+
return null;
|
|
34865
|
+
}
|
|
34866
|
+
function decideWithModel(placement, similarity, answers, index) {
|
|
34867
|
+
const link = answers[`link_${index}`];
|
|
34868
|
+
const role = choiceOf(answers[`role_${index}`]);
|
|
34869
|
+
if (link?.type !== "noul" || !role) return {
|
|
34870
|
+
state: "declined",
|
|
34871
|
+
reason: "no-answer",
|
|
34872
|
+
probability: null
|
|
34873
|
+
};
|
|
34874
|
+
const probability = Math.round(link.noul * 1e3) / 1e3;
|
|
34875
|
+
if (probability < THRESHOLDS.possible) return {
|
|
34876
|
+
state: "declined",
|
|
34877
|
+
reason: "below-threshold",
|
|
34878
|
+
probability
|
|
34879
|
+
};
|
|
34880
|
+
if (role === "weak_or_none" || !Object.hasOwn(LINK_ROLES, role)) return {
|
|
34881
|
+
state: "declined",
|
|
34882
|
+
reason: "weak-role",
|
|
34883
|
+
probability
|
|
34884
|
+
};
|
|
34885
|
+
const where = choiceOf(answers[`where_${index}`]);
|
|
34886
|
+
const sentence = placement.sentences.find((option) => option.id === where) ?? null;
|
|
34887
|
+
const held = floor(sentence, similarity, probability);
|
|
34888
|
+
if (held) return held;
|
|
34889
|
+
let anchor = null;
|
|
34890
|
+
const picked = answers[`anchor_${index}`];
|
|
34891
|
+
if (sentence && picked?.type === "choice" && picked.choice !== "none") {
|
|
34892
|
+
const inSentence = placement.anchors.map((option, at) => ({
|
|
34893
|
+
...option,
|
|
34894
|
+
key: `A${at + 1}`
|
|
34895
|
+
})).filter((option) => option.sentence === sentence.id);
|
|
34896
|
+
anchor = (inSentence.find((option) => option.key === picked.choice) ?? [...inSentence].sort((left, right) => (picked.probabilities[right.key] ?? 0) - (picked.probabilities[left.key] ?? 0) || left.key.localeCompare(right.key))[0])?.text ?? null;
|
|
34897
|
+
}
|
|
34898
|
+
return {
|
|
34899
|
+
state: "suggest",
|
|
34900
|
+
tier: probability >= THRESHOLDS.strong ? "strong" : "possible",
|
|
34901
|
+
probability,
|
|
34902
|
+
role,
|
|
34903
|
+
sentence: sentence ? {
|
|
34904
|
+
text: sentence.text,
|
|
34905
|
+
section: sentence.section,
|
|
34906
|
+
match: sentence.match
|
|
34907
|
+
} : null,
|
|
34908
|
+
anchor
|
|
34909
|
+
};
|
|
34910
|
+
}
|
|
34911
|
+
function decideWithoutModel(placement, similarity) {
|
|
34912
|
+
const sentence = placement.sentences[0] ?? null;
|
|
34913
|
+
const held = floor(sentence, similarity, null);
|
|
34914
|
+
if (held) return held;
|
|
34915
|
+
return {
|
|
34916
|
+
state: "suggest",
|
|
34917
|
+
tier: "heuristic",
|
|
34918
|
+
probability: null,
|
|
34919
|
+
role: null,
|
|
34920
|
+
sentence: sentence ? {
|
|
34921
|
+
text: sentence.text,
|
|
34922
|
+
section: sentence.section,
|
|
34923
|
+
match: sentence.match
|
|
34924
|
+
} : null,
|
|
34925
|
+
anchor: placement.anchors.find((option) => option.sentence === sentence?.id)?.text ?? null
|
|
34368
34926
|
};
|
|
34369
34927
|
}
|
|
34370
34928
|
//#endregion
|
|
34929
|
+
//#region packages/core/dist/analyze/internal-links-report.js
|
|
34930
|
+
const DAY$1 = 864e5;
|
|
34931
|
+
const ISO_DAY = /^\d{4}-\d{2}-\d{2}$/;
|
|
34932
|
+
function internalLinksReportRange(input, fallbackDays, now) {
|
|
34933
|
+
if (!input.startDate && !input.endDate) return {
|
|
34934
|
+
...finalGscDateRange(fallbackDays, now),
|
|
34935
|
+
days: fallbackDays
|
|
34936
|
+
};
|
|
34937
|
+
const { startDate, endDate } = input;
|
|
34938
|
+
if (!startDate || !endDate || !ISO_DAY.test(startDate) || !ISO_DAY.test(endDate) || startDate > endDate) throw new SeoError("INVALID_INPUT", "Send a start date and an end date together, in order, as YYYY-MM-DD.");
|
|
34939
|
+
return {
|
|
34940
|
+
startDate,
|
|
34941
|
+
endDate,
|
|
34942
|
+
days: Math.round((Date.parse(`${endDate}T00:00:00Z`) - Date.parse(`${startDate}T00:00:00Z`)) / DAY$1) + 1
|
|
34943
|
+
};
|
|
34944
|
+
}
|
|
34945
|
+
const plural$2 = (count, word) => `${count} ${word}${count === 1 ? "" : "s"}`;
|
|
34946
|
+
function internalLinksVerdict(report) {
|
|
34947
|
+
const { counts } = report;
|
|
34948
|
+
if (report.dataStatus === "blocked") return "robots.txt does not allow this crawler to read the site, so nothing was checked.";
|
|
34949
|
+
if (report.dataStatus === "empty") return "No HTML page could be crawled, so there is nothing to link.";
|
|
34950
|
+
if (report.dataStatus === "target-not-crawled") return "The target page could not be crawled as its own page (it failed, left the site, or names another page as canonical). Check it with seo page first.";
|
|
34951
|
+
if (report.dataStatus === "target-not-linkable") return `The target is ${report.target?.noindex ? "noindex" : `a ${report.target?.type?.replaceAll("_", " ")} page`}, so no links to it are suggested.`;
|
|
34952
|
+
const judged = report.judge.status === "used" || report.judge.status === "partial" ? `${counts.strong} strong and ${counts.possible} possible link${counts.strong + counts.possible === 1 ? "" : "s"} judged by ${report.judge.model}${counts.heuristic ? `, plus ${plural$2(counts.heuristic, "heuristic link")} where a request failed` : ""}` : `${plural$2(counts.heuristic, "heuristic link")} (no decision model)`;
|
|
34953
|
+
const scope = report.mode === "url" ? `to ${report.target?.url}` : `across ${plural$2(counts.pagesCrawled, "page")}`;
|
|
34954
|
+
const capped = report.crawl.capReached ? ` The crawl stopped at its limit of ${report.crawl.pageLimit} pages, so pages beyond it were not checked.` : "";
|
|
34955
|
+
return `${judged} ${scope}${counts.heldBack ? `; ${counts.heldBack} held back by the evidence floors` : ""}.${capped}`;
|
|
34956
|
+
}
|
|
34957
|
+
//#endregion
|
|
34371
34958
|
//#region packages/core/dist/analyze/internal-links.js
|
|
34959
|
+
const GSC_DAYS = 90;
|
|
34372
34960
|
const MAX_GSC_ROWS = 1e5;
|
|
34373
|
-
const
|
|
34374
|
-
const
|
|
34375
|
-
const
|
|
34376
|
-
const
|
|
34377
|
-
const
|
|
34378
|
-
|
|
34379
|
-
|
|
34380
|
-
|
|
34381
|
-
|
|
34382
|
-
|
|
34383
|
-
|
|
34384
|
-
|
|
34385
|
-
|
|
34386
|
-
const
|
|
34387
|
-
|
|
34388
|
-
|
|
34389
|
-
|
|
34390
|
-
|
|
34391
|
-
const
|
|
34392
|
-
|
|
34393
|
-
|
|
34394
|
-
|
|
34395
|
-
|
|
34396
|
-
|
|
34961
|
+
const MAX_PAGES_PER_QUERY = 25;
|
|
34962
|
+
const MAX_GSC_PAIRS = 5e5;
|
|
34963
|
+
const MIN_SIMILARITY = .08;
|
|
34964
|
+
const TARGETS_PER_SOURCE = 8;
|
|
34965
|
+
const EXTRA_GSC_TARGETS = 2;
|
|
34966
|
+
const URL_SOURCES_PLACED = 60;
|
|
34967
|
+
const URL_SOURCES_JUDGED = 24;
|
|
34968
|
+
const SOURCE_WINDOW = 64;
|
|
34969
|
+
const WEAK_INBOUND = 3;
|
|
34970
|
+
const EXAMPLE_LIMIT = 20;
|
|
34971
|
+
const LIST_LIMIT = 100;
|
|
34972
|
+
const OPENING_CHARS = 300;
|
|
34973
|
+
function siteStartUrl(site) {
|
|
34974
|
+
const value = site.trim();
|
|
34975
|
+
if (value.startsWith("sc-domain:")) return `https://${value.slice(10)}/`;
|
|
34976
|
+
return `${new URL(/^https?:\/\//i.test(value) ? value : `https://${value}`).origin}/`;
|
|
34977
|
+
}
|
|
34978
|
+
function round$1(value, places = 3) {
|
|
34979
|
+
const factor = 10 ** places;
|
|
34980
|
+
return Math.round(value * factor) / factor;
|
|
34981
|
+
}
|
|
34982
|
+
async function internalLinksReport(input, dependencies) {
|
|
34983
|
+
const deps = {
|
|
34984
|
+
crawl: crawlForLinks,
|
|
34985
|
+
now: () => /* @__PURE__ */ new Date(),
|
|
34986
|
+
...dependencies
|
|
34987
|
+
};
|
|
34988
|
+
const mode = input.targetUrl ? "url" : "site";
|
|
34989
|
+
if (mode === "url" && input.maxPages !== void 0) throw new SeoError("INVALID_INPUT", `--max-pages is for a site audit only. A page run crawls up to 500 pages.`);
|
|
34990
|
+
const maxPages = input.maxPages ?? 500;
|
|
34991
|
+
if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > 5e3) throw new SeoError("INVALID_INPUT", `--max-pages is a whole number from 1 to ${MAX_MAX_PAGES.toLocaleString("en-US")}.`);
|
|
34992
|
+
const targetUrl = input.targetUrl ? crawlUrl(input.targetUrl) : void 0;
|
|
34993
|
+
if (input.targetUrl && !targetUrl) throw new SeoError("INVALID_INPUT", "The target is an http or https URL.");
|
|
34994
|
+
const now = deps.now();
|
|
34995
|
+
const startUrl = targetUrl ? `${new URL(targetUrl).origin}/` : siteStartUrl(input.site);
|
|
34996
|
+
deps.progress?.(`Crawling ${startUrl} (up to ${maxPages} pages).`);
|
|
34997
|
+
const crawl = await deps.crawl({
|
|
34998
|
+
startUrl,
|
|
34999
|
+
maxPages,
|
|
35000
|
+
firstUrl: targetUrl,
|
|
34397
35001
|
refresh: input.refresh,
|
|
34398
|
-
|
|
34399
|
-
});
|
|
34400
|
-
const targetRequests = await Promise.all(target.aliases.map(async (alias) => {
|
|
34401
|
-
const result = await dependencies.searchAnalytics(input.site, {
|
|
34402
|
-
...range,
|
|
34403
|
-
dimensions: ["query", "page"],
|
|
34404
|
-
type: "web",
|
|
34405
|
-
dataState: "final",
|
|
34406
|
-
maxRows: MAX_GSC_ROWS,
|
|
34407
|
-
dimensionFilterGroups: [{
|
|
34408
|
-
groupType: "and",
|
|
34409
|
-
filters: [{
|
|
34410
|
-
dimension: "page",
|
|
34411
|
-
operator: "equals",
|
|
34412
|
-
expression: alias
|
|
34413
|
-
}]
|
|
34414
|
-
}]
|
|
34415
|
-
}, { refresh: input.refresh });
|
|
34416
|
-
ledger.addGsc(result.calls, result.rowsFetched);
|
|
34417
|
-
return {
|
|
34418
|
-
alias,
|
|
34419
|
-
result
|
|
34420
|
-
};
|
|
34421
|
-
}));
|
|
34422
|
-
const targetRows = targetRequests.flatMap(({ result }) => result.rows);
|
|
34423
|
-
const preliminary = analyzeInternalLinksFromRows({
|
|
34424
|
-
targetRows,
|
|
34425
|
-
sourceRows: [],
|
|
34426
|
-
site: input.site,
|
|
34427
|
-
targetAliases: target.aliases,
|
|
34428
|
-
minImpressions,
|
|
34429
|
-
brandTerms
|
|
35002
|
+
progress: deps.progress
|
|
34430
35003
|
});
|
|
34431
|
-
|
|
34432
|
-
|
|
34433
|
-
|
|
34434
|
-
|
|
34435
|
-
|
|
34436
|
-
|
|
34437
|
-
|
|
34438
|
-
}
|
|
34439
|
-
|
|
34440
|
-
|
|
34441
|
-
|
|
35004
|
+
try {
|
|
35005
|
+
return await analyse(input, deps, crawl, {
|
|
35006
|
+
mode,
|
|
35007
|
+
targetUrl,
|
|
35008
|
+
startUrl,
|
|
35009
|
+
now
|
|
35010
|
+
});
|
|
35011
|
+
} finally {
|
|
35012
|
+
await crawl.sentences.close();
|
|
35013
|
+
}
|
|
35014
|
+
}
|
|
35015
|
+
async function analyse(input, deps, crawl, context) {
|
|
35016
|
+
const { pages, index } = crawl;
|
|
35017
|
+
const count = pages.length;
|
|
35018
|
+
const warnings = [...crawl.warnings];
|
|
35019
|
+
const lookup = /* @__PURE__ */ new Map();
|
|
35020
|
+
crawl.urls.forEach((url, id) => {
|
|
35021
|
+
const page = crawl.pageOfUrl.get(id);
|
|
35022
|
+
if (page === void 0) return;
|
|
35023
|
+
lookup.set(url, page);
|
|
35024
|
+
if (!lookup.has(normalizePageUrl(url))) lookup.set(normalizePageUrl(url), page);
|
|
35025
|
+
});
|
|
35026
|
+
const pageAt = (url) => {
|
|
35027
|
+
const clean = crawlUrl(url);
|
|
35028
|
+
if (!clean) return void 0;
|
|
35029
|
+
return lookup.get(clean) ?? lookup.get(normalizePageUrl(clean));
|
|
35030
|
+
};
|
|
35031
|
+
const target = context.targetUrl !== void 0 ? pageAt(context.targetUrl) : void 0;
|
|
35032
|
+
const contextual = pages.map(() => /* @__PURE__ */ new Set());
|
|
35033
|
+
const anywhere = pages.map(() => /* @__PURE__ */ new Set());
|
|
35034
|
+
const redirectedLinks = [];
|
|
35035
|
+
pages.forEach((page, source) => {
|
|
35036
|
+
for (const id of page.contextual) {
|
|
35037
|
+
const to = crawl.pageOfUrl.get(id);
|
|
35038
|
+
if (to === void 0 || to === source) continue;
|
|
35039
|
+
contextual[source]?.add(to);
|
|
35040
|
+
anywhere[source]?.add(to);
|
|
35041
|
+
const via = crawl.urls[id];
|
|
35042
|
+
const final = pages[to]?.url;
|
|
35043
|
+
if (normalizePageUrl(via) !== normalizePageUrl(final) && redirectedLinks.length < LIST_LIMIT) redirectedLinks.push({
|
|
35044
|
+
source: page.url,
|
|
35045
|
+
via,
|
|
35046
|
+
target: final
|
|
35047
|
+
});
|
|
35048
|
+
}
|
|
35049
|
+
for (const id of page.other) {
|
|
35050
|
+
const to = crawl.pageOfUrl.get(id);
|
|
35051
|
+
if (to !== void 0 && to !== source) anywhere[source]?.add(to);
|
|
35052
|
+
}
|
|
35053
|
+
});
|
|
35054
|
+
const inbound = new Array(count).fill(0);
|
|
35055
|
+
const inboundAnywhere = new Array(count).fill(0);
|
|
35056
|
+
contextual.forEach((set) => {
|
|
35057
|
+
for (const to of set) inbound[to] = (inbound[to] ?? 0) + 1;
|
|
35058
|
+
});
|
|
35059
|
+
anywhere.forEach((set) => {
|
|
35060
|
+
for (const to of set) inboundAnywhere[to] = (inboundAnywhere[to] ?? 0) + 1;
|
|
35061
|
+
});
|
|
35062
|
+
const existingContextualLinks = contextual.reduce((sum, set) => sum + set.size, 0);
|
|
35063
|
+
const range = internalLinksReportRange(input, GSC_DAYS, context.now);
|
|
35064
|
+
const sharedQueries = /* @__PURE__ */ new Map();
|
|
35065
|
+
const pairKey = (source, target) => source * count + target;
|
|
35066
|
+
let searchConsole = {
|
|
35067
|
+
status: "not-available",
|
|
35068
|
+
property: null,
|
|
35069
|
+
range: null,
|
|
35070
|
+
rows: 0,
|
|
35071
|
+
possiblyTruncated: false,
|
|
35072
|
+
pairsWithSharedQueries: 0,
|
|
35073
|
+
note: "unavailable" in deps.searchConsole ? deps.searchConsole.unavailable : null
|
|
35074
|
+
};
|
|
35075
|
+
if ("property" in deps.searchConsole && count > 1) {
|
|
35076
|
+
const { startDate, endDate } = range;
|
|
35077
|
+
try {
|
|
35078
|
+
deps.progress?.("Reading Search Console queries.");
|
|
35079
|
+
const result = await deps.searchConsole.query({
|
|
35080
|
+
startDate,
|
|
35081
|
+
endDate,
|
|
35082
|
+
dimensions: ["query", "page"],
|
|
35083
|
+
type: "web",
|
|
35084
|
+
dataState: "final",
|
|
35085
|
+
maxRows: MAX_GSC_ROWS
|
|
35086
|
+
});
|
|
35087
|
+
const brandTerms = deriveBrandTerms({ siteUrl: crawl.origin });
|
|
35088
|
+
const byQuery = /* @__PURE__ */ new Map();
|
|
35089
|
+
for (const row of result.rows) {
|
|
35090
|
+
const [query, url] = row.keys;
|
|
35091
|
+
if (!query || !url || row.impressions < 1) continue;
|
|
35092
|
+
if (isBrandQuery(query, brandTerms)) continue;
|
|
35093
|
+
const page = pageAt(url);
|
|
35094
|
+
if (page === void 0) continue;
|
|
35095
|
+
let set = byQuery.get(query);
|
|
35096
|
+
if (!set) {
|
|
35097
|
+
set = /* @__PURE__ */ new Set();
|
|
35098
|
+
byQuery.set(query, set);
|
|
35099
|
+
}
|
|
35100
|
+
set.add(page);
|
|
35101
|
+
}
|
|
35102
|
+
let capped = false;
|
|
35103
|
+
for (const set of byQuery.values()) {
|
|
35104
|
+
if (set.size < 2 || set.size > MAX_PAGES_PER_QUERY) continue;
|
|
35105
|
+
const list = [...set];
|
|
35106
|
+
for (const source of list) for (const to of list) {
|
|
35107
|
+
if (source === to) continue;
|
|
35108
|
+
if (target !== void 0 && to !== target) continue;
|
|
35109
|
+
const key = pairKey(source, to);
|
|
35110
|
+
if (!sharedQueries.has(key) && sharedQueries.size >= MAX_GSC_PAIRS) {
|
|
35111
|
+
capped = true;
|
|
35112
|
+
continue;
|
|
35113
|
+
}
|
|
35114
|
+
sharedQueries.set(key, (sharedQueries.get(key) ?? 0) + 1);
|
|
35115
|
+
}
|
|
35116
|
+
}
|
|
35117
|
+
searchConsole = {
|
|
35118
|
+
status: "used",
|
|
35119
|
+
property: deps.searchConsole.property,
|
|
35120
|
+
range: {
|
|
35121
|
+
startDate,
|
|
35122
|
+
endDate
|
|
35123
|
+
},
|
|
35124
|
+
rows: result.rowsFetched,
|
|
35125
|
+
possiblyTruncated: result.rowsFetched >= MAX_GSC_ROWS,
|
|
35126
|
+
pairsWithSharedQueries: sharedQueries.size,
|
|
35127
|
+
note: capped ? `Shared queries were counted for the first ${MAX_GSC_PAIRS.toLocaleString("en-US")} page pairs.` : null
|
|
35128
|
+
};
|
|
35129
|
+
} catch (error) {
|
|
35130
|
+
searchConsole = {
|
|
35131
|
+
...searchConsole,
|
|
35132
|
+
status: "failed",
|
|
35133
|
+
property: deps.searchConsole.property,
|
|
35134
|
+
note: error instanceof Error ? error.message : String(error)
|
|
35135
|
+
};
|
|
35136
|
+
}
|
|
35137
|
+
}
|
|
35138
|
+
const runner = "model" in deps.decisionModel ? new DecisionRunner(deps.decisionModel.model) : null;
|
|
35139
|
+
const types = pages.map((page) => new URL(page.url).pathname === "/" ? "home" : null);
|
|
35140
|
+
const intents = pages.map(() => null);
|
|
35141
|
+
let pagesClassified = 0;
|
|
35142
|
+
const classify = async (ids) => {
|
|
35143
|
+
if (!runner) return;
|
|
35144
|
+
const batches = [];
|
|
35145
|
+
for (let at = 0; at < ids.length; at += 5) batches.push(ids.slice(at, at + 5));
|
|
35146
|
+
deps.progress?.(`Classifying ${ids.length} pages in ${batches.length} requests.`);
|
|
35147
|
+
await Promise.all(batches.map(async (batch) => {
|
|
35148
|
+
const response = await runner.ask(classifyRequest(batch.map((id) => pages[id])));
|
|
35149
|
+
if (!response) return;
|
|
35150
|
+
batch.forEach((id, at) => {
|
|
35151
|
+
const read = readClassification(response.answers, at);
|
|
35152
|
+
if (read.type) types[id] = read.type;
|
|
35153
|
+
intents[id] = read.intent;
|
|
35154
|
+
pagesClassified += 1;
|
|
35155
|
+
});
|
|
35156
|
+
}));
|
|
34442
35157
|
};
|
|
34443
|
-
|
|
34444
|
-
|
|
34445
|
-
|
|
34446
|
-
|
|
34447
|
-
|
|
34448
|
-
|
|
34449
|
-
|
|
34450
|
-
|
|
34451
|
-
}
|
|
34452
|
-
const
|
|
34453
|
-
|
|
34454
|
-
|
|
34455
|
-
|
|
34456
|
-
|
|
34457
|
-
|
|
34458
|
-
|
|
34459
|
-
|
|
34460
|
-
|
|
34461
|
-
|
|
34462
|
-
|
|
34463
|
-
|
|
34464
|
-
|
|
34465
|
-
|
|
35158
|
+
const linkable = (source, to) => {
|
|
35159
|
+
const from = pages[source];
|
|
35160
|
+
const page = pages[to];
|
|
35161
|
+
if (!from || !page || source === to) return false;
|
|
35162
|
+
if (page.noindex || targetWeight(types[to] ?? null) === 0) return false;
|
|
35163
|
+
if (contextual[source]?.has(to)) return false;
|
|
35164
|
+
if (from.lang && page.lang && from.lang !== page.lang) return false;
|
|
35165
|
+
return canBeSource(types[source] ?? null);
|
|
35166
|
+
};
|
|
35167
|
+
const adjusted = (to, raw) => raw * ((inbound[to] ?? 0) < WEAK_INBOUND ? 1.25 : 1) * targetWeight(types[to] ?? null);
|
|
35168
|
+
let status = "complete";
|
|
35169
|
+
const candidatesBySource = /* @__PURE__ */ new Map();
|
|
35170
|
+
if (crawl.stats.blockedByRobots) status = "blocked";
|
|
35171
|
+
else if (count === 0) status = "empty";
|
|
35172
|
+
else if (context.mode === "url" && target === void 0) status = "target-not-crawled";
|
|
35173
|
+
if (status === "complete" && context.mode === "site") {
|
|
35174
|
+
await classify(pages.map((_, id) => id));
|
|
35175
|
+
const extra = /* @__PURE__ */ new Map();
|
|
35176
|
+
for (const key of sharedQueries.keys()) {
|
|
35177
|
+
const source = Math.floor(key / count);
|
|
35178
|
+
const list = extra.get(source) ?? [];
|
|
35179
|
+
list.push(key % count);
|
|
35180
|
+
extra.set(source, list);
|
|
35181
|
+
}
|
|
35182
|
+
for (let source = 0; source < count; source += 1) {
|
|
35183
|
+
if (!canBeSource(types[source] ?? null)) continue;
|
|
35184
|
+
const chosen = index.neighbours(source, MIN_SIMILARITY).filter(({ page }) => linkable(source, page)).map(({ page, score }) => ({
|
|
35185
|
+
source,
|
|
35186
|
+
target: page,
|
|
35187
|
+
raw: score,
|
|
35188
|
+
similarity: adjusted(page, score),
|
|
35189
|
+
sharedQueries: sharedQueries.get(pairKey(source, page)) ?? 0
|
|
35190
|
+
})).sort((left, right) => right.similarity - left.similarity || left.target - right.target).slice(0, TARGETS_PER_SOURCE);
|
|
35191
|
+
const taken = new Set(chosen.map((candidate) => candidate.target));
|
|
35192
|
+
const fromQueries = (extra.get(source) ?? []).filter((to) => !taken.has(to) && linkable(source, to)).map((to) => {
|
|
35193
|
+
const raw = index.similarity(source, to);
|
|
35194
|
+
return {
|
|
35195
|
+
source,
|
|
35196
|
+
target: to,
|
|
35197
|
+
raw,
|
|
35198
|
+
similarity: adjusted(to, raw),
|
|
35199
|
+
sharedQueries: sharedQueries.get(pairKey(source, to)) ?? 0
|
|
35200
|
+
};
|
|
35201
|
+
}).sort((left, right) => right.sharedQueries - left.sharedQueries || right.similarity - left.similarity || left.target - right.target).slice(0, EXTRA_GSC_TARGETS);
|
|
35202
|
+
const all = [...chosen, ...fromQueries];
|
|
35203
|
+
if (all.length) candidatesBySource.set(source, all);
|
|
35204
|
+
}
|
|
35205
|
+
}
|
|
35206
|
+
if (status === "complete" && context.mode === "url" && target !== void 0) {
|
|
35207
|
+
const near = /* @__PURE__ */ new Map();
|
|
35208
|
+
for (const { page, score } of index.neighbours(target, MIN_SIMILARITY)) near.set(page, score);
|
|
35209
|
+
for (const key of sharedQueries.keys()) {
|
|
35210
|
+
const source = Math.floor(key / count);
|
|
35211
|
+
if (!near.has(source)) near.set(source, index.similarity(source, target));
|
|
35212
|
+
}
|
|
35213
|
+
const ranked = [...near].map(([source, raw]) => ({
|
|
35214
|
+
source,
|
|
34466
35215
|
target,
|
|
34467
|
-
|
|
34468
|
-
|
|
34469
|
-
|
|
34470
|
-
|
|
34471
|
-
|
|
34472
|
-
|
|
34473
|
-
|
|
34474
|
-
|
|
34475
|
-
|
|
34476
|
-
|
|
34477
|
-
|
|
34478
|
-
|
|
34479
|
-
|
|
34480
|
-
|
|
34481
|
-
|
|
34482
|
-
const
|
|
34483
|
-
|
|
34484
|
-
|
|
34485
|
-
|
|
34486
|
-
const
|
|
34487
|
-
|
|
34488
|
-
|
|
34489
|
-
|
|
34490
|
-
|
|
34491
|
-
|
|
34492
|
-
|
|
34493
|
-
|
|
34494
|
-
|
|
34495
|
-
|
|
34496
|
-
|
|
34497
|
-
|
|
34498
|
-
|
|
34499
|
-
|
|
34500
|
-
|
|
34501
|
-
|
|
34502
|
-
|
|
34503
|
-
|
|
34504
|
-
|
|
34505
|
-
|
|
34506
|
-
|
|
34507
|
-
|
|
35216
|
+
raw,
|
|
35217
|
+
similarity: adjusted(target, raw),
|
|
35218
|
+
sharedQueries: sharedQueries.get(pairKey(source, target)) ?? 0
|
|
35219
|
+
})).sort((left, right) => Number(right.sharedQueries > 0) - Number(left.sharedQueries > 0) || right.similarity - left.similarity || left.source - right.source);
|
|
35220
|
+
await classify([target, ...ranked.slice(0, URL_SOURCES_PLACED).map((c) => c.source)]);
|
|
35221
|
+
if (targetWeight(types[target] ?? null) === 0 || pages[target]?.noindex) status = "target-not-linkable";
|
|
35222
|
+
else for (const candidate of ranked) {
|
|
35223
|
+
if (candidatesBySource.size >= URL_SOURCES_PLACED) break;
|
|
35224
|
+
if (!linkable(candidate.source, target)) continue;
|
|
35225
|
+
candidatesBySource.set(candidate.source, [{
|
|
35226
|
+
...candidate,
|
|
35227
|
+
similarity: adjusted(target, candidate.raw)
|
|
35228
|
+
}]);
|
|
35229
|
+
}
|
|
35230
|
+
}
|
|
35231
|
+
const judged = [];
|
|
35232
|
+
let pairsConsidered = 0;
|
|
35233
|
+
let pairsJudged = 0;
|
|
35234
|
+
let fellBack = 0;
|
|
35235
|
+
const sources = [...candidatesBySource.keys()].sort((left, right) => left - right);
|
|
35236
|
+
let sourcesWithPlacement = 0;
|
|
35237
|
+
for (let at = 0; at < sources.length; at += SOURCE_WINDOW) {
|
|
35238
|
+
if (context.mode === "url" && sourcesWithPlacement >= URL_SOURCES_JUDGED) break;
|
|
35239
|
+
const window = sources.slice(at, at + SOURCE_WINDOW);
|
|
35240
|
+
deps.progress?.(`Placing and judging links from pages ${at + 1} to ${at + window.length} of ${sources.length}.`);
|
|
35241
|
+
let pairs = (await Promise.all(window.map(async (source) => {
|
|
35242
|
+
const page = pages[source];
|
|
35243
|
+
if (!page) return [];
|
|
35244
|
+
const sentences = await crawl.sentences.read(page.text);
|
|
35245
|
+
const placed = [];
|
|
35246
|
+
for (const candidate of candidatesBySource.get(source) ?? []) {
|
|
35247
|
+
pairsConsidered += 1;
|
|
35248
|
+
const to = pages[candidate.target];
|
|
35249
|
+
if (!to) continue;
|
|
35250
|
+
const placement = pairPlacement(sentences, to, index);
|
|
35251
|
+
if (!placement.sentences.length) continue;
|
|
35252
|
+
placed.push({
|
|
35253
|
+
candidate,
|
|
35254
|
+
placement
|
|
35255
|
+
});
|
|
35256
|
+
}
|
|
35257
|
+
return placed;
|
|
35258
|
+
}))).flat();
|
|
35259
|
+
if (context.mode === "url") {
|
|
35260
|
+
pairs = pairs.sort((left, right) => Number(right.candidate.sharedQueries > 0) - Number(left.candidate.sharedQueries > 0) || right.candidate.similarity - left.candidate.similarity || left.candidate.source - right.candidate.source).slice(0, URL_SOURCES_JUDGED - sourcesWithPlacement);
|
|
35261
|
+
sourcesWithPlacement += pairs.length;
|
|
35262
|
+
}
|
|
35263
|
+
const bySource = /* @__PURE__ */ new Map();
|
|
35264
|
+
for (const pair of pairs) {
|
|
35265
|
+
const list = bySource.get(pair.candidate.source) ?? [];
|
|
35266
|
+
list.push(pair);
|
|
35267
|
+
bySource.set(pair.candidate.source, list);
|
|
35268
|
+
}
|
|
35269
|
+
const groups = [];
|
|
35270
|
+
for (const list of bySource.values()) for (let start = 0; start < list.length; start += 2) groups.push(list.slice(start, start + 2));
|
|
35271
|
+
await Promise.all(groups.map(async (group) => {
|
|
35272
|
+
const first = group[0];
|
|
35273
|
+
if (!first) return;
|
|
35274
|
+
const source = pages[first.candidate.source];
|
|
35275
|
+
if (!source) return;
|
|
35276
|
+
const response = runner ? await runner.ask(judgeRequest({
|
|
35277
|
+
url: source.url,
|
|
35278
|
+
title: source.title,
|
|
35279
|
+
type: types[first.candidate.source] ?? null
|
|
35280
|
+
}, group.map(({ candidate, placement }) => {
|
|
35281
|
+
const to = pages[candidate.target];
|
|
35282
|
+
return {
|
|
35283
|
+
url: to.url,
|
|
35284
|
+
title: to.title,
|
|
35285
|
+
description: to.description,
|
|
35286
|
+
h1: to.h1,
|
|
35287
|
+
type: types[candidate.target] ?? null,
|
|
35288
|
+
intent: intents[candidate.target] ?? null,
|
|
35289
|
+
opening: to.excerpt.slice(0, OPENING_CHARS),
|
|
35290
|
+
placement
|
|
35291
|
+
};
|
|
35292
|
+
}))) : null;
|
|
35293
|
+
group.forEach(({ candidate, placement }, position) => {
|
|
35294
|
+
let decision;
|
|
35295
|
+
if (response) {
|
|
35296
|
+
pairsJudged += 1;
|
|
35297
|
+
decision = decideWithModel(placement, candidate.similarity, response.answers, position);
|
|
35298
|
+
} else {
|
|
35299
|
+
if (runner) fellBack += 1;
|
|
35300
|
+
decision = decideWithoutModel(placement, candidate.similarity);
|
|
35301
|
+
}
|
|
35302
|
+
judged.push({
|
|
35303
|
+
candidate,
|
|
35304
|
+
placement,
|
|
35305
|
+
decision
|
|
35306
|
+
});
|
|
35307
|
+
});
|
|
35308
|
+
}));
|
|
35309
|
+
}
|
|
35310
|
+
const heuristicScore = (item) => item.candidate.similarity * (.5 + (item.decision.state === "suggest" ? item.decision.sentence?.match ?? 0 : 0));
|
|
35311
|
+
const suggested = judged.filter((item) => item.decision.state === "suggest").sort((left, right) => (right.decision.probability ?? -1) - (left.decision.probability ?? -1) || heuristicScore(right) - heuristicScore(left) || (pages[left.candidate.source]?.url ?? "").localeCompare(pages[right.candidate.source]?.url ?? "") || (pages[left.candidate.target]?.url ?? "").localeCompare(pages[right.candidate.target]?.url ?? ""));
|
|
35312
|
+
const perTarget = /* @__PURE__ */ new Map();
|
|
35313
|
+
const usedSentences = /* @__PURE__ */ new Set();
|
|
35314
|
+
const kept = suggested.filter((item) => {
|
|
35315
|
+
if (item.decision.tier === "heuristic" && item.decision.sentence) {
|
|
35316
|
+
const key = `${item.candidate.source}\u0000${item.decision.sentence.text}`;
|
|
35317
|
+
if (usedSentences.has(key)) return false;
|
|
35318
|
+
usedSentences.add(key);
|
|
35319
|
+
}
|
|
35320
|
+
const seen = (perTarget.get(item.candidate.target) ?? 0) + 1;
|
|
35321
|
+
perTarget.set(item.candidate.target, seen);
|
|
35322
|
+
return seen <= 8;
|
|
35323
|
+
});
|
|
35324
|
+
const items = kept.map(({ candidate, decision }) => ({
|
|
35325
|
+
source: pages[candidate.source]?.url ?? "",
|
|
35326
|
+
target: pages[candidate.target]?.url ?? "",
|
|
35327
|
+
tier: decision.tier,
|
|
35328
|
+
probability: decision.probability,
|
|
35329
|
+
role: decision.role,
|
|
35330
|
+
anchor: decision.anchor,
|
|
35331
|
+
sentence: decision.sentence?.text ?? null,
|
|
35332
|
+
section: decision.sentence?.section || null,
|
|
35333
|
+
sentenceMatch: decision.sentence?.match ?? null,
|
|
35334
|
+
similarity: round$1(candidate.similarity),
|
|
35335
|
+
sharedQueries: candidate.sharedQueries,
|
|
35336
|
+
targetInboundLinks: inbound[candidate.target] ?? 0,
|
|
35337
|
+
sourceType: types[candidate.source] ?? null,
|
|
35338
|
+
targetType: types[candidate.target] ?? null
|
|
35339
|
+
}));
|
|
35340
|
+
const suggestedInbound = new Array(count).fill(0);
|
|
35341
|
+
const suggestedOutbound = new Array(count).fill(0);
|
|
35342
|
+
for (const { candidate } of kept) {
|
|
35343
|
+
suggestedInbound[candidate.target] = (suggestedInbound[candidate.target] ?? 0) + 1;
|
|
35344
|
+
suggestedOutbound[candidate.source] = (suggestedOutbound[candidate.source] ?? 0) + 1;
|
|
35345
|
+
}
|
|
35346
|
+
const held = judged.filter((item) => item.decision.state === "held-back");
|
|
35347
|
+
const reasons = {};
|
|
35348
|
+
for (const item of held) reasons[item.decision.reason] = (reasons[item.decision.reason] ?? 0) + 1;
|
|
35349
|
+
const declined = judged.filter((item) => item.decision.state === "declined").length;
|
|
35350
|
+
const usage = runner ? runner.usage() : null;
|
|
35351
|
+
const judgeStatus = !runner ? "not-used" : runner.authError || usage && usage.requests === 0 && usage.failedRequests > 0 ? "failed" : (usage?.failedRequests ?? 0) > 0 ? "partial" : "used";
|
|
35352
|
+
const modelLabel = usage ? `${usage.model}` : null;
|
|
35353
|
+
const judgeNote = !runner ? `Jev was not used: ${"unavailable" in deps.decisionModel ? deps.decisionModel.unavailable : "no TypeSafe key"}. Links come from TF-IDF similarity and sentence matching (heuristics). Add a TypeSafe API key at https://seoskill.dev/account#keys and run seo login, or set TYPESAFE_API_KEY on this machine, to have Jev classify pages and judge each link.` : runner.authError ? `${runner.authError.message} ${fellBack} link${fellBack === 1 ? "" : "s"} fell back to heuristics.` : judgeStatus === "partial" ? `${usage?.failedRequests} request${usage?.failedRequests === 1 ? "" : "s"} to ${modelLabel} failed after retries; ${fellBack} link${fellBack === 1 ? "" : "s"} fell back to heuristics.` : `${modelLabel} classified ${pagesClassified} page${pagesClassified === 1 ? "" : "s"} and judged ${pairsJudged} link${pairsJudged === 1 ? "" : "s"} in ${usage?.requests ?? 0} requests, for $${(usage?.costUsd ?? 0).toFixed(4)}.`;
|
|
35354
|
+
if (status === "complete" && (crawl.stats.capReached || judgeStatus === "partial" || judgeStatus === "failed")) status = "partial";
|
|
35355
|
+
const counts = {
|
|
35356
|
+
verdict: "",
|
|
35357
|
+
pagesCrawled: count,
|
|
35358
|
+
existingContextualLinks,
|
|
35359
|
+
orphanPages: inboundAnywhere.filter((value, id) => value === 0 && types[id] !== "home").length,
|
|
35360
|
+
weaklyLinkedPages: inbound.filter((value, id) => value < WEAK_INBOUND && types[id] !== "home").length,
|
|
35361
|
+
pairsConsidered,
|
|
35362
|
+
suggestions: items.length,
|
|
35363
|
+
strong: items.filter((item) => item.tier === "strong").length,
|
|
35364
|
+
possible: items.filter((item) => item.tier === "possible").length,
|
|
35365
|
+
heuristic: items.filter((item) => item.tier === "heuristic").length,
|
|
35366
|
+
heldBack: held.length,
|
|
35367
|
+
declined
|
|
35368
|
+
};
|
|
35369
|
+
const targetPage = target !== void 0 ? pages[target] : void 0;
|
|
35370
|
+
const report = {
|
|
34508
35371
|
site: input.site,
|
|
34509
|
-
|
|
34510
|
-
|
|
34511
|
-
|
|
34512
|
-
|
|
34513
|
-
|
|
34514
|
-
|
|
34515
|
-
|
|
34516
|
-
|
|
34517
|
-
|
|
34518
|
-
|
|
34519
|
-
|
|
34520
|
-
|
|
34521
|
-
|
|
34522
|
-
|
|
34523
|
-
|
|
34524
|
-
calls: result.calls,
|
|
34525
|
-
possiblyTruncated: result.rowsFetched >= MAX_GSC_ROWS
|
|
34526
|
-
})),
|
|
34527
|
-
rowsFetched: targetRequests.reduce((sum, { result }) => sum + result.rowsFetched, 0),
|
|
34528
|
-
calls: targetRequests.reduce((sum, { result }) => sum + result.calls, 0),
|
|
34529
|
-
maxRowsPerRequest: MAX_GSC_ROWS,
|
|
34530
|
-
possiblyTruncated: targetTruncated
|
|
34531
|
-
},
|
|
34532
|
-
candidates: {
|
|
34533
|
-
queried: queryCandidates,
|
|
34534
|
-
rowsFetched: sourceResult.rowsFetched,
|
|
34535
|
-
calls: sourceResult.calls,
|
|
34536
|
-
maxRows: MAX_GSC_ROWS,
|
|
34537
|
-
possiblyTruncated: sourceTruncated
|
|
34538
|
-
},
|
|
34539
|
-
completeness: !queryCandidates ? "not-queried" : targetTruncated || sourceTruncated ? "possibly-truncated" : "retained-rows-only"
|
|
35372
|
+
mode: context.mode,
|
|
35373
|
+
targetUrl: context.targetUrl ?? null,
|
|
35374
|
+
generatedAt: context.now.toISOString(),
|
|
35375
|
+
dataStatus: status,
|
|
35376
|
+
crawl: {
|
|
35377
|
+
startUrl: context.startUrl,
|
|
35378
|
+
method: crawl.method,
|
|
35379
|
+
maxPages: crawl.stats.maxPages,
|
|
35380
|
+
pageLimit: crawl.stats.pageLimit,
|
|
35381
|
+
urlsFound: crawl.stats.urlsFound,
|
|
35382
|
+
pagesCrawled: count,
|
|
35383
|
+
capReached: crawl.stats.capReached,
|
|
35384
|
+
crawlDelaySeconds: crawl.stats.crawlDelaySeconds,
|
|
35385
|
+
skipped: crawl.stats.skipped,
|
|
35386
|
+
redirectsChecked: crawl.stats.redirectsChecked
|
|
34540
35387
|
},
|
|
34541
|
-
|
|
34542
|
-
|
|
34543
|
-
|
|
34544
|
-
|
|
34545
|
-
|
|
34546
|
-
|
|
34547
|
-
|
|
34548
|
-
|
|
34549
|
-
|
|
34550
|
-
|
|
34551
|
-
|
|
34552
|
-
|
|
34553
|
-
|
|
34554
|
-
|
|
34555
|
-
|
|
34556
|
-
maxGscRowsPerRequest: MAX_GSC_ROWS
|
|
34557
|
-
},
|
|
34558
|
-
target: {
|
|
34559
|
-
requestedUrl: target.requestedUrl,
|
|
34560
|
-
preferredUrl: target.preferredUrl,
|
|
34561
|
-
finalUrl: target.finalUrl,
|
|
34562
|
-
canonical: target.canonical,
|
|
34563
|
-
status: target.status,
|
|
34564
|
-
aliases: target.aliases,
|
|
34565
|
-
verification: target.verification,
|
|
34566
|
-
technicalSignals: target.technicalSignals,
|
|
34567
|
-
fetchDiagnostics: target.fetchDiagnostics,
|
|
34568
|
-
queries: analysis.targetQueries
|
|
34569
|
-
},
|
|
34570
|
-
selection,
|
|
34571
|
-
summary: {
|
|
34572
|
-
targetQueries: analysis.selection.targetEligibleQueries,
|
|
34573
|
-
candidateSources: analysis.candidates.length,
|
|
34574
|
-
attemptedSources,
|
|
34575
|
-
checkedSources,
|
|
34576
|
-
returnedSources: items.length,
|
|
34577
|
-
existingLinksObserved: existingLinkExclusions,
|
|
34578
|
-
technicalExclusions,
|
|
34579
|
-
failedChecks,
|
|
34580
|
-
uncheckedCandidates,
|
|
34581
|
-
matchedQueryImpressions: items.reduce((sum, item) => sum + item.matchedQueryImpressions, 0),
|
|
34582
|
-
brandFiltering: "excluded",
|
|
34583
|
-
verdict: summaryVerdict
|
|
35388
|
+
searchConsole,
|
|
35389
|
+
judge: {
|
|
35390
|
+
status: judgeStatus,
|
|
35391
|
+
note: judgeNote,
|
|
35392
|
+
keySource: "model" in deps.decisionModel ? deps.decisionModel.keySource : null,
|
|
35393
|
+
provider: usage?.provider ?? null,
|
|
35394
|
+
model: usage?.model ?? null,
|
|
35395
|
+
requests: usage?.requests ?? 0,
|
|
35396
|
+
failedRequests: usage?.failedRequests ?? 0,
|
|
35397
|
+
inputTokens: usage?.inputTokens ?? 0,
|
|
35398
|
+
costUsd: usage?.costUsd ?? 0,
|
|
35399
|
+
pagesClassified,
|
|
35400
|
+
pairsJudged,
|
|
35401
|
+
thresholds: { ...THRESHOLDS },
|
|
35402
|
+
floors: { ...FLOORS }
|
|
34584
35403
|
},
|
|
35404
|
+
target: target !== void 0 && targetPage ? {
|
|
35405
|
+
url: targetPage.url,
|
|
35406
|
+
title: targetPage.title,
|
|
35407
|
+
type: types[target] ?? null,
|
|
35408
|
+
intent: intents[target] ?? null,
|
|
35409
|
+
inboundLinks: inbound[target] ?? 0,
|
|
35410
|
+
inboundLinksAnywhere: inboundAnywhere[target] ?? 0,
|
|
35411
|
+
noindex: targetPage.noindex
|
|
35412
|
+
} : null,
|
|
35413
|
+
summary: counts,
|
|
34585
35414
|
items,
|
|
34586
|
-
|
|
34587
|
-
|
|
34588
|
-
|
|
34589
|
-
|
|
34590
|
-
|
|
34591
|
-
|
|
34592
|
-
|
|
34593
|
-
|
|
34594
|
-
|
|
34595
|
-
|
|
34596
|
-
|
|
34597
|
-
|
|
34598
|
-
|
|
34599
|
-
|
|
34600
|
-
|
|
35415
|
+
heldBack: {
|
|
35416
|
+
count: held.length,
|
|
35417
|
+
reasons,
|
|
35418
|
+
examples: held.slice(0, EXAMPLE_LIMIT).map((item) => ({
|
|
35419
|
+
source: pages[item.candidate.source]?.url ?? "",
|
|
35420
|
+
target: pages[item.candidate.target]?.url ?? "",
|
|
35421
|
+
reason: item.decision.reason,
|
|
35422
|
+
detail: item.decision.detail
|
|
35423
|
+
}))
|
|
35424
|
+
},
|
|
35425
|
+
pageInventory: context.mode === "site" ? pages.map((page, id) => ({
|
|
35426
|
+
url: page.url,
|
|
35427
|
+
title: page.title,
|
|
35428
|
+
type: types[id] ?? null,
|
|
35429
|
+
intent: intents[id] ?? null,
|
|
35430
|
+
words: page.words,
|
|
35431
|
+
inboundLinks: inbound[id] ?? 0,
|
|
35432
|
+
inboundLinksAnywhere: inboundAnywhere[id] ?? 0,
|
|
35433
|
+
outboundLinks: contextual[id]?.size ?? 0,
|
|
35434
|
+
suggestedInbound: suggestedInbound[id] ?? 0,
|
|
35435
|
+
suggestedOutbound: suggestedOutbound[id] ?? 0,
|
|
35436
|
+
noindex: page.noindex
|
|
35437
|
+
})).sort((left, right) => left.inboundLinks - right.inboundLinks || left.url.localeCompare(right.url)) : [],
|
|
35438
|
+
redirectedLinks,
|
|
35439
|
+
genericAnchors: crawl.genericAnchors.map((anchor) => ({
|
|
35440
|
+
source: pages[anchor.source]?.url ?? "",
|
|
35441
|
+
href: anchor.href,
|
|
35442
|
+
text: anchor.text
|
|
35443
|
+
})),
|
|
35444
|
+
warnings,
|
|
35445
|
+
caveats: [],
|
|
35446
|
+
next: null
|
|
34601
35447
|
};
|
|
34602
|
-
|
|
34603
|
-
|
|
34604
|
-
|
|
34605
|
-
const placed = found.map(({ item, text }) => {
|
|
34606
|
-
const placement = target && text ? linkPlacement(text, target, corpus) : {
|
|
34607
|
-
contentSimilarity: 0,
|
|
34608
|
-
sentences: []
|
|
34609
|
-
};
|
|
34610
|
-
const best = placement.sentences[0];
|
|
34611
|
-
const recommendation = best ? {
|
|
34612
|
-
...item.recommendation,
|
|
34613
|
-
action: `${item.recommendation.action} It fits in: "${best.text}" Anchor text could be ${best.anchors.map((anchor) => `"${anchor}"`).join(" or ")}.`
|
|
34614
|
-
} : {
|
|
34615
|
-
...item.recommendation,
|
|
34616
|
-
action: `${item.recommendation.action} No sentence on this page names the target, so a link here needs new text.`
|
|
34617
|
-
};
|
|
34618
|
-
return {
|
|
34619
|
-
...item,
|
|
34620
|
-
placement,
|
|
34621
|
-
recommendation
|
|
34622
|
-
};
|
|
35448
|
+
report.summary.verdict = internalLinksVerdict({
|
|
35449
|
+
...report,
|
|
35450
|
+
counts
|
|
34623
35451
|
});
|
|
34624
|
-
|
|
35452
|
+
report.caveats = caveats$1(report);
|
|
35453
|
+
const top = items[0];
|
|
35454
|
+
report.next = context.mode === "site" ? top ? `seo internal-links --url ${top.target}` : null : `seo page --url ${report.target?.url ?? context.targetUrl}`;
|
|
35455
|
+
return report;
|
|
35456
|
+
}
|
|
35457
|
+
function caveats$1(report) {
|
|
35458
|
+
const crawl = report.crawl;
|
|
35459
|
+
return [
|
|
35460
|
+
`Crawl: ${crawl.pagesCrawled} page${crawl.pagesCrawled === 1 ? "" : "s"} from ${crawl.method === "sitemap" ? "the sitemap" : "links from the start page"}, limit ${crawl.pageLimit}${crawl.capReached ? " (reached, so pages beyond it were not checked and orphan and weak-link counts cover crawled pages only)" : " (not reached)"}.${crawl.crawlDelaySeconds ? ` robots.txt crawl-delay of ${crawl.crawlDelaySeconds}s was honoured.` : ""}`,
|
|
35461
|
+
report.judge.note,
|
|
35462
|
+
report.searchConsole.status === "used" ? `Search Console: ${report.searchConsole.rows.toLocaleString("en-US")} query rows from ${report.searchConsole.range?.startDate} to ${report.searchConsole.range?.endDate}, brand queries left out; ${report.searchConsole.pairsWithSharedQueries} page pairs share a query.${report.searchConsole.possiblyTruncated ? " The 100,000-row cap was reached." : ""}` : `Search Console was not used${report.searchConsole.note ? `: ${report.searchConsole.note}` : ""}. Candidates come from page text only.`,
|
|
35463
|
+
`Rules in code: a link needs a model probability of ${THRESHOLDS.possible} (${THRESHOLDS.strong} is strong), a sentence with at least ${FLOORS.minSentenceMatch * 100}% of the words that name the target or pages at least ${FLOORS.minSimilarityAlone} similar, and an anchor from the chosen sentence. At most 8 links per target.`,
|
|
35464
|
+
"Similarity, sentence match and anchors are text heuristics, and a model probability is a judgement of reader value. None of them predicts rankings or traffic."
|
|
35465
|
+
];
|
|
34625
35466
|
}
|
|
34626
35467
|
//#endregion
|
|
34627
35468
|
//#region packages/core/dist/analyze/url-templates.js
|
|
@@ -37148,10 +37989,10 @@ function applyVerification(item, coverage) {
|
|
|
37148
37989
|
function verdict(report) {
|
|
37149
37990
|
if (report.dataStatus === "empty") return "Google Search Console returned no retained query/page rows for this window.";
|
|
37150
37991
|
if (report.dataStatus === "filtered") return "No pages met the position, brand, and minimum-impression criteria.";
|
|
37151
|
-
if (report.technical > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.technical, "returned page")} ${plural$
|
|
37152
|
-
if (report.failed > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.failed, "returned page")} could not be verified and ${plural$
|
|
37153
|
-
if (report.content > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.content, "returned page")} ${plural$
|
|
37154
|
-
return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.returned, "page")} ${plural$
|
|
37992
|
+
if (report.technical > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.technical, "returned page")} ${plural$4(report.technical, "has", "have")} verified technical issues to fix before content changes.`;
|
|
37993
|
+
if (report.failed > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.failed, "returned page")} could not be verified and ${plural$4(report.failed, "needs", "need")} a fetch check before content changes.`;
|
|
37994
|
+
if (report.content > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.content, "returned page")} ${plural$4(report.content, "has", "have")} verified content or search-framing gaps.`;
|
|
37995
|
+
return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.returned, "page")} ${plural$4(report.returned, "is", "are")} returned in priority order for investigation.`;
|
|
37155
37996
|
}
|
|
37156
37997
|
function reportRecommendations(items) {
|
|
37157
37998
|
const first = items[0];
|
|
@@ -39995,6 +40836,65 @@ async function withDomainRating(rows, key) {
|
|
|
39995
40836
|
};
|
|
39996
40837
|
}
|
|
39997
40838
|
//#endregion
|
|
40839
|
+
//#region packages/core/dist/commands/internal-links.js
|
|
40840
|
+
const SUMMARY_ROWS = 10;
|
|
40841
|
+
const PAGE_SIZE$1 = 50;
|
|
40842
|
+
function rows(report, from, size) {
|
|
40843
|
+
return report.items.slice(from, from + size).map((item) => ({
|
|
40844
|
+
source: item.source,
|
|
40845
|
+
target: item.target,
|
|
40846
|
+
tier: item.tier,
|
|
40847
|
+
p: item.probability ?? "",
|
|
40848
|
+
role: item.role ?? "",
|
|
40849
|
+
anchor: item.anchor ?? "",
|
|
40850
|
+
sentence: item.sentence ?? "no sentence: needs new text"
|
|
40851
|
+
}));
|
|
40852
|
+
}
|
|
40853
|
+
function internalLinksText(report, options) {
|
|
40854
|
+
if (options.detail === "full") return `${report.summary.verdict}\n\n${JSON.stringify(report)}`;
|
|
40855
|
+
const origin = (() => {
|
|
40856
|
+
try {
|
|
40857
|
+
return new URL(report.items[0]?.source ?? report.target?.url ?? report.crawl.startUrl).origin;
|
|
40858
|
+
} catch {
|
|
40859
|
+
return null;
|
|
40860
|
+
}
|
|
40861
|
+
})();
|
|
40862
|
+
const columns = [
|
|
40863
|
+
"source",
|
|
40864
|
+
"target",
|
|
40865
|
+
"tier",
|
|
40866
|
+
"p",
|
|
40867
|
+
"role",
|
|
40868
|
+
"anchor",
|
|
40869
|
+
"sentence"
|
|
40870
|
+
];
|
|
40871
|
+
if (options.detail === "rows") {
|
|
40872
|
+
const page = rows(report, options.offset, PAGE_SIZE$1);
|
|
40873
|
+
const end = options.offset + page.length;
|
|
40874
|
+
return [
|
|
40875
|
+
`Internal links · ${report.mode === "url" ? report.targetUrl : report.site} · ${report.dataStatus}`,
|
|
40876
|
+
`Links ${options.offset + 1} to ${end} of ${report.items.length}:`,
|
|
40877
|
+
table(page, columns, origin, 120),
|
|
40878
|
+
end < report.items.length ? `Next page: ${options.more("rows", end)}` : ""
|
|
40879
|
+
].filter(Boolean).join("\n\n");
|
|
40880
|
+
}
|
|
40881
|
+
const s = report.summary;
|
|
40882
|
+
const judge = report.judge;
|
|
40883
|
+
const shown = rows(report, 0, SUMMARY_ROWS);
|
|
40884
|
+
const held = Object.entries(report.heldBack.reasons).map(([reason, count]) => `${reason} ${count}`).join(", ");
|
|
40885
|
+
return [
|
|
40886
|
+
`Internal links · ${report.mode === "url" ? report.targetUrl : report.site} · ${report.dataStatus}`,
|
|
40887
|
+
s.verdict,
|
|
40888
|
+
`Crawl: ${s.pagesCrawled} pages (limit ${report.crawl.pageLimit}, ${report.crawl.capReached ? "reached" : "not reached"}), ${s.existingContextualLinks} contextual links, ${s.orphanPages} orphan pages, ${s.weaklyLinkedPages} pages with fewer than 3 contextual links in.`,
|
|
40889
|
+
`Judge: ${judge.note}`,
|
|
40890
|
+
report.target ? `Target: ${report.target.type ?? "type unknown"}, ${report.target.intent ?? "intent unknown"}, ${report.target.inboundLinks} contextual links in.` : "",
|
|
40891
|
+
s.heldBack ? `Held back by the evidence floors: ${held}.` : "",
|
|
40892
|
+
shown.length ? `Links (${shown.length} of ${report.items.length}):\n${table(shown, columns, origin, 90)}` : "No links passed the rules.",
|
|
40893
|
+
`Limits:\n${report.caveats.filter((line) => line !== judge.note).map((line) => `- ${line}`).join("\n")}`,
|
|
40894
|
+
`More: ${shown.length < report.items.length ? `${options.more("rows", shown.length)} for more links, or ` : ""}${options.more("full")} for every field.${report.next ? ` Next: ${report.next}` : ""}`
|
|
40895
|
+
].filter(Boolean).join("\n\n");
|
|
40896
|
+
}
|
|
40897
|
+
//#endregion
|
|
39998
40898
|
//#region packages/core/dist/commands/page-facts.js
|
|
39999
40899
|
const MAX_PAGES = 20;
|
|
40000
40900
|
const TIMEOUT_MS = 6e3;
|
|
@@ -41455,6 +42355,25 @@ async function localResearchKeys() {
|
|
|
41455
42355
|
if (ahrefs?.apiKey) keys.set("x-ahrefs-key", ahrefs.apiKey);
|
|
41456
42356
|
return keys;
|
|
41457
42357
|
}
|
|
42358
|
+
const TYPESAFE_API_KEY_ENV = "TYPESAFE_API_KEY";
|
|
42359
|
+
async function localInternalLinksDependencies(options) {
|
|
42360
|
+
const searchConsole = await resolveLocalSite(options.site, { refresh: options.refresh }).then((property) => ({
|
|
42361
|
+
property,
|
|
42362
|
+
query: (request) => querySearchAnalytics(property, request, { refresh: options.refresh })
|
|
42363
|
+
}), (error) => ({ unavailable: `no Search Console property was found for this site (${error instanceof Error ? error.message : String(error)})` }));
|
|
42364
|
+
const local = process.env[TYPESAFE_API_KEY_ENV]?.trim();
|
|
42365
|
+
const saved = local ? void 0 : (await localResearchKeys().catch(() => new Headers())).get("x-typesafe-key");
|
|
42366
|
+
return {
|
|
42367
|
+
searchConsole,
|
|
42368
|
+
decisionModel: local ? {
|
|
42369
|
+
model: typesafeDecisionModel({ apiKey: local }),
|
|
42370
|
+
keySource: "environment"
|
|
42371
|
+
} : saved ? {
|
|
42372
|
+
model: typesafeDecisionModel({ apiKey: saved }),
|
|
42373
|
+
keySource: "account"
|
|
42374
|
+
} : { unavailable: "no TypeSafe key is set" }
|
|
42375
|
+
};
|
|
42376
|
+
}
|
|
41458
42377
|
async function localRefreshClicky() {
|
|
41459
42378
|
if (!cloudLinked()) return null;
|
|
41460
42379
|
await cloudCredentials({ refreshClicky: true });
|
|
@@ -42899,22 +43818,27 @@ function registerOpportunityTools(server, dependencies = {}) {
|
|
|
42899
43818
|
}
|
|
42900
43819
|
});
|
|
42901
43820
|
server.registerTool("seo_internal_links", {
|
|
42902
|
-
description: "
|
|
43821
|
+
description: "Internal links to add across a site, or to one target URL, placed in real sentences",
|
|
42903
43822
|
inputSchema: {
|
|
42904
43823
|
...mcpReportInputSchema(["site", "refresh"]),
|
|
42905
|
-
targetUrl: z$1.string().url().refine((value) => /^https?:\/\//.test(value), "Use an HTTP(S) URL."),
|
|
43824
|
+
targetUrl: z$1.string().url().refine((value) => /^https?:\/\//.test(value), "Use an HTTP(S) URL.").optional(),
|
|
43825
|
+
maxPages: z$1.number().int().min(1).max(MAX_MAX_PAGES).optional(),
|
|
42906
43826
|
startDate: z$1.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional(),
|
|
42907
43827
|
endDate: z$1.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional()
|
|
42908
43828
|
}
|
|
42909
|
-
}, async ({ site, targetUrl, startDate, endDate, refresh }) => {
|
|
43829
|
+
}, async ({ site, targetUrl, maxPages, startDate, endDate, refresh }) => {
|
|
42910
43830
|
try {
|
|
42911
43831
|
const result = await (dependencies.internalLinksReport ?? internalLinksReport)({
|
|
42912
43832
|
site,
|
|
42913
43833
|
targetUrl,
|
|
43834
|
+
maxPages,
|
|
42914
43835
|
startDate,
|
|
42915
43836
|
endDate,
|
|
42916
43837
|
refresh
|
|
42917
|
-
}
|
|
43838
|
+
}, await localInternalLinksDependencies({
|
|
43839
|
+
site: targetUrl ?? site,
|
|
43840
|
+
refresh
|
|
43841
|
+
}));
|
|
42918
43842
|
return toolSuccess(result.summary.verdict, result);
|
|
42919
43843
|
} catch (error) {
|
|
42920
43844
|
return toolError(error);
|
|
@@ -43747,11 +44671,11 @@ const REPORT_GUIDANCE = {
|
|
|
43747
44671
|
outcome: "Cross-source URL groups with clear evidence limits and a representative review queue for URL Inspection."
|
|
43748
44672
|
},
|
|
43749
44673
|
"internal-links": {
|
|
43750
|
-
name: "Internal
|
|
43751
|
-
description: "
|
|
43752
|
-
useWhen: ["
|
|
43753
|
-
avoidWhen: ["You want links added automatically.", "The
|
|
43754
|
-
outcome: "
|
|
44674
|
+
name: "Internal links",
|
|
44675
|
+
description: "Crawl a site and find the internal links it is missing, across the whole site or to one target URL, each placed in a real sentence with a real phrase as anchor text.",
|
|
44676
|
+
useWhen: ["Pages that belong together do not link to each other.", "A target page needs more contextual links from the rest of the site."],
|
|
44677
|
+
avoidWhen: ["You want links added automatically.", "The site cannot be crawled from this machine."],
|
|
44678
|
+
outcome: "Links to add with the source sentence and anchor, judged by Jev when a TypeSafe key is set, plus orphan and weakly linked pages."
|
|
43755
44679
|
},
|
|
43756
44680
|
"link-recovery": {
|
|
43757
44681
|
name: "Recover broken search-value URLs",
|
|
@@ -44426,22 +45350,42 @@ function registerSharedTools(server, options = {}) {
|
|
|
44426
45350
|
saveReport: true
|
|
44427
45351
|
})));
|
|
44428
45352
|
server.registerTool("seo_internal_links", {
|
|
44429
|
-
title: "Internal
|
|
44430
|
-
description: "
|
|
45353
|
+
title: "Internal links",
|
|
45354
|
+
description: "Internal links to add, crawled from this machine. Send site to audit a whole site, or url to find links to one page. Each link names a real sentence on the source and a real phrase from it as anchor text. With a TypeSafe key (TYPESAFE_API_KEY or the seoskill.dev account), Jev classifies pages and judges each link; without one the result says Jev was not used. Search Console query overlap is added when this login has the property.",
|
|
44431
45355
|
inputSchema: {
|
|
44432
|
-
|
|
44433
|
-
|
|
44434
|
-
|
|
45356
|
+
site: z$1.string().min(1).optional().describe("Site to audit: a domain, URL or Search Console property."),
|
|
45357
|
+
url: z$1.url().optional().describe("One page to find links to."),
|
|
45358
|
+
max_pages: z$1.number().int().min(1).max(MAX_MAX_PAGES).optional().describe("site only: pages to crawl, 500 by default, up to 5,000."),
|
|
45359
|
+
start_date: z$1.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional().describe("YYYY-MM-DD for Search Console. Optional; send with end_date."),
|
|
45360
|
+
end_date: z$1.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional().describe("YYYY-MM-DD. Optional; send with start_date."),
|
|
45361
|
+
detail: z$1.enum([
|
|
45362
|
+
"summary",
|
|
45363
|
+
"rows",
|
|
45364
|
+
"full"
|
|
45365
|
+
]).default("summary").describe("summary first. rows: the next page of links. full: every field as JSON."),
|
|
45366
|
+
offset: z$1.number().int().min(0).default(0).describe("rows only: the first link to return.")
|
|
44435
45367
|
},
|
|
44436
45368
|
annotations: readOnly
|
|
44437
|
-
}, async ({ url, start_date, end_date }) => counted("internal-links", {}, options.telemetry, async () => {
|
|
45369
|
+
}, async ({ site, url, max_pages, start_date, end_date, detail, offset }) => counted("internal-links", {}, options.telemetry, async () => {
|
|
44438
45370
|
try {
|
|
44439
|
-
|
|
44440
|
-
|
|
45371
|
+
if (Boolean(site) === Boolean(url)) throw new SeoError("INVALID_INPUT", "Send site to audit a whole site, or url to find links to one page.");
|
|
45372
|
+
const target = url ?? site;
|
|
45373
|
+
return text(internalLinksText(await internalLinksReport({
|
|
45374
|
+
site: target,
|
|
44441
45375
|
targetUrl: url,
|
|
45376
|
+
maxPages: max_pages,
|
|
44442
45377
|
startDate: start_date,
|
|
44443
45378
|
endDate: end_date
|
|
44444
|
-
})
|
|
45379
|
+
}, await localInternalLinksDependencies({ site: target })), {
|
|
45380
|
+
detail,
|
|
45381
|
+
offset,
|
|
45382
|
+
more: (view, from) => mcpCall("internal-links", {
|
|
45383
|
+
...url ? { url } : { site },
|
|
45384
|
+
...max_pages ? { max_pages } : {},
|
|
45385
|
+
detail: view,
|
|
45386
|
+
offset: from
|
|
45387
|
+
})
|
|
45388
|
+
}));
|
|
44445
45389
|
} catch (error) {
|
|
44446
45390
|
return toolError(localError(error));
|
|
44447
45391
|
}
|