seo 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +22 -7
- package/README.md +1 -1
- package/dist/cli.js +77 -77
- package/dist/cli.js.map +1 -1
- package/dist/{dist-BvbFtizP.js → dist-B1bivk1S.js} +1 -1
- package/dist/{dist-CsWo-wJG.js → dist-Ban8uP1U.js} +117 -101
- package/dist/dist-Ban8uP1U.js.map +1 -0
- package/dist/{esm-CM7xTOjT.js → esm-CQJgJ8TD.js} +2 -2
- package/dist/{esm-CM7xTOjT.js.map → esm-CQJgJ8TD.js.map} +1 -1
- package/dist/index.d.ts +99 -45
- package/dist/index.js +378 -122
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +1022 -177
- package/dist/mcp.js.map +1 -1
- package/package.json +1 -1
- package/skills/seo/SKILL.md +15 -1
- package/dist/dist-CsWo-wJG.js.map +0 -1
package/dist/mcp.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-
|
|
2
|
+
import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-CQJgJ8TD.js";
|
|
3
3
|
import { createHash, randomUUID, sign } from "node:crypto";
|
|
4
4
|
import { ZodOptional, z } from "zod";
|
|
5
5
|
import { arch, homedir, platform, tmpdir, totalmem } from "node:os";
|
|
@@ -26,6 +26,7 @@ import { TextDecoder as TextDecoder$1, promisify } from "node:util";
|
|
|
26
26
|
import { execFile } from "node:child_process";
|
|
27
27
|
import "pino";
|
|
28
28
|
import "pino-pretty";
|
|
29
|
+
import { request } from "node:https";
|
|
29
30
|
import * as z3rt from "zod/v3";
|
|
30
31
|
import { ZodFirstPartyTypeKind } from "zod/v3";
|
|
31
32
|
import * as z4mini from "zod/v4-mini";
|
|
@@ -13789,7 +13790,7 @@ async function googleJson(gfetch, url, input) {
|
|
|
13789
13790
|
return readJson(response, input.label, { maxBytes: input.maxBytes });
|
|
13790
13791
|
}
|
|
13791
13792
|
const GSC = "https://www.googleapis.com/webmasters/v3";
|
|
13792
|
-
const PAGE_ROWS = 25e3;
|
|
13793
|
+
const PAGE_ROWS$1 = 25e3;
|
|
13793
13794
|
async function searchAnalyticsRows(gfetch, site, body, options) {
|
|
13794
13795
|
const rows = [];
|
|
13795
13796
|
const { maxRows: requested, ...apiBody } = body;
|
|
@@ -13800,7 +13801,7 @@ async function searchAnalyticsRows(gfetch, site, body, options) {
|
|
|
13800
13801
|
while (true) {
|
|
13801
13802
|
const remaining = typeof maxRows === "number" ? maxRows - rows.length : void 0;
|
|
13802
13803
|
if (typeof remaining === "number" && remaining <= 0) break;
|
|
13803
|
-
const rowLimit = Math.min(apiBody.rowLimit ?? PAGE_ROWS, remaining ?? PAGE_ROWS, PAGE_ROWS);
|
|
13804
|
+
const rowLimit = Math.min(apiBody.rowLimit ?? PAGE_ROWS$1, remaining ?? PAGE_ROWS$1, PAGE_ROWS$1);
|
|
13804
13805
|
const json = await googleJson(gfetch, url, {
|
|
13805
13806
|
...options,
|
|
13806
13807
|
label: "Search Console",
|
|
@@ -13877,7 +13878,7 @@ async function runAnalyticsReport(gfetch, propertyId, body, options) {
|
|
|
13877
13878
|
//#endregion
|
|
13878
13879
|
//#region packages/core/dist/bing/api.js
|
|
13879
13880
|
const BASE = "https://ssl.bing.com/webmaster/api.svc/json";
|
|
13880
|
-
const MAX_BYTES$
|
|
13881
|
+
const MAX_BYTES$4 = 8e6;
|
|
13881
13882
|
function bingCall(auth, options) {
|
|
13882
13883
|
return async (method, params = {}) => {
|
|
13883
13884
|
const url = new URL(`${BASE}/${method}`);
|
|
@@ -13902,7 +13903,7 @@ function bingCall(auth, options) {
|
|
|
13902
13903
|
if (response.status === 429 || /throttle/i.test(detail)) throw new SeoError("RATE_LIMITED", "Bing Webmaster is limiting requests. Try again in a few minutes.");
|
|
13903
13904
|
throw new SeoError("OPTIONAL_PROVIDER_UNAVAILABLE", `Bing Webmaster answered ${response.status}. ${detail}`.trim());
|
|
13904
13905
|
}
|
|
13905
|
-
return (await readJson(response, "Bing Webmaster", { maxBytes: MAX_BYTES$
|
|
13906
|
+
return (await readJson(response, "Bing Webmaster", { maxBytes: MAX_BYTES$4 })).d;
|
|
13906
13907
|
};
|
|
13907
13908
|
}
|
|
13908
13909
|
function day(value) {
|
|
@@ -16411,6 +16412,7 @@ const MEBIBYTE$2 = 1024 * 1024;
|
|
|
16411
16412
|
const CACHE_MAX_AGE_MS = 720 * 60 * 60 * 1e3;
|
|
16412
16413
|
const CACHE_LIMITS = {
|
|
16413
16414
|
gsc_cache: 48 * MEBIBYTE$2,
|
|
16415
|
+
url_inspection_cache: 16 * MEBIBYTE$2,
|
|
16414
16416
|
ga4_cache: 24 * MEBIBYTE$2,
|
|
16415
16417
|
provider_cache: 32 * MEBIBYTE$2,
|
|
16416
16418
|
http_cache: 128 * MEBIBYTE$2,
|
|
@@ -16428,6 +16430,16 @@ const CACHE_POLICIES = [
|
|
|
16428
16430
|
"response_json"
|
|
16429
16431
|
]
|
|
16430
16432
|
},
|
|
16433
|
+
{
|
|
16434
|
+
table: "url_inspection_cache",
|
|
16435
|
+
keys: ["site_url", "url"],
|
|
16436
|
+
timestamp: "fetched_at",
|
|
16437
|
+
sizeColumns: [
|
|
16438
|
+
"site_url",
|
|
16439
|
+
"url",
|
|
16440
|
+
"status_json"
|
|
16441
|
+
]
|
|
16442
|
+
},
|
|
16431
16443
|
{
|
|
16432
16444
|
table: "ga4_cache",
|
|
16433
16445
|
keys: ["property_id", "query_hash"],
|
|
@@ -16731,6 +16743,17 @@ CREATE INDEX IF NOT EXISTS idx_url_inspection_reservations_minute
|
|
|
16731
16743
|
credential_key, property_site_url, created_at
|
|
16732
16744
|
);
|
|
16733
16745
|
|
|
16746
|
+
CREATE TABLE IF NOT EXISTS url_inspection_cache (
|
|
16747
|
+
site_url TEXT NOT NULL,
|
|
16748
|
+
url TEXT NOT NULL,
|
|
16749
|
+
status_json TEXT NOT NULL,
|
|
16750
|
+
fetched_at INTEGER NOT NULL,
|
|
16751
|
+
expires_at INTEGER NOT NULL,
|
|
16752
|
+
PRIMARY KEY(site_url, url)
|
|
16753
|
+
) WITHOUT ROWID;
|
|
16754
|
+
CREATE INDEX IF NOT EXISTS idx_url_inspection_cache_expires
|
|
16755
|
+
ON url_inspection_cache(expires_at);
|
|
16756
|
+
|
|
16734
16757
|
CREATE TABLE IF NOT EXISTS performance_reports (
|
|
16735
16758
|
id TEXT PRIMARY KEY,
|
|
16736
16759
|
url TEXT NOT NULL,
|
|
@@ -16763,10 +16786,12 @@ function initDb(database, isNewDatabase) {
|
|
|
16763
16786
|
database.exec("DROP TABLE IF EXISTS provider_spend_notices");
|
|
16764
16787
|
ensureColumn(database, "http_cache", "metadata_json", "TEXT");
|
|
16765
16788
|
}).immediate();
|
|
16766
|
-
|
|
16767
|
-
|
|
16768
|
-
|
|
16769
|
-
|
|
16789
|
+
try {
|
|
16790
|
+
runCacheMaintenance(database, {
|
|
16791
|
+
compact: true,
|
|
16792
|
+
allowFullVacuum: true
|
|
16793
|
+
});
|
|
16794
|
+
} catch {}
|
|
16770
16795
|
}
|
|
16771
16796
|
function getDb() {
|
|
16772
16797
|
if (db) return db;
|
|
@@ -16777,12 +16802,41 @@ function getDb() {
|
|
|
16777
16802
|
mode: 448
|
|
16778
16803
|
});
|
|
16779
16804
|
ensurePrivateDatabaseFiles(path);
|
|
16780
|
-
|
|
16781
|
-
|
|
16782
|
-
|
|
16783
|
-
|
|
16805
|
+
const opened = new SqliteDatabase(path);
|
|
16806
|
+
try {
|
|
16807
|
+
ensurePrivateDatabaseFiles(path);
|
|
16808
|
+
initDb(opened, isNewDatabase);
|
|
16809
|
+
ensurePrivateDatabaseFiles(path);
|
|
16810
|
+
} catch (error) {
|
|
16811
|
+
opened.close();
|
|
16812
|
+
throw error;
|
|
16813
|
+
}
|
|
16814
|
+
db = opened;
|
|
16784
16815
|
return db;
|
|
16785
16816
|
}
|
|
16817
|
+
let memoryDb;
|
|
16818
|
+
function memoryCache() {
|
|
16819
|
+
if (!memoryDb) {
|
|
16820
|
+
memoryDb = new SqliteDatabase(":memory:");
|
|
16821
|
+
memoryDb.exec(CREATE_SQL);
|
|
16822
|
+
}
|
|
16823
|
+
return memoryDb;
|
|
16824
|
+
}
|
|
16825
|
+
function cacheDb() {
|
|
16826
|
+
if (memoryDb) return memoryDb;
|
|
16827
|
+
try {
|
|
16828
|
+
return getDb();
|
|
16829
|
+
} catch {
|
|
16830
|
+
return memoryCache();
|
|
16831
|
+
}
|
|
16832
|
+
}
|
|
16833
|
+
function writeToCache(write) {
|
|
16834
|
+
try {
|
|
16835
|
+
write(cacheDb());
|
|
16836
|
+
} catch {
|
|
16837
|
+
memoryCache();
|
|
16838
|
+
}
|
|
16839
|
+
}
|
|
16786
16840
|
function noteCacheWrite(sizeBytes = 0, database = getDb()) {
|
|
16787
16841
|
const counter = cacheMaintenanceCounters.get(database) ?? {
|
|
16788
16842
|
writes: 0,
|
|
@@ -16810,7 +16864,7 @@ function ga4RequestCanUseCache(body) {
|
|
|
16810
16864
|
}
|
|
16811
16865
|
async function runGa4Report(propertyId, body, opts = {}) {
|
|
16812
16866
|
const { client, identity } = await createGoogleAccessTokenClient(opts.accountEmail);
|
|
16813
|
-
const db =
|
|
16867
|
+
const db = cacheDb();
|
|
16814
16868
|
const queryHash = hashKey([
|
|
16815
16869
|
propertyId,
|
|
16816
16870
|
identity,
|
|
@@ -16823,10 +16877,12 @@ async function runGa4Report(propertyId, body, opts = {}) {
|
|
|
16823
16877
|
if (cacheable) {
|
|
16824
16878
|
const requestJson = JSON.stringify(body);
|
|
16825
16879
|
const responseJson = JSON.stringify(result);
|
|
16826
|
-
|
|
16827
|
-
|
|
16828
|
-
|
|
16829
|
-
|
|
16880
|
+
writeToCache((database) => {
|
|
16881
|
+
database.prepare(`INSERT OR REPLACE INTO ga4_cache
|
|
16882
|
+
(property_id, query_hash, request_json, response_json, row_count, fetched_at, expires_at)
|
|
16883
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`).run(propertyId, queryHash, requestJson, responseJson, result.rowCount ?? result.rows?.length ?? 0, Date.now(), Date.now() + 864e5);
|
|
16884
|
+
noteCacheWrite(Buffer.byteLength(requestJson) + Buffer.byteLength(responseJson), database);
|
|
16885
|
+
});
|
|
16830
16886
|
}
|
|
16831
16887
|
return result;
|
|
16832
16888
|
}
|
|
@@ -19198,7 +19254,7 @@ async function fetchRobotsSource(origin, refresh, options) {
|
|
|
19198
19254
|
const robotsUrl = new URL("/robots.txt", origin).toString();
|
|
19199
19255
|
const key = hashKey(["robots", robotsUrl]);
|
|
19200
19256
|
if (!refresh) {
|
|
19201
|
-
const cached =
|
|
19257
|
+
const cached = cacheDb().prepare("SELECT status, body_blob, expires_at FROM http_cache WHERE url_hash = ?").get(key);
|
|
19202
19258
|
if (cached?.body_blob && cached.status !== void 0 && cacheableStatus(cached.status) && cached.expires_at && cached.expires_at > Date.now()) return {
|
|
19203
19259
|
robotsUrl,
|
|
19204
19260
|
status: cached.status,
|
|
@@ -19217,12 +19273,12 @@ async function fetchRobotsSource(origin, refresh, options) {
|
|
|
19217
19273
|
signal: controller.signal
|
|
19218
19274
|
});
|
|
19219
19275
|
const text = await readBoundedResponseText(response, MAX_ROBOTS_RESPONSE_BYTES, "robots.txt response");
|
|
19220
|
-
if (options.writeCache !== false && cacheableStatus(response.status)) {
|
|
19221
|
-
|
|
19222
|
-
|
|
19223
|
-
|
|
19224
|
-
noteCacheWrite(Buffer.byteLength(text));
|
|
19225
|
-
}
|
|
19276
|
+
if (options.writeCache !== false && cacheableStatus(response.status)) writeToCache((database) => {
|
|
19277
|
+
database.prepare(`INSERT OR REPLACE INTO http_cache
|
|
19278
|
+
(url_hash, url, status, headers_json, body_blob, etag, fetched_at, expires_at)
|
|
19279
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(key, robotsUrl, response.status, "{}", Buffer.from(text), null, Date.now(), Date.now() + 864e5);
|
|
19280
|
+
noteCacheWrite(Buffer.byteLength(text), database);
|
|
19281
|
+
});
|
|
19226
19282
|
return {
|
|
19227
19283
|
robotsUrl,
|
|
19228
19284
|
status: response.status,
|
|
@@ -19329,7 +19385,6 @@ function decodePageFetchCacheEvidence(value) {
|
|
|
19329
19385
|
}
|
|
19330
19386
|
async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, respectRobots = false, writeCache = true) {
|
|
19331
19387
|
const startedAt = Date.now();
|
|
19332
|
-
const db = getDb();
|
|
19333
19388
|
const key = hashKey(["page", url]);
|
|
19334
19389
|
const host = new URL(url).host;
|
|
19335
19390
|
let robots = respectRobots ? await fetchRobots(new URL(url).origin, url, refresh, {
|
|
@@ -19337,7 +19392,7 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
|
|
|
19337
19392
|
signal
|
|
19338
19393
|
}) : void 0;
|
|
19339
19394
|
if (robots && robots.allowed !== true) throw new RobotsAccessError(robots.allowed === false ? "robots-disallowed" : "robots-deferred", robots);
|
|
19340
|
-
const cached =
|
|
19395
|
+
const cached = cacheDb().prepare("SELECT status, headers_json, body_blob, metadata_json, expires_at FROM http_cache WHERE url_hash = ? AND expires_at > ?").get(key, Date.now());
|
|
19341
19396
|
if (!refresh && cached) {
|
|
19342
19397
|
const evidence = decodePageFetchCacheEvidence(cached.metadata_json);
|
|
19343
19398
|
return {
|
|
@@ -19439,12 +19494,12 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
|
|
|
19439
19494
|
warnings: [...robots.allowed === null ? [`robots.txt availability is unknown${robots.status ? ` after HTTP ${robots.status}` : ""}; this fetch does not prove crawler access is allowed.`] : [], ...accessBlock ? [`${accessBlock.guidance.summary} Identify the crawler as ${accessBlock.crawler.userAgent}.`] : []],
|
|
19440
19495
|
robotsTxt: pageRobotsEvidence(robots)
|
|
19441
19496
|
};
|
|
19442
|
-
if (!accessBlock && writeCache) {
|
|
19443
|
-
|
|
19444
|
-
|
|
19445
|
-
|
|
19446
|
-
noteCacheWrite(Buffer.byteLength(html));
|
|
19447
|
-
}
|
|
19497
|
+
if (!accessBlock && writeCache) writeToCache((database) => {
|
|
19498
|
+
database.prepare(`INSERT OR REPLACE INTO http_cache
|
|
19499
|
+
(url_hash, url, status, headers_json, body_blob, metadata_json, etag, fetched_at, expires_at)
|
|
19500
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(key, url, response.response.status, JSON.stringify(headerMap), Buffer.from(html), encodePageFetchCacheEvidence(result), response.response.headers.get("etag"), Date.now(), Date.now() + 36e5);
|
|
19501
|
+
noteCacheWrite(Buffer.byteLength(html), database);
|
|
19502
|
+
});
|
|
19448
19503
|
return result;
|
|
19449
19504
|
}
|
|
19450
19505
|
//#endregion
|
|
@@ -20197,6 +20252,19 @@ function reserveUrlInspectionQuota(input) {
|
|
|
20197
20252
|
if ("error" in result) throw result.error;
|
|
20198
20253
|
return result.reservation;
|
|
20199
20254
|
}
|
|
20255
|
+
function readUrlInspectionQuota(input) {
|
|
20256
|
+
const database = input.database ?? getDb();
|
|
20257
|
+
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
20258
|
+
const bucket = database.prepare(`SELECT limit_count, used_count, reserved_count, blocked_until
|
|
20259
|
+
FROM url_inspection_quota_buckets
|
|
20260
|
+
WHERE credential_key = ? AND property_site_url = ? AND quota_date = ?`).get(input.credentialKey, input.property, urlInspectionQuotaDate(now));
|
|
20261
|
+
const limit = bucket?.limit_count ?? 2e3;
|
|
20262
|
+
return {
|
|
20263
|
+
used: (bucket?.blocked_until ?? 0) > now.getTime() ? limit : Math.min(limit, (bucket?.used_count ?? 0) + (bucket?.reserved_count ?? 0)),
|
|
20264
|
+
limit,
|
|
20265
|
+
resetAt: urlInspectionQuotaResetAt(now).toISOString()
|
|
20266
|
+
};
|
|
20267
|
+
}
|
|
20200
20268
|
function finalizeUrlInspectionQuota(input) {
|
|
20201
20269
|
const database = input.database ?? getDb();
|
|
20202
20270
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
@@ -20241,6 +20309,13 @@ function inspectionResponseError(response, input) {
|
|
|
20241
20309
|
if (response.status >= 500) return new SeoError("PROVIDER_UNAVAILABLE", `Google URL Inspection is unavailable (${response.status}). Try again later.`);
|
|
20242
20310
|
return new SeoError("PROVIDER_UNAVAILABLE", `URL Inspection failed with ${response.status}.`);
|
|
20243
20311
|
}
|
|
20312
|
+
async function urlInspectionQuota(input) {
|
|
20313
|
+
const { quotaIdentity } = await getAuthorized(input.accountEmail ?? await cloudAccountFor(input.siteUrl));
|
|
20314
|
+
return readUrlInspectionQuota({
|
|
20315
|
+
credentialKey: credentialKey(quotaIdentity),
|
|
20316
|
+
property: input.siteUrl
|
|
20317
|
+
});
|
|
20318
|
+
}
|
|
20244
20319
|
async function inspectUrl(input) {
|
|
20245
20320
|
const inspectionUrl = assertUrlMatchesGscProperty(input.siteUrl, input.inspectionUrl);
|
|
20246
20321
|
const { client, quotaIdentity } = await getAuthorized(input.accountEmail ?? await cloudAccountFor(input.siteUrl));
|
|
@@ -20302,7 +20377,7 @@ function resolveSearchDateWindow(input = 28) {
|
|
|
20302
20377
|
}
|
|
20303
20378
|
async function querySearchAnalytics(site, body, opts = {}) {
|
|
20304
20379
|
const { client, identity } = await getAuthorized(opts.accountEmail ?? await cloudAccountFor(site));
|
|
20305
|
-
const db =
|
|
20380
|
+
const db = cacheDb();
|
|
20306
20381
|
const queryHash = hashKey([
|
|
20307
20382
|
site,
|
|
20308
20383
|
identity,
|
|
@@ -20322,10 +20397,12 @@ async function querySearchAnalytics(site, body, opts = {}) {
|
|
|
20322
20397
|
const calls = result.calls;
|
|
20323
20398
|
const requestJson = JSON.stringify(body);
|
|
20324
20399
|
const responseJson = JSON.stringify(allRows);
|
|
20325
|
-
|
|
20326
|
-
|
|
20327
|
-
|
|
20328
|
-
|
|
20400
|
+
writeToCache((database) => {
|
|
20401
|
+
database.prepare(`INSERT OR REPLACE INTO gsc_cache
|
|
20402
|
+
(site_url, query_hash, request_json, response_json, row_count, fetched_at, expires_at)
|
|
20403
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`).run(site, queryHash, requestJson, responseJson, allRows.length, Date.now(), Date.now() + 864e5);
|
|
20404
|
+
noteCacheWrite(Buffer.byteLength(requestJson) + Buffer.byteLength(responseJson), database);
|
|
20405
|
+
});
|
|
20329
20406
|
return {
|
|
20330
20407
|
rows: allRows,
|
|
20331
20408
|
calls,
|
|
@@ -20461,8 +20538,10 @@ async function queryPagesTopQueriesBatch(site, pageUrls, range = 28) {
|
|
|
20461
20538
|
//#region packages/core/dist/gsc/client/sites.js
|
|
20462
20539
|
async function listSites(_refresh = false, accountEmail) {
|
|
20463
20540
|
const entries = !accountEmail && cloudLinked() ? await allCloudSites() : await listSearchConsoleSites(googleFetchFor((await getAuthorized(accountEmail)).client), LOCAL_SIGN_IN);
|
|
20464
|
-
|
|
20465
|
-
|
|
20541
|
+
writeToCache((database) => {
|
|
20542
|
+
const insert = database.prepare("INSERT OR REPLACE INTO sites (site_url, display_name, permission, added_at, is_default) VALUES (?, ?, ?, ?, COALESCE((SELECT is_default FROM sites WHERE site_url = ?), 0))");
|
|
20543
|
+
for (const site of entries) insert.run(site.siteUrl, site.siteUrl, site.permissionLevel ?? null, Date.now(), site.siteUrl);
|
|
20544
|
+
});
|
|
20466
20545
|
return entries;
|
|
20467
20546
|
}
|
|
20468
20547
|
async function allCloudSites() {
|
|
@@ -22083,7 +22162,7 @@ const actionPriority = new Map([
|
|
|
22083
22162
|
"unverified",
|
|
22084
22163
|
"covered"
|
|
22085
22164
|
].map((type, index) => [type, index]));
|
|
22086
|
-
function plural$
|
|
22165
|
+
function plural$7(count, singular, pluralLabel) {
|
|
22087
22166
|
return count === 1 ? singular : pluralLabel;
|
|
22088
22167
|
}
|
|
22089
22168
|
function actionable(item) {
|
|
@@ -22117,7 +22196,7 @@ function pageOpportunityVerdict(input) {
|
|
|
22117
22196
|
if (focus === "unverified") return "GSC visibility exists, but the page was not verified and no specific change is justified yet.";
|
|
22118
22197
|
if (focus === "covered") return "The verified page covers the sampled queries and CTR is not materially below its directional benchmark.";
|
|
22119
22198
|
const count = analysis.summary.opportunities;
|
|
22120
|
-
const prefix = `${count} ${plural$
|
|
22199
|
+
const prefix = `${count} ${plural$7(count, "actionable observation", "actionable observations")} found.`;
|
|
22121
22200
|
if (focus === "content-gap") return `${prefix} Verified body coverage is the main issue; confirm intent before expanding the page.`;
|
|
22122
22201
|
if (focus === "serp-framing") return `${prefix} The body is relevant, but its title, meta description, or H1 framing is weak.`;
|
|
22123
22202
|
if (focus === "ranking") return `${prefix} Most upside requires stronger relevance or authority; no CTR-only lift is claimed for page-two queries.`;
|
|
@@ -22132,7 +22211,7 @@ function pageOpportunityRecommendations(analysis) {
|
|
|
22132
22211
|
}
|
|
22133
22212
|
const recommendations = [`Start with "${top.query}". ${top.recommendation}`];
|
|
22134
22213
|
const technical = actions.filter((item) => item.opportunityType === "technical-check").length;
|
|
22135
|
-
if (technical) recommendations.push(`Resolve the technical evidence affecting ${technical} ${plural$
|
|
22214
|
+
if (technical) recommendations.push(`Resolve the technical evidence affecting ${technical} ${plural$7(technical, "row", "rows")} before content work.`);
|
|
22136
22215
|
const gaps = actions.filter((item) => item.opportunityType === "content-gap").length;
|
|
22137
22216
|
if (gaps >= 3) recommendations.push(`Group the ${gaps} verified content gaps into useful sections rather than adding one thin paragraph per query.`);
|
|
22138
22217
|
return recommendations;
|
|
@@ -25207,7 +25286,7 @@ function availabilityOf(status) {
|
|
|
25207
25286
|
function errorText(error) {
|
|
25208
25287
|
return error instanceof Error ? error.message : String(error);
|
|
25209
25288
|
}
|
|
25210
|
-
async function get(input, url, headers, readBody = true) {
|
|
25289
|
+
async function get$1(input, url, headers, readBody = true) {
|
|
25211
25290
|
const controller = abortController({
|
|
25212
25291
|
timeoutMs: input.timeoutMs,
|
|
25213
25292
|
signal: input.signal
|
|
@@ -25264,7 +25343,7 @@ function llmsLinks(text, base) {
|
|
|
25264
25343
|
return links;
|
|
25265
25344
|
}
|
|
25266
25345
|
async function checkLlmsTxt(input) {
|
|
25267
|
-
const result = await get(input, new URL("/llms.txt", input.url).toString());
|
|
25346
|
+
const result = await get$1(input, new URL("/llms.txt", input.url).toString());
|
|
25268
25347
|
const availability = result.availability === "available" && isHtml(result.contentType) ? "absent" : result.availability;
|
|
25269
25348
|
const exists = availability === "available";
|
|
25270
25349
|
const base = {
|
|
@@ -25281,7 +25360,7 @@ async function checkLlmsTxt(input) {
|
|
|
25281
25360
|
for (const url of found.slice(0, LLMS_LINK_LIMIT)) {
|
|
25282
25361
|
if (input.signal?.aborted) break;
|
|
25283
25362
|
checked += 1;
|
|
25284
|
-
const link = brokenLink(await get(input, url, void 0, false), false);
|
|
25363
|
+
const link = brokenLink(await get$1(input, url, void 0, false), false);
|
|
25285
25364
|
if (link) broken.push(link);
|
|
25286
25365
|
}
|
|
25287
25366
|
return {
|
|
@@ -25325,7 +25404,7 @@ function sitemapUrlsFromRobots(text) {
|
|
|
25325
25404
|
return [...urls];
|
|
25326
25405
|
}
|
|
25327
25406
|
async function checkRobotsAiAccess(input) {
|
|
25328
|
-
const result = await get(input, new URL("/robots.txt", input.url).toString());
|
|
25407
|
+
const result = await get$1(input, new URL("/robots.txt", input.url).toString());
|
|
25329
25408
|
const exists = result.availability === "available";
|
|
25330
25409
|
const unknown = result.availability === "rate-limited" || result.availability === "unreachable";
|
|
25331
25410
|
const declared = exists ? declaredUserAgents(result.text) : /* @__PURE__ */ new Set();
|
|
@@ -25354,7 +25433,7 @@ async function checkAgentResources(input) {
|
|
|
25354
25433
|
const resources = [];
|
|
25355
25434
|
for (const path of AGENT_RESOURCE_PATHS) {
|
|
25356
25435
|
if (input.signal?.aborted) break;
|
|
25357
|
-
const result = await get(input, new URL(path, input.url).toString());
|
|
25436
|
+
const result = await get$1(input, new URL(path, input.url).toString());
|
|
25358
25437
|
const exists = result.availability === "available" && !isHtml(result.contentType);
|
|
25359
25438
|
let validJson;
|
|
25360
25439
|
if (exists && /\bjson\b/i.test(result.contentType)) try {
|
|
@@ -25374,7 +25453,7 @@ async function checkAgentResources(input) {
|
|
|
25374
25453
|
return resources;
|
|
25375
25454
|
}
|
|
25376
25455
|
async function checkMarkdownNegotiation(input) {
|
|
25377
|
-
const result = await get(input, input.url, { accept: "text/markdown" }, false);
|
|
25456
|
+
const result = await get$1(input, input.url, { accept: "text/markdown" }, false);
|
|
25378
25457
|
const vary = result.headers?.get("vary") ?? "";
|
|
25379
25458
|
return {
|
|
25380
25459
|
url: result.url,
|
|
@@ -25421,7 +25500,7 @@ async function checkMarkdownCopies(input) {
|
|
|
25421
25500
|
const url = page.markdownAlternates?.[0];
|
|
25422
25501
|
if (!url || !input.allows(url)) continue;
|
|
25423
25502
|
checked += 1;
|
|
25424
|
-
const result = await get({
|
|
25503
|
+
const result = await get$1({
|
|
25425
25504
|
...input,
|
|
25426
25505
|
url
|
|
25427
25506
|
}, url);
|
|
@@ -26587,7 +26666,7 @@ function taskIds(value) {
|
|
|
26587
26666
|
}
|
|
26588
26667
|
}
|
|
26589
26668
|
function readProviderCache(key, schema, options = {}) {
|
|
26590
|
-
const database = options.database ??
|
|
26669
|
+
const database = options.database ?? cacheDb();
|
|
26591
26670
|
const now = options.now ?? Date.now();
|
|
26592
26671
|
const hash = requestHash(key, stableProviderRequestJson(key.request));
|
|
26593
26672
|
const row = database.prepare(`SELECT response_json, row_count, source_cost_micros, task_ids_json,
|
|
@@ -26608,9 +26687,11 @@ function readProviderCache(key, schema, options = {}) {
|
|
|
26608
26687
|
taskIds: taskIds(row.task_ids_json)
|
|
26609
26688
|
};
|
|
26610
26689
|
} catch {
|
|
26611
|
-
|
|
26612
|
-
|
|
26613
|
-
AND
|
|
26690
|
+
try {
|
|
26691
|
+
database.prepare(`DELETE FROM provider_cache
|
|
26692
|
+
WHERE provider = ? AND credential_scope = ? AND operation = ?
|
|
26693
|
+
AND request_hash = ?`).run(key.provider, key.credentialScope, key.operation, hash);
|
|
26694
|
+
} catch {}
|
|
26614
26695
|
return null;
|
|
26615
26696
|
}
|
|
26616
26697
|
}
|
|
@@ -26621,19 +26702,22 @@ function writeProviderCache(key, input, options = {}) {
|
|
|
26621
26702
|
code: "configuration",
|
|
26622
26703
|
message: `Provider cache TTL must be between 1 and ${CACHE_MAX_AGE_MS} milliseconds.`
|
|
26623
26704
|
});
|
|
26624
|
-
const database = options.database ?? getDb();
|
|
26625
26705
|
const now = options.now ?? Date.now();
|
|
26626
26706
|
const expiresAt = now + input.ttlMs;
|
|
26627
26707
|
const requestJson = stableProviderRequestJson(key.request);
|
|
26628
26708
|
const responseJson = JSON.stringify(input.data);
|
|
26629
26709
|
const normalizedTaskIds = [...new Set(input.taskIds)].filter((id) => id.length > 0 && id.length <= 100).sort((left, right) => left < right ? -1 : left > right ? 1 : 0).slice(0, 20);
|
|
26630
26710
|
const taskIdsJson = JSON.stringify(normalizedTaskIds);
|
|
26631
|
-
|
|
26632
|
-
|
|
26633
|
-
|
|
26634
|
-
|
|
26635
|
-
|
|
26636
|
-
|
|
26711
|
+
const save = (database) => {
|
|
26712
|
+
database.prepare(`INSERT OR REPLACE INTO provider_cache (
|
|
26713
|
+
provider, credential_scope, operation, request_hash, request_json,
|
|
26714
|
+
response_json, row_count, source_cost_micros, task_ids_json,
|
|
26715
|
+
fetched_at, expires_at
|
|
26716
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(key.provider, key.credentialScope, key.operation, requestHash(key, requestJson), requestJson, responseJson, input.rowCount, input.sourceCostMicros, taskIdsJson, now, expiresAt);
|
|
26717
|
+
noteCacheWrite(Buffer.byteLength(requestJson) + Buffer.byteLength(responseJson) + Buffer.byteLength(taskIdsJson), database);
|
|
26718
|
+
};
|
|
26719
|
+
if (options.database) save(options.database);
|
|
26720
|
+
else writeToCache(save);
|
|
26637
26721
|
return {
|
|
26638
26722
|
data: input.data,
|
|
26639
26723
|
storedAt: new Date(now).toISOString(),
|
|
@@ -27181,7 +27265,7 @@ const installedPackageJsonSchema = z.object({
|
|
|
27181
27265
|
function packageDirectory(packagesDir, packageName) {
|
|
27182
27266
|
return join(packagesDir, "node_modules", ...packageName.split("/"));
|
|
27183
27267
|
}
|
|
27184
|
-
function inside(parent, child) {
|
|
27268
|
+
function inside$1(parent, child) {
|
|
27185
27269
|
const path = relative(parent, child);
|
|
27186
27270
|
return path === "" || !path.startsWith("..") && !isAbsolute(path);
|
|
27187
27271
|
}
|
|
@@ -27213,14 +27297,14 @@ async function loadOne(installed, registry, options) {
|
|
|
27213
27297
|
async function loadProviderPackage(input, options) {
|
|
27214
27298
|
const importModule = options.importModule ?? ((url) => import(url));
|
|
27215
27299
|
const packageDir = realpathSync(packageDirectory(options.packagesDir, input.package));
|
|
27216
|
-
if (!inside(realpathSync(options.packagesDir), packageDir)) throw new Error("Installed provider package resolves outside the provider directory.");
|
|
27300
|
+
if (!inside$1(realpathSync(options.packagesDir), packageDir)) throw new Error("Installed provider package resolves outside the provider directory.");
|
|
27217
27301
|
const packageJsonPath = join(packageDir, "package.json");
|
|
27218
27302
|
const packageJson = installedPackageJsonSchema.parse(JSON.parse(readFileSync(packageJsonPath, "utf8")));
|
|
27219
27303
|
if (packageJson.name !== input.package || packageJson.version !== input.version) throw new Error("Installed provider package does not match its saved record.");
|
|
27220
27304
|
const packageRegistry = new ProviderExtensionRegistry();
|
|
27221
27305
|
for (const entry of packageJson.seo.providers) {
|
|
27222
27306
|
const entryPath = realpathSync(resolve(dirname(packageJsonPath), entry));
|
|
27223
|
-
if (!inside(packageDir, entryPath)) throw new Error("Provider entry point resolves outside its package.");
|
|
27307
|
+
if (!inside$1(packageDir, entryPath)) throw new Error("Provider entry point resolves outside its package.");
|
|
27224
27308
|
await providerActivate((await importModule(`${pathToFileURL(entryPath).href}?seo-provider=${encodeURIComponent(input.version)}`)).default)(packageRegistry.hostFor({
|
|
27225
27309
|
package: input.package,
|
|
27226
27310
|
version: input.version
|
|
@@ -27256,7 +27340,7 @@ async function loadInstalledProviderExtensions(options = {}) {
|
|
|
27256
27340
|
const DEFAULT_MAX_REQUESTS = 8;
|
|
27257
27341
|
const MAX_RESPONSE_BYTES$1 = 10 * 1024 * 1024;
|
|
27258
27342
|
const MAX_REQUEST_BODY_BYTES = 64 * 1024;
|
|
27259
|
-
const TIMEOUT_MS$
|
|
27343
|
+
const TIMEOUT_MS$3 = 15e3;
|
|
27260
27344
|
function normalizeProviderExtensionAccount(provider, account) {
|
|
27261
27345
|
const fields = provider.connection.fields.filter((field) => field.kind === "account");
|
|
27262
27346
|
const allowed = new Set(fields.map((field) => field.id));
|
|
@@ -27303,7 +27387,7 @@ function runtimeForProvider(provider, options = {}) {
|
|
|
27303
27387
|
},
|
|
27304
27388
|
fetch: providerFetch,
|
|
27305
27389
|
maxResponseBytes: MAX_RESPONSE_BYTES$1,
|
|
27306
|
-
timeoutMs: TIMEOUT_MS$
|
|
27390
|
+
timeoutMs: TIMEOUT_MS$3,
|
|
27307
27391
|
retry: (input.method ?? "GET") === "GET" ? "safe" : "never",
|
|
27308
27392
|
schema: z.unknown()
|
|
27309
27393
|
});
|
|
@@ -28866,7 +28950,7 @@ function analyzeCtrUnderperformersFromRows(input) {
|
|
|
28866
28950
|
const MAX_GSC_ROWS$3 = 1e5;
|
|
28867
28951
|
const DEFAULT_DAYS$3 = 28;
|
|
28868
28952
|
const MAX_DAYS$2 = 548;
|
|
28869
|
-
function plural$
|
|
28953
|
+
function plural$6(count, singular, pluralLabel = `${singular}s`) {
|
|
28870
28954
|
return count === 1 ? singular : pluralLabel;
|
|
28871
28955
|
}
|
|
28872
28956
|
async function ctrUnderperformersReportWith(input, dependencies) {
|
|
@@ -28969,11 +29053,11 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
28969
29053
|
minImpressions,
|
|
28970
29054
|
limit,
|
|
28971
29055
|
brandFiltering: input.includeBrand ? "included" : "excluded",
|
|
28972
|
-
verdict: top ? `${selection.eligibleUnderperformers} CTR ${plural$
|
|
29056
|
+
verdict: top ? `${selection.eligibleUnderperformers} CTR ${plural$6(selection.eligibleUnderperformers, "underperformer")} found in retained rows, with a calculated heuristic shortfall of ${totalClickShortfall.toFixed(0)} clicks for this window. Start review with "${top.query}" because it has the largest gap.` : dataStatus === "partial" ? "No material CTR underperformers remained in the validated retained rows, but partial evidence prevents an all-clear." : `No high-impression rows at average position below 4 in the retained rows are materially below the heuristic CTR benchmark.`
|
|
28973
29057
|
},
|
|
28974
29058
|
items,
|
|
28975
29059
|
caveats: [
|
|
28976
|
-
`Date window: ${range.startDate} to ${range.endDate} (${days} ${plural$
|
|
29060
|
+
`Date window: ${range.startDate} to ${range.endDate} (${days} ${plural$6(days, "day")}), using final GSC data where available.`,
|
|
28977
29061
|
`Brand queries: ${input.includeBrand ? "included" : "excluded"}. Brand terms used: ${brand.terms.join(", ") || "none"} (${brand.source === "provided" ? "provided" : "host forms, plus the bare domain stem only when Search Console shows the site is its navigational answer"}).`,
|
|
28978
29062
|
`Only rows at average position below 4 with at least ${minImpressions} impressions were checked. Rows at position 4 to 10 belong to quick-wins, which uses the same site CTR curve.`,
|
|
28979
29063
|
minImpressionsCaveat({
|
|
@@ -28981,13 +29065,13 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
28981
29065
|
belowMinimumRows: selection.belowMinimumRows
|
|
28982
29066
|
}),
|
|
28983
29067
|
"Expected CTR uses the site CTR curve from all page-one rows (75th percentile of qualified URL CTRs per position) when enough peer data exists, otherwise the fallback position curve.",
|
|
28984
|
-
...selection.fragmentRows ? [`${selection.fragmentRows} source ${plural$
|
|
28985
|
-
...selection.flaggedRows ? [`${selection.flaggedRows} eligible ${plural$
|
|
29068
|
+
...selection.fragmentRows ? [`${selection.fragmentRows} source ${plural$6(selection.fragmentRows, "row")} with a #fragment URL ${selection.fragmentRows === 1 ? "was" : "were"} merged into the base URL.`] : [],
|
|
29069
|
+
...selection.flaggedRows ? [`${selection.flaggedRows} eligible ${plural$6(selection.flaggedRows, "row")} carry heuristic query flags (likely-automated or navigational-other) and rank after unflagged rows.`] : [],
|
|
28986
29070
|
"CTR benchmarks and calculated click shortfalls are directional heuristics, not forecasts or proof that clicks are available. Validate the search intent and live SERP before editing.",
|
|
28987
|
-
...selection.invalidRows ? [`${selection.invalidRows} invalid provider ${plural$
|
|
28988
|
-
...selection.duplicateRows ? [`${selection.duplicateRows} repeated query/page ${plural$
|
|
29071
|
+
...selection.invalidRows ? [`${selection.invalidRows} invalid provider ${plural$6(selection.invalidRows, "row")} ${selection.invalidRows === 1 ? "was" : "were"} excluded, so this report is partial.`] : [],
|
|
29072
|
+
...selection.duplicateRows ? [`${selection.duplicateRows} repeated query/page ${plural$6(selection.duplicateRows, "row")} ${selection.duplicateRows === 1 ? "was" : "were"} aggregated before CTR and average position were calculated.`] : [],
|
|
28989
29073
|
...possiblyTruncated ? [`The retained-row limit of ${MAX_GSC_ROWS$3} was reached, so lower-ranked source rows may be absent.`] : [],
|
|
28990
|
-
...selection.limitedUnderperformers ? [`${selection.limitedUnderperformers} eligible ${plural$
|
|
29074
|
+
...selection.limitedUnderperformers ? [`${selection.limitedUnderperformers} eligible ${plural$6(selection.limitedUnderperformers, "underperformer")} ${selection.limitedUnderperformers === 1 ? "was" : "were"} omitted by the output limit.`] : []
|
|
28991
29075
|
],
|
|
28992
29076
|
recommendations: top ? [
|
|
28993
29077
|
`Review the live SERP and displayed snippet for "${top.query}" first. Test the title or meta description only if the observed framing does not match the query intent.`,
|
|
@@ -29007,11 +29091,11 @@ function ctrUnderperformersReport(input, dependencies = defaultDependencies$9) {
|
|
|
29007
29091
|
}
|
|
29008
29092
|
//#endregion
|
|
29009
29093
|
//#region packages/core/dist/phrasing.js
|
|
29010
|
-
function plural$
|
|
29094
|
+
function plural$5(count, singular, pluralLabel = `${singular}s`) {
|
|
29011
29095
|
return count === 1 ? singular : pluralLabel;
|
|
29012
29096
|
}
|
|
29013
29097
|
function countLabel(count, singular, pluralLabel = `${singular}s`) {
|
|
29014
|
-
return `${count.toLocaleString("en-GB")} ${plural$
|
|
29098
|
+
return `${count.toLocaleString("en-GB")} ${plural$5(count, singular, pluralLabel)}`;
|
|
29015
29099
|
}
|
|
29016
29100
|
//#endregion
|
|
29017
29101
|
//#region packages/core/dist/analyze/diagnosis-priorities.js
|
|
@@ -32770,7 +32854,7 @@ function anomalyEvidence(anomaly) {
|
|
|
32770
32854
|
if (anomaly.direction === "normal") return `${formatMetric(anomaly.metric)} changed ${changePhrase} (${anomaly.baselineMean.toLocaleString("en-GB")} -> ${anomaly.comparisonMean.toLocaleString("en-GB")} per observed day${anomaly.zScore === null ? "; the baseline had no observed variance" : `, z ${anomaly.zScore}`}).`;
|
|
32771
32855
|
return `${formatMetric(anomaly.metric)} ${direction} ${changePhrase} (${anomaly.baselineMean.toLocaleString("en-GB")} -> ${anomaly.comparisonMean.toLocaleString("en-GB")} per observed day${anomaly.zScore === null ? "; outside the flat observed baseline" : `, z ${anomaly.zScore}`}).`;
|
|
32772
32856
|
}
|
|
32773
|
-
function plural$
|
|
32857
|
+
function plural$4(count, singular, pluralLabel = `${singular}s`) {
|
|
32774
32858
|
return count === 1 ? singular : pluralLabel;
|
|
32775
32859
|
}
|
|
32776
32860
|
function strongestSignal(anomalies) {
|
|
@@ -32826,13 +32910,13 @@ function interpretUpdateCorrelation(input) {
|
|
|
32826
32910
|
summary,
|
|
32827
32911
|
evidence: [
|
|
32828
32912
|
...input.anomalies.map(anomalyEvidence),
|
|
32829
|
-
hasUpdate ? `${input.overlappingUpdates.length} official Google Ranking update ${plural$
|
|
32913
|
+
hasUpdate ? `${input.overlappingUpdates.length} official Google Ranking update ${plural$4(input.overlappingUpdates.length, "window")} overlapped the change date or comparison window: ${updateNames.join(", ")}.` : "No official Google Ranking update windows overlapped the change date or comparison window.",
|
|
32830
32914
|
largeMovement ? "Both clicks and impressions moved sharply in the same direction." : "Clicks and impressions did not both meet the sharp-movement evidence threshold.",
|
|
32831
32915
|
hasConfounders ? `Known overlapping site changes: ${confounders.map((confounder) => confounder.title).join("; ")}.` : "No overlapping site changes were supplied."
|
|
32832
32916
|
],
|
|
32833
32917
|
caveats: [
|
|
32834
32918
|
`GSC window: ${input.days} days; recent comparison window: ${input.recentDays} days.`,
|
|
32835
|
-
`Update overlap padding: ${input.paddingDays} ${plural$
|
|
32919
|
+
`Update overlap padding: ${input.paddingDays} ${plural$4(input.paddingDays, "day")} either side of the step-change date, or of the recent comparison window when no step change was found.`,
|
|
32836
32920
|
input.refresh ? "Data freshness: cache bypassed for GSC data." : "Data freshness: local GSC cache may have been used.",
|
|
32837
32921
|
"Source: official Google Search Status Dashboard Ranking incidents only; third-party volatility and unconfirmed chatter are not included.",
|
|
32838
32922
|
"An update-window overlap cannot establish causation, even when movement is large and no known site change was recorded.",
|
|
@@ -33000,7 +33084,7 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
33000
33084
|
if (step && changeDate) {
|
|
33001
33085
|
const previousDate = aggregated.rows[step.index - 1]?.keys[0];
|
|
33002
33086
|
const gap = previousDate ? calendarDays(previousDate, changeDate) - 2 : 0;
|
|
33003
|
-
stepCaveats.push(`${formatMetric(metric)} changed level on ${changeDate}, so its baseline is every observed day before that date and its comparison is every observed day from it.`, ...gap > 0 ? [`The ${metric} change date follows ${gap} missing ${plural$
|
|
33087
|
+
stepCaveats.push(`${formatMetric(metric)} changed level on ${changeDate}, so its baseline is every observed day before that date and its comparison is every observed day from it.`, ...gap > 0 ? [`The ${metric} change date follows ${gap} missing ${plural$4(gap, "day")}; the change may have started on one of them.`] : []);
|
|
33004
33088
|
return compareSegments({
|
|
33005
33089
|
site: input.site,
|
|
33006
33090
|
metric,
|
|
@@ -33031,9 +33115,9 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
33031
33115
|
const comparisonMissingDays = input.recentDays - comparisonRows.length;
|
|
33032
33116
|
const missingDays = expectedDays - aggregated.rows.length;
|
|
33033
33117
|
const caveats = [
|
|
33034
|
-
...missingDays > 0 ? [`${missingDays} requested calendar ${plural$
|
|
33035
|
-
...aggregated.invalidRows > 0 ? [`${aggregated.invalidRows} invalid provider ${plural$
|
|
33036
|
-
...aggregated.duplicateRows > 0 ? [`${aggregated.duplicateRows} duplicate date ${plural$
|
|
33118
|
+
...missingDays > 0 ? [`${missingDays} requested calendar ${plural$4(missingDays, "day")} had no returned date aggregate and ${missingDays === 1 ? "was" : "were"} not filled with zeros.`] : [],
|
|
33119
|
+
...aggregated.invalidRows > 0 ? [`${aggregated.invalidRows} invalid provider ${plural$4(aggregated.invalidRows, "row")} ${aggregated.invalidRows === 1 ? "was" : "were"} excluded.`] : [],
|
|
33120
|
+
...aggregated.duplicateRows > 0 ? [`${aggregated.duplicateRows} duplicate date ${plural$4(aggregated.duplicateRows, "row")} ${aggregated.duplicateRows === 1 ? "was" : "were"} aggregated before analysis.`] : [],
|
|
33037
33121
|
...stepCaveats
|
|
33038
33122
|
];
|
|
33039
33123
|
return {
|
|
@@ -34114,7 +34198,7 @@ const VOID = /* @__PURE__ */ new Set([
|
|
|
34114
34198
|
"col",
|
|
34115
34199
|
"embed"
|
|
34116
34200
|
]);
|
|
34117
|
-
const ENTITIES = {
|
|
34201
|
+
const ENTITIES$1 = {
|
|
34118
34202
|
amp: "&",
|
|
34119
34203
|
lt: "<",
|
|
34120
34204
|
gt: ">",
|
|
@@ -34139,7 +34223,7 @@ function decode(text) {
|
|
|
34139
34223
|
const point = Number.parseInt(code.slice(hex ? 2 : 1), hex ? 16 : 10);
|
|
34140
34224
|
return point > 0 && point <= 1114111 ? String.fromCodePoint(point) : "";
|
|
34141
34225
|
}
|
|
34142
|
-
return ENTITIES[code.toLowerCase()] ?? match;
|
|
34226
|
+
return ENTITIES$1[code.toLowerCase()] ?? match;
|
|
34143
34227
|
});
|
|
34144
34228
|
}
|
|
34145
34229
|
const clean$1 = (value) => value.split(A_OPEN).join("").replace(/\s+/g, " ").trim();
|
|
@@ -34379,7 +34463,7 @@ var SentenceStore = class SentenceStore {
|
|
|
34379
34463
|
});
|
|
34380
34464
|
}
|
|
34381
34465
|
};
|
|
34382
|
-
const bareHost = (host) => host.toLowerCase().replace(/^www\./, "");
|
|
34466
|
+
const bareHost$1 = (host) => host.toLowerCase().replace(/^www\./, "");
|
|
34383
34467
|
function crawlUrl(value, base) {
|
|
34384
34468
|
try {
|
|
34385
34469
|
const url = base ? new URL(value, base) : new URL(value);
|
|
@@ -34445,10 +34529,10 @@ async function crawlForLinks(input) {
|
|
|
34445
34529
|
async function crawl(input, robots, sentences, renderer, warnings) {
|
|
34446
34530
|
const start = crawlUrl(input.startUrl) ?? input.startUrl;
|
|
34447
34531
|
let origin = new URL(start).origin;
|
|
34448
|
-
const site = bareHost(new URL(start).host);
|
|
34532
|
+
const site = bareHost$1(new URL(start).host);
|
|
34449
34533
|
const sameSite = (url) => {
|
|
34450
34534
|
try {
|
|
34451
|
-
return bareHost(new URL(url).host) === site;
|
|
34535
|
+
return bareHost$1(new URL(url).host) === site;
|
|
34452
34536
|
} catch {
|
|
34453
34537
|
return false;
|
|
34454
34538
|
}
|
|
@@ -34942,15 +35026,15 @@ function internalLinksReportRange(input, fallbackDays, now) {
|
|
|
34942
35026
|
days: Math.round((Date.parse(`${endDate}T00:00:00Z`) - Date.parse(`${startDate}T00:00:00Z`)) / DAY$1) + 1
|
|
34943
35027
|
};
|
|
34944
35028
|
}
|
|
34945
|
-
const plural$
|
|
35029
|
+
const plural$3 = (count, word) => `${count} ${word}${count === 1 ? "" : "s"}`;
|
|
34946
35030
|
function internalLinksVerdict(report) {
|
|
34947
35031
|
const { counts } = report;
|
|
34948
35032
|
if (report.dataStatus === "blocked") return "robots.txt does not allow this crawler to read the site, so nothing was checked.";
|
|
34949
35033
|
if (report.dataStatus === "empty") return "No HTML page could be crawled, so there is nothing to link.";
|
|
34950
35034
|
if (report.dataStatus === "target-not-crawled") return "The target page could not be crawled as its own page (it failed, left the site, or names another page as canonical). Check it with seo page first.";
|
|
34951
35035
|
if (report.dataStatus === "target-not-linkable") return `The target is ${report.target?.noindex ? "noindex" : `a ${report.target?.type?.replaceAll("_", " ")} page`}, so no links to it are suggested.`;
|
|
34952
|
-
const judged = report.judge.status === "used" || report.judge.status === "partial" ? `${counts.strong} strong and ${counts.possible} possible link${counts.strong + counts.possible === 1 ? "" : "s"} judged by ${report.judge.model}${counts.heuristic ? `, plus ${plural$
|
|
34953
|
-
const scope = report.mode === "url" ? `to ${report.target?.url}` : `across ${plural$
|
|
35036
|
+
const judged = report.judge.status === "used" || report.judge.status === "partial" ? `${counts.strong} strong and ${counts.possible} possible link${counts.strong + counts.possible === 1 ? "" : "s"} judged by ${report.judge.model}${counts.heuristic ? `, plus ${plural$3(counts.heuristic, "heuristic link")} where a request failed` : ""}` : `${plural$3(counts.heuristic, "heuristic link")} (no decision model)`;
|
|
35037
|
+
const scope = report.mode === "url" ? `to ${report.target?.url}` : `across ${plural$3(counts.pagesCrawled, "page")}`;
|
|
34954
35038
|
const capped = report.crawl.capReached ? ` The crawl stopped at its limit of ${report.crawl.pageLimit} pages, so pages beyond it were not checked.` : "";
|
|
34955
35039
|
return `${judged} ${scope}${counts.heldBack ? `; ${counts.heldBack} held back by the evidence floors` : ""}.${capped}`;
|
|
34956
35040
|
}
|
|
@@ -36851,7 +36935,7 @@ function reportId$1(input) {
|
|
|
36851
36935
|
})).digest("hex").slice(0, 20)}`;
|
|
36852
36936
|
}
|
|
36853
36937
|
function cacheGet(id, now) {
|
|
36854
|
-
const row =
|
|
36938
|
+
const row = cacheDb().prepare("SELECT report_json FROM performance_reports WHERE id = ? AND expires_at > ?").get(id, now);
|
|
36855
36939
|
if (!row?.report_json) return void 0;
|
|
36856
36940
|
try {
|
|
36857
36941
|
const report = JSON.parse(row.report_json);
|
|
@@ -36866,10 +36950,12 @@ function cacheSet(report, now) {
|
|
|
36866
36950
|
raw: void 0
|
|
36867
36951
|
};
|
|
36868
36952
|
const reportJson = JSON.stringify(stored);
|
|
36869
|
-
|
|
36870
|
-
|
|
36871
|
-
|
|
36872
|
-
|
|
36953
|
+
writeToCache((database) => {
|
|
36954
|
+
database.prepare(`INSERT OR REPLACE INTO performance_reports
|
|
36955
|
+
(id, url, strategy, report_json, created_at, expires_at)
|
|
36956
|
+
VALUES (?, ?, ?, ?, ?, ?)`).run(report.id, report.url, report.strategy, reportJson, Date.parse(report.generatedAt), now + CACHE_TTL_MS);
|
|
36957
|
+
noteCacheWrite(Buffer.byteLength(reportJson), database);
|
|
36958
|
+
});
|
|
36873
36959
|
}
|
|
36874
36960
|
async function fallbackReport(input) {
|
|
36875
36961
|
const fetched = await fetchPage(input.url, {
|
|
@@ -37386,14 +37472,14 @@ function* retainedQueryClusterRows(input) {
|
|
|
37386
37472
|
};
|
|
37387
37473
|
}
|
|
37388
37474
|
}
|
|
37389
|
-
function plural$
|
|
37475
|
+
function plural$2(count, singular, pluralLabel = `${singular}s`) {
|
|
37390
37476
|
return count === 1 ? singular : pluralLabel;
|
|
37391
37477
|
}
|
|
37392
37478
|
function reportVerdict(input) {
|
|
37393
37479
|
if (!input.clusters.length) return "No query clusters were generated for this scope.";
|
|
37394
37480
|
if (input.highOpportunityClusters > 0) {
|
|
37395
37481
|
const verb = input.highOpportunityClusters === 1 ? "has" : "have";
|
|
37396
|
-
return `${input.highOpportunityClusters} ${plural$
|
|
37482
|
+
return `${input.highOpportunityClusters} ${plural$2(input.highOpportunityClusters, "cluster")} ${verb} enough demand and weak enough performance to review first.`;
|
|
37397
37483
|
}
|
|
37398
37484
|
return "No obvious high-opportunity clusters were found. Use this report for content structure and intent mapping rather than urgent fixes.";
|
|
37399
37485
|
}
|
|
@@ -37534,7 +37620,7 @@ function movementLine(report) {
|
|
|
37534
37620
|
}
|
|
37535
37621
|
function countPhrase(count, singular, pluralLabel = `${singular}s`) {
|
|
37536
37622
|
if (count === 0) return `no ${pluralLabel}`;
|
|
37537
|
-
return `${count} ${plural(count, singular, pluralLabel)}`;
|
|
37623
|
+
return `${count} ${plural$1(count, singular, pluralLabel)}`;
|
|
37538
37624
|
}
|
|
37539
37625
|
function headlineLine(report) {
|
|
37540
37626
|
const parts = [
|
|
@@ -37589,7 +37675,7 @@ function decayDiagnosisLabel(diagnosis) {
|
|
|
37589
37675
|
if (diagnosis === "lost_clicks") return "clicks dropped without one dominant signal";
|
|
37590
37676
|
return diagnosis.replaceAll("_", " ");
|
|
37591
37677
|
}
|
|
37592
|
-
function plural(count, singular, pluralLabel = `${singular}s`) {
|
|
37678
|
+
function plural$1(count, singular, pluralLabel = `${singular}s`) {
|
|
37593
37679
|
return count === 1 ? singular : pluralLabel;
|
|
37594
37680
|
}
|
|
37595
37681
|
function decayClusterLine(report) {
|
|
@@ -37597,7 +37683,7 @@ function decayClusterLine(report) {
|
|
|
37597
37683
|
if (!group) return "No material decay cluster stood out.";
|
|
37598
37684
|
const diagnosis = decayDiagnosisLabel(group.diagnosis);
|
|
37599
37685
|
const samples = group.sampleQueries.slice(0, 2).join("; ");
|
|
37600
|
-
return `Top decay cluster: ${group.count} ${plural(group.count, "finding")} in the ${group.template.label} template (${diagnosis}), ${group.totalClickLoss.toFixed(0)} fewer observed clicks across rows retained in both windows. Action: ${group.recommendation}${samples ? ` Example queries: ${samples}.` : ""}`;
|
|
37686
|
+
return `Top decay cluster: ${group.count} ${plural$1(group.count, "finding")} in the ${group.template.label} template (${diagnosis}), ${group.totalClickLoss.toFixed(0)} fewer observed clicks across rows retained in both windows. Action: ${group.recommendation}${samples ? ` Example queries: ${samples}.` : ""}`;
|
|
37601
37687
|
}
|
|
37602
37688
|
function cannibalSuppressionLine(report) {
|
|
37603
37689
|
const suppressed = report.cannibalization.selection.suppressedQueries;
|
|
@@ -37615,21 +37701,21 @@ function contentOpportunityBullets(report) {
|
|
|
37615
37701
|
if (unavailableSections.length) bullets.push(`Could not assess ${unavailableSections.join(", ")} opportunities in this run; see report caveats.`);
|
|
37616
37702
|
if (report.summary.decayItems) {
|
|
37617
37703
|
const top = report.decay.items[0];
|
|
37618
|
-
bullets.push(top ? `${report.summary.decayItems} observed retained query/page ${plural(report.summary.decayItems, "decline")} need review. Start with "${top.query}" on ${top.url}; it had ${top.clickLoss.toFixed(0)} fewer clicks.` : `${report.summary.decayItems} observed retained query/page ${plural(report.summary.decayItems, "decline")} need review.`);
|
|
37704
|
+
bullets.push(top ? `${report.summary.decayItems} observed retained query/page ${plural$1(report.summary.decayItems, "decline")} need review. Start with "${top.query}" on ${top.url}; it had ${top.clickLoss.toFixed(0)} fewer clicks.` : `${report.summary.decayItems} observed retained query/page ${plural$1(report.summary.decayItems, "decline")} need review.`);
|
|
37619
37705
|
bullets.push(decayClusterLine(report));
|
|
37620
37706
|
}
|
|
37621
37707
|
if (report.summary.cannibalItems) {
|
|
37622
37708
|
const top = report.cannibalization.items[0];
|
|
37623
|
-
bullets.push(top ? `${report.summary.cannibalItems} multi-URL query ${plural(report.summary.cannibalItems, "candidate")} need intent and technical review. Start with "${top.query}" and decide whether ${top.pages.length} ranking URLs answer the same intent.` : `${report.summary.cannibalItems} multi-URL query ${plural(report.summary.cannibalItems, "candidate")} need intent and technical review.`);
|
|
37709
|
+
bullets.push(top ? `${report.summary.cannibalItems} multi-URL query ${plural$1(report.summary.cannibalItems, "candidate")} need intent and technical review. Start with "${top.query}" and decide whether ${top.pages.length} ranking URLs answer the same intent.` : `${report.summary.cannibalItems} multi-URL query ${plural$1(report.summary.cannibalItems, "candidate")} need intent and technical review.`);
|
|
37624
37710
|
}
|
|
37625
37711
|
if (report.cannibalization.selection.suppressedQueries) bullets.push(cannibalSuppressionLine(report));
|
|
37626
37712
|
if (report.summary.strikingDistanceItems) {
|
|
37627
37713
|
const top = report.strikingDistance.items[0];
|
|
37628
|
-
bullets.push(top ? `${countLabel(report.summary.strikingDistanceItems, "query/page candidate")} ${plural(report.summary.strikingDistanceItems, "has", "have")} an average GSC position above 10 and at most 20. Start by investigating "${top.query}" on ${top.url}; it averages position ${top.position.toFixed(1)} with ${top.impressions.toFixed(0)} impressions.` : `${countLabel(report.summary.strikingDistanceItems, "query/page candidate")} ${plural(report.summary.strikingDistanceItems, "has", "have")} an average GSC position above 10 and at most 20.`);
|
|
37714
|
+
bullets.push(top ? `${countLabel(report.summary.strikingDistanceItems, "query/page candidate")} ${plural$1(report.summary.strikingDistanceItems, "has", "have")} an average GSC position above 10 and at most 20. Start by investigating "${top.query}" on ${top.url}; it averages position ${top.position.toFixed(1)} with ${top.impressions.toFixed(0)} impressions.` : `${countLabel(report.summary.strikingDistanceItems, "query/page candidate")} ${plural$1(report.summary.strikingDistanceItems, "has", "have")} an average GSC position above 10 and at most 20.`);
|
|
37629
37715
|
}
|
|
37630
37716
|
if (!skippedReason(report, "quick-win opportunities") && report.summary.quickWinItems) {
|
|
37631
37717
|
const top = report.quickWins.items[0];
|
|
37632
|
-
bullets.push(top ? `${report.summary.quickWinItems} retained query/page ${plural(report.summary.quickWinItems, "candidate")} fell below a site-peer or versioned heuristic CTR target. Start with "${top.query}" on ${top.url}; it had ${top.impressions.toFixed(0)} impressions at average position ${top.position.toFixed(1)}.` : `${report.summary.quickWinItems} CTR-target ${plural(report.summary.quickWinItems, "candidate")} need review.`);
|
|
37718
|
+
bullets.push(top ? `${report.summary.quickWinItems} retained query/page ${plural$1(report.summary.quickWinItems, "candidate")} fell below a site-peer or versioned heuristic CTR target. Start with "${top.query}" on ${top.url}; it had ${top.impressions.toFixed(0)} impressions at average position ${top.position.toFixed(1)}.` : `${report.summary.quickWinItems} CTR-target ${plural$1(report.summary.quickWinItems, "candidate")} need review.`);
|
|
37633
37719
|
}
|
|
37634
37720
|
const templateLine = templateOpportunityLine(report);
|
|
37635
37721
|
if (!templateLine.startsWith("No dominant")) bullets.push(templateLine);
|
|
@@ -37989,10 +38075,10 @@ function applyVerification(item, coverage) {
|
|
|
37989
38075
|
function verdict(report) {
|
|
37990
38076
|
if (report.dataStatus === "empty") return "Google Search Console returned no retained query/page rows for this window.";
|
|
37991
38077
|
if (report.dataStatus === "filtered") return "No pages met the position, brand, and minimum-impression criteria.";
|
|
37992
|
-
if (report.technical > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.technical, "returned page")} ${plural$
|
|
37993
|
-
if (report.failed > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.failed, "returned page")} could not be verified and ${plural$
|
|
37994
|
-
if (report.content > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.content, "returned page")} ${plural$
|
|
37995
|
-
return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.returned, "page")} ${plural$
|
|
38078
|
+
if (report.technical > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.technical, "returned page")} ${plural$5(report.technical, "has", "have")} verified technical issues to fix before content changes.`;
|
|
38079
|
+
if (report.failed > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.failed, "returned page")} could not be verified and ${plural$5(report.failed, "needs", "need")} a fetch check before content changes.`;
|
|
38080
|
+
if (report.content > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.content, "returned page")} ${plural$5(report.content, "has", "have")} verified content or search-framing gaps.`;
|
|
38081
|
+
return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.returned, "page")} ${plural$5(report.returned, "is", "are")} returned in priority order for investigation.`;
|
|
37996
38082
|
}
|
|
37997
38083
|
function reportRecommendations(items) {
|
|
37998
38084
|
const first = items[0];
|
|
@@ -39053,7 +39139,7 @@ async function updatePostmortemWorkflow(input, dependencies = {}) {
|
|
|
39053
39139
|
//#endregion
|
|
39054
39140
|
//#region packages/core/dist/clicky/api.js
|
|
39055
39141
|
const ENDPOINT$1 = "https://api.clicky.com/api/stats/4";
|
|
39056
|
-
const MAX_BYTES$
|
|
39142
|
+
const MAX_BYTES$3 = 5e6;
|
|
39057
39143
|
var ClickyError = class extends Error {
|
|
39058
39144
|
code;
|
|
39059
39145
|
constructor(code, message) {
|
|
@@ -39080,7 +39166,7 @@ async function clickyStats(site, types, options = {}) {
|
|
|
39080
39166
|
throw new ClickyError("remote", "Could not reach Clicky. Try again.");
|
|
39081
39167
|
});
|
|
39082
39168
|
if (!response.ok) throw new ClickyError("remote", `Clicky returned ${response.status}.`);
|
|
39083
|
-
const groups = await readJson(response, "Clicky", { maxBytes: MAX_BYTES$
|
|
39169
|
+
const groups = await readJson(response, "Clicky", { maxBytes: MAX_BYTES$3 });
|
|
39084
39170
|
if (!Array.isArray(groups)) throw new ClickyError("remote", "Clicky returned invalid data.");
|
|
39085
39171
|
const failed = groups.find((group) => typeof group?.error === "string");
|
|
39086
39172
|
if (failed?.error) throw /sitekey|site id|authenticat|permission/iu.test(failed.error) ? new ClickyError("auth", `Clicky rejected site ID ${site.siteId} or its sitekey.`) : new ClickyError("remote", `Clicky: ${failed.error.slice(0, 200)}`);
|
|
@@ -40835,6 +40921,392 @@ async function withDomainRating(rows, key) {
|
|
|
40835
40921
|
added: true
|
|
40836
40922
|
};
|
|
40837
40923
|
}
|
|
40924
|
+
const MAX_INSPECT_LIMIT = 2e3;
|
|
40925
|
+
const CONCURRENCY = 20;
|
|
40926
|
+
const REFERRING_KEPT = 10;
|
|
40927
|
+
const LABELS = {
|
|
40928
|
+
fix: "the evidence shows a fault on the site",
|
|
40929
|
+
check: "a person should look; the evidence alone does not settle it",
|
|
40930
|
+
expected: "the evidence shows the site works as built, nothing to fix",
|
|
40931
|
+
indexed: "Google indexes it and the live check shows no fault"
|
|
40932
|
+
};
|
|
40933
|
+
const AUTH_SEGMENTS = /* @__PURE__ */ new Set([
|
|
40934
|
+
"account",
|
|
40935
|
+
"admin",
|
|
40936
|
+
"api",
|
|
40937
|
+
"auth",
|
|
40938
|
+
"cart",
|
|
40939
|
+
"checkout",
|
|
40940
|
+
"dashboard",
|
|
40941
|
+
"login",
|
|
40942
|
+
"logout",
|
|
40943
|
+
"my-account",
|
|
40944
|
+
"oauth",
|
|
40945
|
+
"sign-in",
|
|
40946
|
+
"signin",
|
|
40947
|
+
"wp-admin",
|
|
40948
|
+
"wp-login.php"
|
|
40949
|
+
]);
|
|
40950
|
+
const bareHost = (url) => new URL(url).hostname.replace(/^www\./, "");
|
|
40951
|
+
const sameSite = (a, b) => {
|
|
40952
|
+
try {
|
|
40953
|
+
return bareHost(a) === bareHost(b);
|
|
40954
|
+
} catch {
|
|
40955
|
+
return false;
|
|
40956
|
+
}
|
|
40957
|
+
};
|
|
40958
|
+
function variantOf(url, target) {
|
|
40959
|
+
const from = new URL(url);
|
|
40960
|
+
const to = new URL(target);
|
|
40961
|
+
if (from.pathname !== to.pathname || from.search !== to.search) return null;
|
|
40962
|
+
if (!sameSite(url, target)) return null;
|
|
40963
|
+
if (from.protocol !== to.protocol) return "http:// variant";
|
|
40964
|
+
if (from.host !== to.host) return `host variant (${from.host})`;
|
|
40965
|
+
return null;
|
|
40966
|
+
}
|
|
40967
|
+
const samePath = (a, b) => {
|
|
40968
|
+
const [from, to] = [new URL(a), new URL(b)];
|
|
40969
|
+
return from.pathname === to.pathname && from.search === to.search;
|
|
40970
|
+
};
|
|
40971
|
+
const authPath = (url) => AUTH_SEGMENTS.has(new URL(url).pathname.split("/")[1]?.toLowerCase() ?? "");
|
|
40972
|
+
const plural = (count, word) => `${count} ${word}${count === 1 ? "" : "s"}`;
|
|
40973
|
+
function labelOf(e) {
|
|
40974
|
+
const { url, status, live } = e;
|
|
40975
|
+
const { first, hops, last } = live;
|
|
40976
|
+
const indexed = status.verdict === "PASS";
|
|
40977
|
+
const coverage = status.coverage ?? "";
|
|
40978
|
+
const referring = status.referring_urls.filter((item) => item !== url && sameSite(item, url) && !samePath(item, url));
|
|
40979
|
+
const pointers = [e.inSitemap ? "the sitemap lists it" : null, referring.length ? `Google lists ${plural(referring.length, "referring page")} on this site as of its last crawl (${referring[0]}${referring.length > 1 ? ", and more" : ""})` : null].filter(Boolean);
|
|
40980
|
+
if (first.error) return indexed ? {
|
|
40981
|
+
label: "indexed",
|
|
40982
|
+
why: `Google indexes it. The live check failed (${first.error}).`
|
|
40983
|
+
} : {
|
|
40984
|
+
label: "check",
|
|
40985
|
+
why: `The live check failed (${first.error}), so only Google's reason is shown.`
|
|
40986
|
+
};
|
|
40987
|
+
const temporary = hops.find((hop) => hop.kind === "temporary" && new URL(hop.from).host === new URL(hop.to).host);
|
|
40988
|
+
if (temporary) return {
|
|
40989
|
+
label: "fix",
|
|
40990
|
+
why: `${temporary.from} sends a ${temporary.status} (temporary) redirect to ${temporary.to} on the same host; a permanent move uses 301 or 308.`
|
|
40991
|
+
};
|
|
40992
|
+
if (!last) return {
|
|
40993
|
+
label: "fix",
|
|
40994
|
+
why: `The redirects loop or pass 5 hops (${hops.map((hop) => hop.status).join(" > ")}).`
|
|
40995
|
+
};
|
|
40996
|
+
if (last.error) return {
|
|
40997
|
+
label: "check",
|
|
40998
|
+
why: `It redirects to ${last.url}, which could not be fetched (${last.error}).`
|
|
40999
|
+
};
|
|
41000
|
+
const end = last.status ?? 0;
|
|
41001
|
+
const now = hops.length ? `redirects to ${last.url}, which returns ${end}` : `returns ${end} now`;
|
|
41002
|
+
const via = `It ${now}`;
|
|
41003
|
+
if (end === 404 || end === 410) {
|
|
41004
|
+
if (pointers.length) return {
|
|
41005
|
+
label: "fix",
|
|
41006
|
+
why: `${via}, and ${pointers.join(" and ")}.`
|
|
41007
|
+
};
|
|
41008
|
+
if (hops.length) return {
|
|
41009
|
+
label: "fix",
|
|
41010
|
+
why: `${via}: a redirect to a missing page.`
|
|
41011
|
+
};
|
|
41012
|
+
return {
|
|
41013
|
+
label: "expected",
|
|
41014
|
+
why: `${via}, and neither the sitemap nor any referring page Google lists on this site points to it.`
|
|
41015
|
+
};
|
|
41016
|
+
}
|
|
41017
|
+
if ((end === 401 || end === 403) && !hops.length) {
|
|
41018
|
+
if (e.inSitemap) return {
|
|
41019
|
+
label: "fix",
|
|
41020
|
+
why: `${via}, but the sitemap lists it.`
|
|
41021
|
+
};
|
|
41022
|
+
if (end === 401 && first.www_authenticate) return {
|
|
41023
|
+
label: "expected",
|
|
41024
|
+
why: `${via} with a WWW-Authenticate header (${first.www_authenticate.split(/\s/)[0]}), so the server asks for sign-in here.`
|
|
41025
|
+
};
|
|
41026
|
+
if (authPath(url)) return {
|
|
41027
|
+
label: "expected",
|
|
41028
|
+
why: `${via} on a path that looks like a sign-in area (heuristic: from the path only).`
|
|
41029
|
+
};
|
|
41030
|
+
return {
|
|
41031
|
+
label: "check",
|
|
41032
|
+
why: `${via} with no WWW-Authenticate header and no sign-in path, so the evidence does not show whether it should be public.`
|
|
41033
|
+
};
|
|
41034
|
+
}
|
|
41035
|
+
if (hops.length) {
|
|
41036
|
+
const codes = hops.map((hop) => hop.status).join(" > ");
|
|
41037
|
+
if (end !== 200) return {
|
|
41038
|
+
label: "check",
|
|
41039
|
+
why: `${via} (${codes}).`
|
|
41040
|
+
};
|
|
41041
|
+
if (e.inSitemap) return {
|
|
41042
|
+
label: "fix",
|
|
41043
|
+
why: `The sitemap lists it, but it redirects (${codes}) to ${last.url}.`
|
|
41044
|
+
};
|
|
41045
|
+
const declared = last.canonical === last.url ? "declares itself canonical" : e.sitemap.has(last.url) ? "is in the sitemap" : null;
|
|
41046
|
+
const variant = variantOf(url, last.url);
|
|
41047
|
+
const moved = `${variant ? `An ${variant}` : "A permanent redirect"}: ${codes} to ${last.url}`;
|
|
41048
|
+
const stale = indexed ? " Google still indexes this URL as of its last crawl." : "";
|
|
41049
|
+
const links = referring.length ? ` ${pointers.at(-1)}.` : "";
|
|
41050
|
+
if (declared) return {
|
|
41051
|
+
label: "expected",
|
|
41052
|
+
why: `${moved}, which returns 200 and ${declared}.${stale}${links}`
|
|
41053
|
+
};
|
|
41054
|
+
return {
|
|
41055
|
+
label: "check",
|
|
41056
|
+
why: `${moved}, which declares ${last.canonical ? `${last.canonical} as canonical` : "no canonical"}.${stale}${links}`
|
|
41057
|
+
};
|
|
41058
|
+
}
|
|
41059
|
+
if (/^(Crawled|Discovered) - currently not indexed$|^URL is unknown to Google$/i.test(coverage)) return {
|
|
41060
|
+
label: "check",
|
|
41061
|
+
why: `${coverage.startsWith("Crawled") ? "Google crawled it but has not indexed it, and gives no further reason" : coverage.startsWith("Discovered") ? "Google knows the URL but has not crawled it yet" : "Google has no record of this URL"}.${e.inSitemap ? " The sitemap lists it." : ""} It returns ${end} now.`
|
|
41062
|
+
};
|
|
41063
|
+
const robots = last.robots ? ` with robots "${last.robots}"` : " with no robots meta tag or X-Robots-Tag header";
|
|
41064
|
+
if (/noindex/i.test(coverage) && !/noindex/i.test(last.robots ?? "")) return {
|
|
41065
|
+
label: "check",
|
|
41066
|
+
why: `Google saw noindex when it last crawled${status.last_crawl ? ` (${status.last_crawl.slice(0, 10)})` : ""}, but it ${now}${robots}, so Google's reason may be out of date.${e.inSitemap ? " The sitemap lists it." : ""}`
|
|
41067
|
+
};
|
|
41068
|
+
const declaredCanonical = status.user_canonical ?? last.canonical;
|
|
41069
|
+
if (status.google_canonical && declaredCanonical && status.google_canonical !== declaredCanonical) return {
|
|
41070
|
+
label: "check",
|
|
41071
|
+
why: `Google chose ${status.google_canonical} as canonical; the page declares ${declaredCanonical}.`
|
|
41072
|
+
};
|
|
41073
|
+
if (indexed) return end === 200 ? {
|
|
41074
|
+
label: "indexed",
|
|
41075
|
+
why: "Google indexes it and it returns 200 now."
|
|
41076
|
+
} : {
|
|
41077
|
+
label: "check",
|
|
41078
|
+
why: `Google indexes it, but it ${now}.`
|
|
41079
|
+
};
|
|
41080
|
+
if (e.inSitemap) return {
|
|
41081
|
+
label: "fix",
|
|
41082
|
+
why: `The sitemap lists it, but Google does not index it. It ${now}${robots}.`
|
|
41083
|
+
};
|
|
41084
|
+
if (status.google_canonical && status.google_canonical === status.user_canonical && status.google_canonical !== url) return {
|
|
41085
|
+
label: "expected",
|
|
41086
|
+
why: `The page declares ${status.user_canonical} as canonical and Google uses it.`
|
|
41087
|
+
};
|
|
41088
|
+
return {
|
|
41089
|
+
label: "check",
|
|
41090
|
+
why: /noindex/i.test(last.robots ?? "") ? `${via}${robots} and the sitemap does not list it. Whether it should be indexed is the site's call.` : `${via}${robots}. No rule here settles this reason.`
|
|
41091
|
+
};
|
|
41092
|
+
}
|
|
41093
|
+
function indexingRow(e, sources, inspectedAt) {
|
|
41094
|
+
const { status, live } = e;
|
|
41095
|
+
const { label, why } = labelOf(e);
|
|
41096
|
+
return {
|
|
41097
|
+
reason: status.coverage ?? status.verdict ?? "No reason returned",
|
|
41098
|
+
url: e.url,
|
|
41099
|
+
label,
|
|
41100
|
+
why,
|
|
41101
|
+
verdict: status.verdict,
|
|
41102
|
+
indexing: status.indexing,
|
|
41103
|
+
status: live.first.status,
|
|
41104
|
+
redirects: live.hops.length ? live.hops.map((hop) => `${hop.status} ${hop.kind}`).join(" > ") : null,
|
|
41105
|
+
redirects_to: live.first.redirects_to,
|
|
41106
|
+
final_url: live.last?.url ?? null,
|
|
41107
|
+
final_status: live.last?.status ?? null,
|
|
41108
|
+
page_canonical: live.last?.canonical ?? null,
|
|
41109
|
+
user_canonical: status.user_canonical,
|
|
41110
|
+
google_canonical: status.google_canonical,
|
|
41111
|
+
in_sitemap: e.inSitemap,
|
|
41112
|
+
referring_urls: status.referring_urls.slice(0, REFERRING_KEPT),
|
|
41113
|
+
last_crawl: status.last_crawl,
|
|
41114
|
+
inspected_at: inspectedAt,
|
|
41115
|
+
page_fetch: status.page_fetch,
|
|
41116
|
+
robots_txt: status.robots_txt,
|
|
41117
|
+
crawled_as: status.crawled_as,
|
|
41118
|
+
sources,
|
|
41119
|
+
...live.first.error ? { live_error: live.first.error } : {}
|
|
41120
|
+
};
|
|
41121
|
+
}
|
|
41122
|
+
const byText = (a, b) => a < b ? -1 : a > b ? 1 : 0;
|
|
41123
|
+
function sortRows(rows) {
|
|
41124
|
+
return [...rows].sort((a, b) => byText(a.reason, b.reason) || byText(a.url, b.url));
|
|
41125
|
+
}
|
|
41126
|
+
function byReason(rows) {
|
|
41127
|
+
const groups = /* @__PURE__ */ new Map();
|
|
41128
|
+
for (const row of rows) {
|
|
41129
|
+
const counts = groups.get(row.reason) ?? {
|
|
41130
|
+
fix: 0,
|
|
41131
|
+
check: 0,
|
|
41132
|
+
expected: 0,
|
|
41133
|
+
indexed: 0
|
|
41134
|
+
};
|
|
41135
|
+
counts[row.label] += 1;
|
|
41136
|
+
groups.set(row.reason, counts);
|
|
41137
|
+
}
|
|
41138
|
+
return [...groups].sort((a, b) => byText(a[0], b[0])).map(([reason, counts]) => ({
|
|
41139
|
+
reason,
|
|
41140
|
+
urls: counts.fix + counts.check + counts.expected + counts.indexed,
|
|
41141
|
+
...counts
|
|
41142
|
+
}));
|
|
41143
|
+
}
|
|
41144
|
+
const isCode = (error, codes) => codes.includes(String(error?.code));
|
|
41145
|
+
function knownUrls(input) {
|
|
41146
|
+
const found = /* @__PURE__ */ new Map();
|
|
41147
|
+
const add = (url, source) => {
|
|
41148
|
+
const sources = found.get(url) ?? [];
|
|
41149
|
+
if (!sources.includes(source)) sources.push(source);
|
|
41150
|
+
found.set(url, sources);
|
|
41151
|
+
};
|
|
41152
|
+
const pages = input.pages.filter((page) => page.impressions > 0 && !page.url.includes("#")).sort((a, b) => b.clicks - a.clicks || b.impressions - a.impressions || byText(a.url, b.url));
|
|
41153
|
+
for (const url of input.lost) add(url, "lost_clicks");
|
|
41154
|
+
for (const page of pages) add(page.url, "search_console");
|
|
41155
|
+
for (const url of input.sitemap.urls) add(url, "sitemap");
|
|
41156
|
+
if (input.home) add(input.home, "home");
|
|
41157
|
+
return {
|
|
41158
|
+
urls: [...found].map(([url, sources]) => ({
|
|
41159
|
+
url,
|
|
41160
|
+
sources
|
|
41161
|
+
})),
|
|
41162
|
+
sitemap: input.sitemap,
|
|
41163
|
+
searchConsole: pages.length,
|
|
41164
|
+
searchConsoleCapped: input.pagesCapped,
|
|
41165
|
+
lostClicks: input.lost.length
|
|
41166
|
+
};
|
|
41167
|
+
}
|
|
41168
|
+
async function inspectKnown(site, known, deps, asked = 100) {
|
|
41169
|
+
const cap = Math.min(Math.max(1, Math.floor(asked)), MAX_INSPECT_LIMIT);
|
|
41170
|
+
const wanted = known.urls.slice(0, cap);
|
|
41171
|
+
let saved = /* @__PURE__ */ new Map();
|
|
41172
|
+
try {
|
|
41173
|
+
saved = deps.savedInspections?.read(site, wanted.map((item) => item.url)) ?? saved;
|
|
41174
|
+
} catch {}
|
|
41175
|
+
const missing = wanted.filter((item) => !saved.has(item.url));
|
|
41176
|
+
const quota = deps.inspectionQuota ? await deps.inspectionQuota(site).catch(() => null) : null;
|
|
41177
|
+
const left = quota ? Math.max(0, quota.limit - quota.used) : missing.length;
|
|
41178
|
+
const asking = new Set(missing.slice(0, left).map((item) => item.url));
|
|
41179
|
+
const queue = wanted.filter((item) => saved.has(item.url) || asking.has(item.url));
|
|
41180
|
+
const quotaShort = left < missing.length;
|
|
41181
|
+
const statuses = /* @__PURE__ */ new Map();
|
|
41182
|
+
const rows = [];
|
|
41183
|
+
const failed = [];
|
|
41184
|
+
const sitemap = new Set(known.sitemap.urls);
|
|
41185
|
+
const now = () => (deps.now?.() ?? /* @__PURE__ */ new Date()).toISOString();
|
|
41186
|
+
let quotaStop = false;
|
|
41187
|
+
let next = 0;
|
|
41188
|
+
let fatal = null;
|
|
41189
|
+
const worker = async () => {
|
|
41190
|
+
while (fatal === null && next < queue.length) {
|
|
41191
|
+
const item = queue[next++];
|
|
41192
|
+
const stored = saved.get(item.url);
|
|
41193
|
+
let status;
|
|
41194
|
+
let inspectedAt;
|
|
41195
|
+
if (stored) {
|
|
41196
|
+
status = stored.status;
|
|
41197
|
+
inspectedAt = stored.inspectedAt;
|
|
41198
|
+
} else {
|
|
41199
|
+
if (quotaStop) continue;
|
|
41200
|
+
try {
|
|
41201
|
+
status = await deps.inspect(site, item.url);
|
|
41202
|
+
inspectedAt = now();
|
|
41203
|
+
} catch (error) {
|
|
41204
|
+
if (isCode(error, ["RATE_LIMITED"])) quotaStop = true;
|
|
41205
|
+
else if (isCode(error, [
|
|
41206
|
+
"AUTH_EXPIRED",
|
|
41207
|
+
"AUTH_REQUIRED",
|
|
41208
|
+
"NOT_CONNECTED"
|
|
41209
|
+
])) fatal = error;
|
|
41210
|
+
else failed.push({
|
|
41211
|
+
url: item.url,
|
|
41212
|
+
error: error instanceof Error ? error.message : String(error)
|
|
41213
|
+
});
|
|
41214
|
+
continue;
|
|
41215
|
+
}
|
|
41216
|
+
try {
|
|
41217
|
+
deps.savedInspections?.save(site, item.url, status);
|
|
41218
|
+
} catch {}
|
|
41219
|
+
}
|
|
41220
|
+
statuses.set(item.url, status);
|
|
41221
|
+
const live = await deps.liveCheck(item.url);
|
|
41222
|
+
rows.push(indexingRow({
|
|
41223
|
+
url: item.url,
|
|
41224
|
+
status,
|
|
41225
|
+
live,
|
|
41226
|
+
inSitemap: item.sources.includes("sitemap"),
|
|
41227
|
+
sitemap
|
|
41228
|
+
}, item.sources, inspectedAt));
|
|
41229
|
+
}
|
|
41230
|
+
};
|
|
41231
|
+
await Promise.all(Array.from({ length: CONCURRENCY }, worker));
|
|
41232
|
+
if (fatal !== null) throw fatal;
|
|
41233
|
+
const reused = rows.filter((row) => saved.has(row.url));
|
|
41234
|
+
const oldest = reused.reduce((first, row) => first === null || row.inspected_at < first ? row.inspected_at : first, null);
|
|
41235
|
+
return {
|
|
41236
|
+
rows: sortRows(rows),
|
|
41237
|
+
statuses,
|
|
41238
|
+
known,
|
|
41239
|
+
limit: cap,
|
|
41240
|
+
inspected: rows.length,
|
|
41241
|
+
fromSaved: reused.length,
|
|
41242
|
+
oldestSaved: oldest,
|
|
41243
|
+
failed: failed.sort((a, b) => byText(a.url, b.url)),
|
|
41244
|
+
stoppedBy: quotaStop || quotaShort ? "quota" : known.urls.length > wanted.length ? "limit" : null,
|
|
41245
|
+
quota
|
|
41246
|
+
};
|
|
41247
|
+
}
|
|
41248
|
+
const n = (value) => value.toLocaleString("en-US");
|
|
41249
|
+
function liveCell(row) {
|
|
41250
|
+
if (row.live_error) return `error: ${row.live_error}`;
|
|
41251
|
+
if (!row.redirects) return String(row.status ?? "");
|
|
41252
|
+
return `${row.redirects.replace(/ (permanent|temporary)/g, "")} > ${row.final_status ?? "loop"}`;
|
|
41253
|
+
}
|
|
41254
|
+
function indexingText(site, run, page, origin, askMore, now) {
|
|
41255
|
+
const { known } = run;
|
|
41256
|
+
const total = known.urls.length;
|
|
41257
|
+
const notInspected = total - run.inspected - run.failed.length;
|
|
41258
|
+
const sources = [
|
|
41259
|
+
`lost-click pages ${n(known.lostClicks)}`,
|
|
41260
|
+
`Search Console pages with impressions in the last 28 days ${n(known.searchConsole)}${known.searchConsoleCapped ? ", capped" : ""}`,
|
|
41261
|
+
`sitemap ${n(known.sitemap.urls.length)} from ${plural(known.sitemap.files, "file")}${known.sitemap.partial ? ", partial" : ""}`
|
|
41262
|
+
];
|
|
41263
|
+
const lowerBound = known.sitemap.partial || known.searchConsoleCapped ? " at least" : "";
|
|
41264
|
+
const asked = run.inspected - run.fromSaved;
|
|
41265
|
+
const most = Math.min(total, MAX_INSPECT_LIMIT, run.quota ? run.inspected + run.quota.limit - run.quota.used - asked : MAX_INSPECT_LIMIT);
|
|
41266
|
+
const days = run.oldestSaved ? Math.floor((now.getTime() - Date.parse(run.oldestSaved)) / 864e5) : 0;
|
|
41267
|
+
const savedNote = run.fromSaved ? ` (${n(asked)} asked of Google now, ${n(run.fromSaved)} from saved results; the oldest was inspected ${run.oldestSaved?.slice(0, 16).replace("T", " ")} UTC, ${days === 0 ? "under a day" : plural(days, "day")} ago. Saved results are reused for 7 days)` : "";
|
|
41268
|
+
const stop = run.stoppedBy === "quota" ? "The URL Inspection quota stopped it." : run.stoppedBy === "limit" ? `It stopped at ${n(run.limit)}, the number of inspections for this run${askMore && most > run.limit ? `; to inspect more, up to the quota left today: ${askMore(most)}` : ""}.` : "Every known URL has a result.";
|
|
41269
|
+
const lines = [
|
|
41270
|
+
`Google's indexing reasons for ${site}, from URL Inspection. Search Console's Page indexing report has no API, so known URLs are inspected one at a time, in a fixed order (lost-click pages, Search Console pages by clicks then impressions, sitemap URLs, then the rest), and checked live from here.`,
|
|
41271
|
+
`Known URLs:${lowerBound} ${n(total)} (${sources.join("; ")}). Inspected ${n(run.inspected)} of ${n(total)}${savedNote}. ${stop}${notInspected > 0 ? ` The ${n(notInspected)} URLs not inspected have no result here, so this is not an all-clear for them.` : ""}${known.sitemap.notes.length ? ` Sitemap: ${known.sitemap.notes.join("; ")}.` : ""}`,
|
|
41272
|
+
run.quota ? `URL Inspection quota: this machine had used ${n(run.quota.used)} of ${n(run.quota.limit)} today for this property before the run. Google also counts inspections from other tools.` : "URL Inspection quota: not counted on this server; Google stops inspections at 2,000 a day per property."
|
|
41273
|
+
];
|
|
41274
|
+
if (run.failed.length) lines.push(`URL Inspection failed for ${plural(run.failed.length, "URL")}: ${run.failed.slice(0, 5).map((item) => `${item.url} (${item.error})`).join("; ")}${run.failed.length > 5 ? "; and more" : ""}.`);
|
|
41275
|
+
const groups = byReason(run.rows);
|
|
41276
|
+
if (groups.length) lines.push(`By Google's reason, with labels from the rules below (not Google's):\n${table(groups, [
|
|
41277
|
+
"reason",
|
|
41278
|
+
"urls",
|
|
41279
|
+
"fix",
|
|
41280
|
+
"check",
|
|
41281
|
+
"expected",
|
|
41282
|
+
"indexed"
|
|
41283
|
+
], null)}`);
|
|
41284
|
+
const listed = run.rows.filter((row) => row.label !== "indexed");
|
|
41285
|
+
const shown = page.summary ? listed.filter((row) => row.label === "fix" || row.label === "check") : listed;
|
|
41286
|
+
const indexedCount = run.rows.length - listed.length;
|
|
41287
|
+
const expectedCount = listed.length - shown.length;
|
|
41288
|
+
const pageRows = shown.slice(page.offset, page.offset + page.size).map((row) => ({
|
|
41289
|
+
reason: row.reason,
|
|
41290
|
+
label: row.label,
|
|
41291
|
+
url: row.url,
|
|
41292
|
+
live: liveCell(row),
|
|
41293
|
+
final_url: row.final_url === row.url ? "" : row.final_url,
|
|
41294
|
+
in_sitemap: row.in_sitemap ? "yes" : "no",
|
|
41295
|
+
last_crawl: row.last_crawl?.slice(0, 10) ?? "",
|
|
41296
|
+
why: row.why
|
|
41297
|
+
}));
|
|
41298
|
+
const end = page.offset + pageRows.length;
|
|
41299
|
+
const more = shown.length - end;
|
|
41300
|
+
const others = page.summary ? [
|
|
41301
|
+
more ? `${more} more labelled fix or check` : null,
|
|
41302
|
+
expectedCount ? `${expectedCount} expected` : null,
|
|
41303
|
+
indexedCount ? `${indexedCount} indexed with no fault found (in the full result)` : null
|
|
41304
|
+
].filter(Boolean) : [];
|
|
41305
|
+
if (page.summary && (more || expectedCount)) others.push(`every row but the indexed ones: ${page.more(0)}`);
|
|
41306
|
+
lines.push(`${shown.length === 0 ? `No inspected URL is labelled ${page.summary ? "fix or check" : "fix, check or expected"}.` : `${page.summary ? "URLs labelled fix or check" : "URLs that are not indexed or show a fault"}, ${page.offset + 1} to ${end} of ${shown.length}:\n${table(pageRows, Object.keys(pageRows[0] ?? {}), origin, 400)}`}${others.length ? `\nNot listed here: ${others.join("; ")}.` : ""}`, `Labels: ${Object.entries(LABELS).map(([label, text]) => `${label}, ${text}`).join("; ")}. Rules: a same-host 302 or 307 is fix; a 404 or 410 that the sitemap or a referring page on this site points to is fix; a sitemap URL that redirects or that Google does not index is fix; a URL outside the sitemap (such as an http:// or host variant) that permanently redirects to a page declaring itself canonical is expected; a 401 with a WWW-Authenticate header is expected, and a 401 or 403 on a sign-in path is expected by heuristic; "Crawled - currently not indexed", "Discovered - currently not indexed" and a Google canonical that differs from the declared one are check. Referring pages are as of Google's last crawl. Live checks are server responses, so a browser-only HSTS "307 Internal Redirect" never shows here.`);
|
|
41307
|
+
if (!page.summary && more) lines.push(`More rows: ${page.more(end)}.`);
|
|
41308
|
+
return lines;
|
|
41309
|
+
}
|
|
40838
41310
|
//#endregion
|
|
40839
41311
|
//#region packages/core/dist/commands/internal-links.js
|
|
40840
41312
|
const SUMMARY_ROWS = 10;
|
|
@@ -40897,8 +41369,8 @@ function internalLinksText(report, options) {
|
|
|
40897
41369
|
//#endregion
|
|
40898
41370
|
//#region packages/core/dist/commands/page-facts.js
|
|
40899
41371
|
const MAX_PAGES = 20;
|
|
40900
|
-
const TIMEOUT_MS = 6e3;
|
|
40901
|
-
const MAX_BYTES = 1e6;
|
|
41372
|
+
const TIMEOUT_MS$2 = 6e3;
|
|
41373
|
+
const MAX_BYTES$2 = 1e6;
|
|
40902
41374
|
const clean = (value) => value?.trim().replace(/\s+/g, " ") || null;
|
|
40903
41375
|
async function boundedText(response) {
|
|
40904
41376
|
if (!response.body) return "";
|
|
@@ -40906,7 +41378,7 @@ async function boundedText(response) {
|
|
|
40906
41378
|
const decoder = new TextDecoder();
|
|
40907
41379
|
let text = "";
|
|
40908
41380
|
let total = 0;
|
|
40909
|
-
while (total < MAX_BYTES) {
|
|
41381
|
+
while (total < MAX_BYTES$2) {
|
|
40910
41382
|
const { done, value } = await reader.read();
|
|
40911
41383
|
if (done) return text + decoder.decode();
|
|
40912
41384
|
total += value.byteLength;
|
|
@@ -40986,12 +41458,14 @@ async function facts(url) {
|
|
|
40986
41458
|
"user-agent": SEO_CRAWLER_USER_AGENT,
|
|
40987
41459
|
accept: "text/html"
|
|
40988
41460
|
},
|
|
40989
|
-
signal: AbortSignal.timeout(TIMEOUT_MS)
|
|
41461
|
+
signal: AbortSignal.timeout(TIMEOUT_MS$2)
|
|
40990
41462
|
});
|
|
40991
41463
|
out.status = response.status;
|
|
40992
41464
|
const location = response.headers.get("location");
|
|
40993
41465
|
if (location) out.redirects_to = new URL(location, url).toString();
|
|
40994
41466
|
const headerRobots = response.headers.get("x-robots-tag");
|
|
41467
|
+
const auth = response.headers.get("www-authenticate");
|
|
41468
|
+
if (auth) out.www_authenticate = auth.slice(0, 200);
|
|
40995
41469
|
if (response.status !== 200 || !response.headers.get("content-type")?.includes("html")) {
|
|
40996
41470
|
out.robots = headerRobots;
|
|
40997
41471
|
await response.body?.cancel();
|
|
@@ -41008,6 +41482,37 @@ async function facts(url) {
|
|
|
41008
41482
|
function checkPages(urls) {
|
|
41009
41483
|
return Promise.all(urls.slice(0, MAX_PAGES).map(facts));
|
|
41010
41484
|
}
|
|
41485
|
+
const MAX_HOPS = 5;
|
|
41486
|
+
const redirectKind = (status) => status === 301 || status === 308 ? "permanent" : status === 302 || status === 303 || status === 307 ? "temporary" : null;
|
|
41487
|
+
async function liveCheck(url) {
|
|
41488
|
+
const first = await facts(url);
|
|
41489
|
+
const hops = [];
|
|
41490
|
+
const seen = /* @__PURE__ */ new Set([url]);
|
|
41491
|
+
let last = first;
|
|
41492
|
+
while (last.redirects_to && last.status !== null) {
|
|
41493
|
+
const kind = redirectKind(last.status);
|
|
41494
|
+
if (!kind) break;
|
|
41495
|
+
const to = last.redirects_to;
|
|
41496
|
+
hops.push({
|
|
41497
|
+
from: last.url,
|
|
41498
|
+
status: last.status,
|
|
41499
|
+
to,
|
|
41500
|
+
kind
|
|
41501
|
+
});
|
|
41502
|
+
if (seen.has(to) || hops.length > MAX_HOPS) return {
|
|
41503
|
+
first,
|
|
41504
|
+
hops,
|
|
41505
|
+
last: null
|
|
41506
|
+
};
|
|
41507
|
+
seen.add(to);
|
|
41508
|
+
last = await facts(to);
|
|
41509
|
+
}
|
|
41510
|
+
return {
|
|
41511
|
+
first,
|
|
41512
|
+
hops,
|
|
41513
|
+
last
|
|
41514
|
+
};
|
|
41515
|
+
}
|
|
41011
41516
|
function temporaryRedirectNote(pages) {
|
|
41012
41517
|
const temporary = pages.filter((page) => (page.status === 302 || page.status === 307) && page.redirects_to !== null && new URL(page.redirects_to).host === new URL(page.url).host);
|
|
41013
41518
|
if (!temporary.length) return null;
|
|
@@ -41602,17 +42107,137 @@ async function runResearch(input, keys, options) {
|
|
|
41602
42107
|
};
|
|
41603
42108
|
}
|
|
41604
42109
|
//#endregion
|
|
42110
|
+
//#region packages/core/dist/commands/sitemap-urls.js
|
|
42111
|
+
const MAX_FILES = 20;
|
|
42112
|
+
const MAX_URLS = 5e4;
|
|
42113
|
+
const MAX_BYTES$1 = 2e7;
|
|
42114
|
+
const TIMEOUT_MS$1 = 1e4;
|
|
42115
|
+
async function boundedBytes(body) {
|
|
42116
|
+
if (!body) return /* @__PURE__ */ new Uint8Array();
|
|
42117
|
+
const reader = body.getReader();
|
|
42118
|
+
const chunks = [];
|
|
42119
|
+
let total = 0;
|
|
42120
|
+
while (true) {
|
|
42121
|
+
const { done, value } = await reader.read();
|
|
42122
|
+
if (done) break;
|
|
42123
|
+
total += value.byteLength;
|
|
42124
|
+
if (total > MAX_BYTES$1) {
|
|
42125
|
+
await reader.cancel();
|
|
42126
|
+
return null;
|
|
42127
|
+
}
|
|
42128
|
+
chunks.push(value);
|
|
42129
|
+
}
|
|
42130
|
+
const bytes = new Uint8Array(total);
|
|
42131
|
+
let offset = 0;
|
|
42132
|
+
for (const chunk of chunks) {
|
|
42133
|
+
bytes.set(chunk, offset);
|
|
42134
|
+
offset += chunk.byteLength;
|
|
42135
|
+
}
|
|
42136
|
+
return bytes;
|
|
42137
|
+
}
|
|
42138
|
+
async function get(url) {
|
|
42139
|
+
const response = await fetch(url, {
|
|
42140
|
+
headers: { "user-agent": SEO_CRAWLER_USER_AGENT },
|
|
42141
|
+
signal: AbortSignal.timeout(TIMEOUT_MS$1)
|
|
42142
|
+
});
|
|
42143
|
+
if (!response.ok) {
|
|
42144
|
+
await response.body?.cancel();
|
|
42145
|
+
throw new Error(`answered ${response.status}`);
|
|
42146
|
+
}
|
|
42147
|
+
let bytes = await boundedBytes(response.body);
|
|
42148
|
+
if (bytes && bytes[0] === 31 && bytes[1] === 139) bytes = await boundedBytes(new Response(bytes).body?.pipeThrough(new DecompressionStream("gzip")) ?? null);
|
|
42149
|
+
if (!bytes) throw new Error(`is larger than ${MAX_BYTES$1} bytes`);
|
|
42150
|
+
return new TextDecoder().decode(bytes);
|
|
42151
|
+
}
|
|
42152
|
+
const ENTITIES = {
|
|
42153
|
+
amp: "&",
|
|
42154
|
+
lt: "<",
|
|
42155
|
+
gt: ">",
|
|
42156
|
+
quot: "\"",
|
|
42157
|
+
apos: "'"
|
|
42158
|
+
};
|
|
42159
|
+
function locs(xml) {
|
|
42160
|
+
return [...xml.matchAll(/<loc>\s*(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?\s*<\/loc>/gi)].map((match) => (match[1] ?? "").trim().replace(/&(amp|lt|gt|quot|apos);/g, (_, name) => ENTITIES[name] ?? ""));
|
|
42161
|
+
}
|
|
42162
|
+
const inside = (site, url) => {
|
|
42163
|
+
try {
|
|
42164
|
+
return assertUrlMatchesGscProperty(site, url);
|
|
42165
|
+
} catch {
|
|
42166
|
+
return null;
|
|
42167
|
+
}
|
|
42168
|
+
};
|
|
42169
|
+
async function sitemapUrls(site) {
|
|
42170
|
+
const origin = new URL(originOf(site)).origin;
|
|
42171
|
+
const notes = [];
|
|
42172
|
+
const declared = [...(await get(`${origin}/robots.txt`).catch(() => "")).matchAll(/^\s*sitemap:\s*(\S+)/gim)].map((match) => match[1] ?? "").filter(Boolean);
|
|
42173
|
+
const queue = declared.length ? declared : [`${origin}/sitemap.xml`];
|
|
42174
|
+
const seenFiles = /* @__PURE__ */ new Set();
|
|
42175
|
+
const urls = /* @__PURE__ */ new Set();
|
|
42176
|
+
let files = 0;
|
|
42177
|
+
let partial = false;
|
|
42178
|
+
while (queue.length) {
|
|
42179
|
+
const file = queue.shift();
|
|
42180
|
+
if (seenFiles.has(file)) continue;
|
|
42181
|
+
seenFiles.add(file);
|
|
42182
|
+
if (files >= MAX_FILES) {
|
|
42183
|
+
partial = true;
|
|
42184
|
+
notes.push(`stopped at ${MAX_FILES} sitemap files`);
|
|
42185
|
+
break;
|
|
42186
|
+
}
|
|
42187
|
+
files += 1;
|
|
42188
|
+
let xml;
|
|
42189
|
+
try {
|
|
42190
|
+
xml = await get(file);
|
|
42191
|
+
} catch (error) {
|
|
42192
|
+
if (!declared.length && files === 1) {
|
|
42193
|
+
notes.push(`robots.txt names no sitemap and ${file} ${error instanceof Error ? error.message : "could not be read"}`);
|
|
42194
|
+
continue;
|
|
42195
|
+
}
|
|
42196
|
+
partial = true;
|
|
42197
|
+
notes.push(`${file} ${error instanceof Error ? error.message : "could not be read"}`);
|
|
42198
|
+
continue;
|
|
42199
|
+
}
|
|
42200
|
+
if (/<sitemapindex[\s>]/i.test(xml)) {
|
|
42201
|
+
queue.push(...locs(xml));
|
|
42202
|
+
continue;
|
|
42203
|
+
}
|
|
42204
|
+
for (const loc of locs(xml)) {
|
|
42205
|
+
const url = inside(site, loc);
|
|
42206
|
+
if (!url) continue;
|
|
42207
|
+
if (urls.size >= MAX_URLS) {
|
|
42208
|
+
partial = true;
|
|
42209
|
+
notes.push(`stopped at ${MAX_URLS.toLocaleString("en-US")} URLs`);
|
|
42210
|
+
queue.length = 0;
|
|
42211
|
+
break;
|
|
42212
|
+
}
|
|
42213
|
+
urls.add(url);
|
|
42214
|
+
}
|
|
42215
|
+
}
|
|
42216
|
+
return {
|
|
42217
|
+
urls: [...urls],
|
|
42218
|
+
files,
|
|
42219
|
+
partial,
|
|
42220
|
+
notes: [...new Set(notes)]
|
|
42221
|
+
};
|
|
42222
|
+
}
|
|
42223
|
+
//#endregion
|
|
41605
42224
|
//#region packages/core/dist/commands/site.js
|
|
41606
42225
|
function indexStatusFrom(result) {
|
|
41607
42226
|
const status = result?.inspectionResult?.indexStatusResult ?? result?.indexStatusResult ?? {};
|
|
41608
42227
|
const text = (value) => typeof value === "string" ? value : null;
|
|
42228
|
+
const list = (value) => Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
41609
42229
|
return {
|
|
41610
42230
|
verdict: text(status.verdict),
|
|
41611
42231
|
coverage: text(status.coverageState),
|
|
41612
42232
|
indexing: text(status.indexingState),
|
|
41613
42233
|
last_crawl: text(status.lastCrawlTime),
|
|
41614
42234
|
google_canonical: text(status.googleCanonical),
|
|
41615
|
-
user_canonical: text(status.userCanonical)
|
|
42235
|
+
user_canonical: text(status.userCanonical),
|
|
42236
|
+
page_fetch: text(status.pageFetchState),
|
|
42237
|
+
robots_txt: text(status.robotsTxtState),
|
|
42238
|
+
crawled_as: text(status.crawledAs),
|
|
42239
|
+
referring_urls: list(status.referringUrls),
|
|
42240
|
+
sitemaps: list(status.sitemap)
|
|
41616
42241
|
};
|
|
41617
42242
|
}
|
|
41618
42243
|
async function runPage(input, deps) {
|
|
@@ -41633,13 +42258,22 @@ async function runPage(input, deps) {
|
|
|
41633
42258
|
}]
|
|
41634
42259
|
});
|
|
41635
42260
|
const full = input.detail === "full";
|
|
41636
|
-
const [
|
|
41637
|
-
deps.
|
|
41638
|
-
deps.inspect(input.site, input.url),
|
|
42261
|
+
const [live, indexed, sitemap, now, then, markdown] = await Promise.all([
|
|
42262
|
+
deps.liveCheck(input.url),
|
|
42263
|
+
deps.inspect(input.site, input.url).catch(() => null),
|
|
42264
|
+
sitemapUrls(input.site),
|
|
41639
42265
|
query(range),
|
|
41640
42266
|
query(before),
|
|
41641
42267
|
full && deps.pageMarkdown ? deps.pageMarkdown(input.url).catch((error) => ({ error: error instanceof Error ? error.message : String(error) })) : null
|
|
41642
42268
|
]);
|
|
42269
|
+
const facts = live.first;
|
|
42270
|
+
const reason = indexed ? indexingRow({
|
|
42271
|
+
url: input.url,
|
|
42272
|
+
status: indexed,
|
|
42273
|
+
live,
|
|
42274
|
+
inSitemap: sitemap.urls.includes(input.url),
|
|
42275
|
+
sitemap: new Set(sitemap.urls)
|
|
42276
|
+
}, [], (deps.now?.() ?? /* @__PURE__ */ new Date()).toISOString()) : null;
|
|
41643
42277
|
const old = new Map(then.rows.map((row) => [row.keys[0], row]));
|
|
41644
42278
|
const seen = /* @__PURE__ */ new Set();
|
|
41645
42279
|
const rows = now.rows.map((row) => {
|
|
@@ -41677,8 +42311,9 @@ async function runPage(input, deps) {
|
|
|
41677
42311
|
rowsText({
|
|
41678
42312
|
title: `${input.url} in ${input.site}, ${range.startDate} to ${range.endDate} against ${before.startDate} to ${before.endDate}.`,
|
|
41679
42313
|
notes: [
|
|
41680
|
-
`Live page (fetched now
|
|
42314
|
+
`Live page (fetched now):\n${pageFactsTable([facts], null)}${live.hops.length ? `\nRedirects: ${live.hops.map((hop) => `${hop.status} to ${hop.to}`).join(", ")}${live.last ? `, which returns ${live.last.status ?? live.last.error}` : ", then loop or pass 5 hops"}.` : ""}`,
|
|
41681
42315
|
indexed ? `Google index status:\n${table([indexed], Object.keys(indexed), null)}` : "Google index status: not available for this URL.",
|
|
42316
|
+
...reason ? [`Google's reason: ${reason.reason}. Label: ${reason.label}. ${reason.why} (Labels are rules over this evidence, not Google's; ${deps.call("index")} explains them.)`] : [],
|
|
41682
42317
|
`Query totals: clicks ${sum(now.rows, "clicks")} (was ${sum(then.rows, "clicks")}), impressions ${sum(now.rows, "impressions")} (was ${sum(then.rows, "impressions")}). Search Console hides rare queries, so these can be lower than the page total.`,
|
|
41683
42318
|
deps.pageMarkdown ? `Main content as Markdown: ${deps.call("page", {
|
|
41684
42319
|
url: input.url,
|
|
@@ -41703,8 +42338,9 @@ async function runPage(input, deps) {
|
|
|
41703
42338
|
url: input.url,
|
|
41704
42339
|
range,
|
|
41705
42340
|
comparisonRange: before,
|
|
41706
|
-
page: facts
|
|
42341
|
+
page: facts,
|
|
41707
42342
|
index: indexed,
|
|
42343
|
+
reason,
|
|
41708
42344
|
totalQueries: rows.length,
|
|
41709
42345
|
queries: rows.slice(offset, offset + 50),
|
|
41710
42346
|
...full ? { markdown } : {}
|
|
@@ -41759,6 +42395,7 @@ const MIN_LOST = 3;
|
|
|
41759
42395
|
const CHECKED = 20;
|
|
41760
42396
|
const INSPECTED = 10;
|
|
41761
42397
|
const SITE_WIDE_MIN = 5;
|
|
42398
|
+
const PAGE_ROWS = 25e3;
|
|
41762
42399
|
function indexSummary(inspected, home) {
|
|
41763
42400
|
const known = inspected.filter((item) => item?.verdict != null);
|
|
41764
42401
|
const indexed = known.filter((item) => item.verdict === "PASS").length;
|
|
@@ -41791,6 +42428,7 @@ function indexSummary(inspected, home) {
|
|
|
41791
42428
|
};
|
|
41792
42429
|
}
|
|
41793
42430
|
const LOST_PAGES_RULE = `Pages that had at least ${MIN_LOST} clicks in the previous 28 days and kept ${LOST_SHARE * 100}% or less of them in the last 28 days, with a live check of the top ${CHECKED} and Google’s index status for the top ${INSPECTED} and the home page.`;
|
|
42431
|
+
const INDEXING_RULE = `Google’s indexing reasons for 100 known URLs a run in a fixed order (lost-click pages, Search Console pages by clicks then impressions, sitemap URLs, then the rest), each checked live and labelled fix, check, expected or indexed from the evidence, sorted by reason then URL.`;
|
|
41794
42432
|
async function runLostPages(input, deps) {
|
|
41795
42433
|
const range = dateWindow(input, 28);
|
|
41796
42434
|
const before = earlierRange(range);
|
|
@@ -41798,9 +42436,13 @@ async function runLostPages(input, deps) {
|
|
|
41798
42436
|
...window,
|
|
41799
42437
|
dimensions: ["page"],
|
|
41800
42438
|
type: "web",
|
|
41801
|
-
maxRows:
|
|
42439
|
+
maxRows: PAGE_ROWS
|
|
41802
42440
|
});
|
|
41803
|
-
const [now, then] = await Promise.all([
|
|
42441
|
+
const [now, then, sitemap] = await Promise.all([
|
|
42442
|
+
read(range),
|
|
42443
|
+
read(before),
|
|
42444
|
+
sitemapUrls(input.site)
|
|
42445
|
+
]);
|
|
41804
42446
|
const current = new Map(now.rows.map((row) => [row.keys[0], row.clicks]));
|
|
41805
42447
|
const lost = then.rows.filter((row) => row.clicks >= MIN_LOST && !row.keys[0]?.includes("#")).map((row) => ({
|
|
41806
42448
|
url: row.keys[0] ?? "",
|
|
@@ -41808,48 +42450,107 @@ async function runLostPages(input, deps) {
|
|
|
41808
42450
|
clicks: current.get(row.keys[0]) ?? 0
|
|
41809
42451
|
})).filter((row) => row.clicks <= row.previous_clicks * LOST_SHARE).sort((a, b) => b.previous_clicks - b.clicks - (a.previous_clicks - a.clicks) || a.url.localeCompare(b.url));
|
|
41810
42452
|
const top = lost.slice(0, CHECKED);
|
|
41811
|
-
const
|
|
41812
|
-
const
|
|
41813
|
-
|
|
41814
|
-
|
|
41815
|
-
|
|
41816
|
-
|
|
41817
|
-
|
|
41818
|
-
|
|
42453
|
+
const first = top[0]?.url ?? sitemap.urls[0] ?? now.rows[0]?.keys[0];
|
|
42454
|
+
const homeUrl = first ? `${new URL(first).origin}/` : null;
|
|
42455
|
+
const known = knownUrls({
|
|
42456
|
+
lost: lost.map((row) => row.url),
|
|
42457
|
+
home: homeUrl,
|
|
42458
|
+
sitemap,
|
|
42459
|
+
pages: now.rows.map((row) => ({
|
|
42460
|
+
url: row.keys[0] ?? "",
|
|
42461
|
+
clicks: row.clicks,
|
|
42462
|
+
impressions: row.impressions
|
|
42463
|
+
})),
|
|
42464
|
+
pagesCapped: now.rows.length >= PAGE_ROWS
|
|
42465
|
+
});
|
|
42466
|
+
const [facts, run] = await Promise.all([deps.checkPages(top.map((row) => row.url)), inspectKnown(input.site, known, deps, input.maxUrls)]);
|
|
42467
|
+
const status = (url) => url ? run.statuses.get(url) ?? null : null;
|
|
42468
|
+
const summary = indexSummary(top.slice(0, INSPECTED).filter((row) => row.url !== homeUrl).map((row) => status(row.url)), top.length && homeUrl ? {
|
|
41819
42469
|
url: homeUrl,
|
|
41820
|
-
status:
|
|
42470
|
+
status: status(homeUrl)
|
|
41821
42471
|
} : null);
|
|
41822
|
-
const rows = top.map((row, index) =>
|
|
41823
|
-
|
|
41824
|
-
|
|
41825
|
-
|
|
41826
|
-
|
|
41827
|
-
|
|
41828
|
-
|
|
41829
|
-
|
|
41830
|
-
|
|
41831
|
-
|
|
41832
|
-
|
|
41833
|
-
|
|
41834
|
-
|
|
41835
|
-
|
|
41836
|
-
|
|
41837
|
-
|
|
41838
|
-
|
|
41839
|
-
|
|
41840
|
-
|
|
42472
|
+
const rows = top.map((row, index) => {
|
|
42473
|
+
const indexed = index < INSPECTED ? status(row.url) : null;
|
|
42474
|
+
return {
|
|
42475
|
+
...row,
|
|
42476
|
+
status: facts[index]?.status ?? null,
|
|
42477
|
+
redirects_to: facts[index]?.redirects_to ?? null,
|
|
42478
|
+
robots: facts[index]?.robots ?? null,
|
|
42479
|
+
canonical: facts[index]?.canonical ?? null,
|
|
42480
|
+
verdict: indexed?.verdict ?? null,
|
|
42481
|
+
coverage: indexed?.coverage ?? null,
|
|
42482
|
+
google_canonical: indexed?.google_canonical ?? null
|
|
42483
|
+
};
|
|
42484
|
+
});
|
|
42485
|
+
const detail = input.detail ?? "summary";
|
|
42486
|
+
const offset = input.offset ?? 0;
|
|
42487
|
+
const origin = originOf(input.site);
|
|
42488
|
+
const firstFix = run.rows.find((row) => row.label === "fix");
|
|
42489
|
+
const reasons = indexingText(input.site, run, {
|
|
42490
|
+
summary: detail === "summary",
|
|
42491
|
+
offset,
|
|
42492
|
+
size: detail === "summary" ? 25 : 50,
|
|
42493
|
+
more: (next) => deps.call("index", {
|
|
42494
|
+
site: input.site,
|
|
42495
|
+
max_urls: input.maxUrls,
|
|
42496
|
+
detail: "rows",
|
|
42497
|
+
offset: next
|
|
42498
|
+
})
|
|
42499
|
+
}, origin, deps.inspectionQuota ? (count) => deps.call("index", {
|
|
42500
|
+
site: input.site,
|
|
42501
|
+
max_urls: count
|
|
42502
|
+
}) : null, deps.now?.() ?? /* @__PURE__ */ new Date());
|
|
42503
|
+
const lostText = [
|
|
42504
|
+
`Pages that lost their clicks on ${input.site}: ${range.startDate} to ${range.endDate} against ${before.startDate} to ${before.endDate}.`,
|
|
42505
|
+
`${lost.length} pages kept ${LOST_SHARE * 100}% or less of at least ${MIN_LOST} clicks${lost.length > CHECKED ? `; the top ${CHECKED} by clicks lost are shown` : ""}.`,
|
|
42506
|
+
rows.length ? table(rows, Object.keys(rows[0] ?? {}), origin) : "No pages lost their clicks.",
|
|
42507
|
+
summary.text,
|
|
42508
|
+
"Live checks are fetched now with no redirects followed. Index status is from URL Inspection.",
|
|
42509
|
+
temporaryRedirectNote(rows)
|
|
42510
|
+
];
|
|
42511
|
+
const indexing = {
|
|
42512
|
+
rule: INDEXING_RULE,
|
|
42513
|
+
known: {
|
|
42514
|
+
total: known.urls.length,
|
|
42515
|
+
sitemap: sitemap.urls.length,
|
|
42516
|
+
sitemapFiles: sitemap.files,
|
|
42517
|
+
sitemapPartial: sitemap.partial,
|
|
42518
|
+
searchConsole: known.searchConsole,
|
|
42519
|
+
searchConsoleCapped: known.searchConsoleCapped,
|
|
42520
|
+
lostClicks: known.lostClicks
|
|
42521
|
+
},
|
|
42522
|
+
limit: run.limit,
|
|
42523
|
+
inspected: run.inspected,
|
|
42524
|
+
fromSaved: run.fromSaved,
|
|
42525
|
+
oldestSaved: run.oldestSaved,
|
|
42526
|
+
failed: run.failed,
|
|
42527
|
+
stoppedBy: run.stoppedBy,
|
|
42528
|
+
quota: run.quota,
|
|
42529
|
+
byReason: byReason(run.rows),
|
|
42530
|
+
rows: run.rows
|
|
42531
|
+
};
|
|
42532
|
+
return {
|
|
42533
|
+
text: [...detail === "full" ? [`Google's indexing reasons for ${input.site}, every inspected URL as JSON, sorted by reason then URL:`, JSON.stringify(indexing)] : [
|
|
42534
|
+
...detail === "summary" ? lostText : [],
|
|
42535
|
+
...reasons,
|
|
42536
|
+
`To see what changed, keep this run's full result (${deps.call("index", {
|
|
42537
|
+
site: input.site,
|
|
42538
|
+
max_urls: input.maxUrls,
|
|
42539
|
+
detail: "full"
|
|
42540
|
+
})}) and compare it with the next run by url and reason. Inspect the same number each week: the order is fixed, so the same first URLs come back. Where results are saved (the CLI and the local MCP), a URL inspected in the last 7 days is reused; any other URL is inspected again and uses URL Inspection quota.`,
|
|
42541
|
+
`Next: ${firstFix ? deps.call("page", { url: firstFix.url }) : deps.call("page")} for one URL, including a URL from a Search Console email that is not in this set; ${deps.call("report", {
|
|
41841
42542
|
site: input.site,
|
|
41842
42543
|
section: "content-decay"
|
|
41843
|
-
})} for pages that lost part of their clicks
|
|
41844
|
-
|
|
41845
|
-
].filter(Boolean).join("\n\n"),
|
|
42544
|
+
})} for pages that lost part of their clicks.`
|
|
42545
|
+
], FACTS_ONLY].filter(Boolean).join("\n\n"),
|
|
41846
42546
|
data: {
|
|
41847
42547
|
range,
|
|
41848
42548
|
comparisonRange: before,
|
|
41849
42549
|
rule: LOST_PAGES_RULE,
|
|
41850
42550
|
lostPages: lost.length,
|
|
41851
42551
|
index: summary.data,
|
|
41852
|
-
pages: rows
|
|
42552
|
+
pages: rows,
|
|
42553
|
+
indexing
|
|
41853
42554
|
}
|
|
41854
42555
|
};
|
|
41855
42556
|
}
|
|
@@ -41992,16 +42693,26 @@ const SHARED_TOOLS = [
|
|
|
41992
42693
|
tool({
|
|
41993
42694
|
name: "seo_index",
|
|
41994
42695
|
command: "index",
|
|
41995
|
-
title: "
|
|
41996
|
-
description: `${
|
|
42696
|
+
title: "Google indexing reasons and pages that lost their clicks",
|
|
42697
|
+
description: `${INDEXING_RULE} Use it for a first baseline and then weekly: keep the full result and compare by URL and reason to find new reasons. Also: ${LOST_PAGES_RULE}`,
|
|
41997
42698
|
input: z.object({
|
|
41998
42699
|
site,
|
|
41999
|
-
...dates
|
|
42700
|
+
...dates,
|
|
42701
|
+
...detail
|
|
42702
|
+
}),
|
|
42703
|
+
localInput: z.object({
|
|
42704
|
+
site,
|
|
42705
|
+
...dates,
|
|
42706
|
+
max_urls: z.number().int().min(1).max(MAX_INSPECT_LIMIT).optional().describe(`URLs to inspect with Google, 100 by default, up to the URL Inspection quota left today.`),
|
|
42707
|
+
...detail
|
|
42000
42708
|
}),
|
|
42001
42709
|
run: async (surface, input) => runLostPages({
|
|
42002
42710
|
site: await surface.resolveSite(input.site),
|
|
42003
42711
|
startDate: input.start_date,
|
|
42004
|
-
endDate: input.end_date
|
|
42712
|
+
endDate: input.end_date,
|
|
42713
|
+
detail: input.detail,
|
|
42714
|
+
offset: input.offset,
|
|
42715
|
+
maxUrls: input.max_urls
|
|
42005
42716
|
}, surface.deps)
|
|
42006
42717
|
}),
|
|
42007
42718
|
tool({
|
|
@@ -42098,15 +42809,15 @@ const SHARED_TOOLS = [
|
|
|
42098
42809
|
}
|
|
42099
42810
|
})
|
|
42100
42811
|
];
|
|
42101
|
-
async function runSharedTool(tool, surface, input) {
|
|
42102
|
-
const parsed = tool.input.safeParse(input ?? {});
|
|
42812
|
+
async function runSharedTool(tool, surface, input, local = false) {
|
|
42813
|
+
const parsed = (local && tool.localInput || tool.input).safeParse(input ?? {});
|
|
42103
42814
|
if (!parsed.success) throw new CommandError("INVALID_INPUT", parsed.error.issues.map((issue) => `${issue.path.join(".") || "input"}: ${issue.message}`).join("; "));
|
|
42104
42815
|
return tool.run(surface, parsed.data);
|
|
42105
42816
|
}
|
|
42106
42817
|
const MCP_INSTRUCTIONS = `SEO Skill reads the person's own Search Console, Google Analytics, Bing Webmaster Tools, Clicky and optional keyword accounts, and returns facts with their dates and limits. It gives no advice.
|
|
42107
42818
|
1. seo_sites: the properties the connected accounts can read.
|
|
42108
42819
|
2. seo_report: the main report. Run it without a section first; it shows the overview and every section's count and top rows. Then ask for the one section you need.
|
|
42109
|
-
3. seo_page for one URL, seo_compare for before and after a date, seo_index for pages that lost their clicks, seo_research for keyword, competitor and backlink data, and whether AI answers cite the site, from the DataForSEO or Ahrefs key the person saved, seo_data for raw rows, including Google Analytics, Bing and Clicky.`;
|
|
42820
|
+
3. seo_page for one URL, seo_compare for before and after a date, seo_index for Google's indexing reasons and pages that lost their clicks, seo_research for keyword, competitor and backlink data, and whether AI answers cite the site, from the DataForSEO or Ahrefs key the person saved, seo_data for raw rows, including Google Analytics, Bing and Clicky.`;
|
|
42110
42821
|
const mcpCall = (command, args) => {
|
|
42111
42822
|
const name = `seo_${command.replace("-", "_")}`;
|
|
42112
42823
|
const shown = Object.entries(args ?? {}).filter(([, value]) => value !== void 0);
|
|
@@ -42120,6 +42831,36 @@ const SKILL_RESOURCE = {
|
|
|
42120
42831
|
mimeType: "text/markdown"
|
|
42121
42832
|
};
|
|
42122
42833
|
//#endregion
|
|
42834
|
+
//#region packages/core/dist/gsc/client/inspection-cache.js
|
|
42835
|
+
const INSPECTION_FRESH_MS = 7 * 864e5;
|
|
42836
|
+
function readSavedInspections(site, urls, now = Date.now()) {
|
|
42837
|
+
const found = /* @__PURE__ */ new Map();
|
|
42838
|
+
try {
|
|
42839
|
+
const statement = cacheDb().prepare(`SELECT status_json, fetched_at FROM url_inspection_cache
|
|
42840
|
+
WHERE site_url = ? AND url = ? AND fetched_at > ?`);
|
|
42841
|
+
for (const url of urls) {
|
|
42842
|
+
const row = statement.get(site, url, now - INSPECTION_FRESH_MS);
|
|
42843
|
+
if (!row) continue;
|
|
42844
|
+
found.set(url, {
|
|
42845
|
+
status: JSON.parse(row.status_json),
|
|
42846
|
+
inspectedAt: new Date(row.fetched_at).toISOString()
|
|
42847
|
+
});
|
|
42848
|
+
}
|
|
42849
|
+
} catch {
|
|
42850
|
+
return /* @__PURE__ */ new Map();
|
|
42851
|
+
}
|
|
42852
|
+
return found;
|
|
42853
|
+
}
|
|
42854
|
+
function saveInspection(site, url, status, now = Date.now()) {
|
|
42855
|
+
const json = JSON.stringify(status);
|
|
42856
|
+
writeToCache((database) => {
|
|
42857
|
+
database.prepare(`INSERT OR REPLACE INTO url_inspection_cache
|
|
42858
|
+
(site_url, url, status_json, fetched_at, expires_at)
|
|
42859
|
+
VALUES (?, ?, ?, ?, ?)`).run(site, url, json, now, now + INSPECTION_FRESH_MS);
|
|
42860
|
+
noteCacheWrite(Buffer.byteLength(json) + url.length, database);
|
|
42861
|
+
});
|
|
42862
|
+
}
|
|
42863
|
+
//#endregion
|
|
42123
42864
|
//#region packages/core/dist/providers/api-key-credentials.js
|
|
42124
42865
|
function createApiKeyCredentialStore(input) {
|
|
42125
42866
|
const normalize = (value) => {
|
|
@@ -42299,16 +43040,19 @@ function nodeCommandDeps(options) {
|
|
|
42299
43040
|
accountEmail: options.account
|
|
42300
43041
|
}),
|
|
42301
43042
|
checkPages,
|
|
42302
|
-
|
|
42303
|
-
|
|
42304
|
-
|
|
42305
|
-
|
|
42306
|
-
|
|
42307
|
-
|
|
42308
|
-
|
|
42309
|
-
|
|
42310
|
-
|
|
42311
|
-
|
|
43043
|
+
liveCheck,
|
|
43044
|
+
inspect: async (site, url) => indexStatusFrom(await inspectUrl({
|
|
43045
|
+
siteUrl: site,
|
|
43046
|
+
inspectionUrl: url,
|
|
43047
|
+
accountEmail: options.account
|
|
43048
|
+
})),
|
|
43049
|
+
inspectionQuota: (site) => urlInspectionQuota({
|
|
43050
|
+
siteUrl: site,
|
|
43051
|
+
accountEmail: options.account
|
|
43052
|
+
}),
|
|
43053
|
+
savedInspections: {
|
|
43054
|
+
read: (site, urls) => options.refresh ? /* @__PURE__ */ new Map() : readSavedInspections(site, urls),
|
|
43055
|
+
save: saveInspection
|
|
42312
43056
|
},
|
|
42313
43057
|
pageMarkdown: async (url) => extractMainMarkdown(await fetchPage(url, { refresh: options.refresh })),
|
|
42314
43058
|
call: options.call,
|
|
@@ -42759,6 +43503,94 @@ var LocalCredentialsProvider = class {
|
|
|
42759
43503
|
};
|
|
42760
43504
|
new LocalCredentialsProvider();
|
|
42761
43505
|
//#endregion
|
|
43506
|
+
//#region packages/core/dist/update-notice.js
|
|
43507
|
+
const REGISTRY_URL = "https://registry.npmjs.org/seo/latest";
|
|
43508
|
+
const CHECK_EVERY_MS = 864e5;
|
|
43509
|
+
const TIMEOUT_MS = 2e3;
|
|
43510
|
+
const MAX_BYTES = 65536;
|
|
43511
|
+
let checking = false;
|
|
43512
|
+
function runByNpx(env, cliPath) {
|
|
43513
|
+
if (env.npm_command === "exec" || /npx/i.test(env.npm_execpath ?? "")) return true;
|
|
43514
|
+
if (!cliPath) return false;
|
|
43515
|
+
let path = cliPath;
|
|
43516
|
+
try {
|
|
43517
|
+
path = realpathSync(cliPath);
|
|
43518
|
+
} catch {}
|
|
43519
|
+
return /[\\/](?:_npx|dlx)[\\/]/.test(path);
|
|
43520
|
+
}
|
|
43521
|
+
function isNewerVersion(current, latest) {
|
|
43522
|
+
const parse = (value) => value.replace(/^v/, "").split("-")[0]?.split(".").map((part) => Number.parseInt(part, 10)) ?? [];
|
|
43523
|
+
const currentParts = parse(current);
|
|
43524
|
+
const latestParts = parse(latest);
|
|
43525
|
+
for (let index = 0; index < 3; index += 1) {
|
|
43526
|
+
const currentPart = currentParts[index] ?? 0;
|
|
43527
|
+
const latestPart = latestParts[index] ?? 0;
|
|
43528
|
+
if (!Number.isFinite(currentPart) || !Number.isFinite(latestPart)) return false;
|
|
43529
|
+
if (latestPart > currentPart) return true;
|
|
43530
|
+
if (latestPart < currentPart) return false;
|
|
43531
|
+
}
|
|
43532
|
+
return false;
|
|
43533
|
+
}
|
|
43534
|
+
function isLatestCache(value) {
|
|
43535
|
+
if (!value || typeof value !== "object") return false;
|
|
43536
|
+
const cache = value;
|
|
43537
|
+
return typeof cache.checkedAt === "number" && typeof cache.latest === "string" && /^\d+\.\d+\.\d+$/.test(cache.latest);
|
|
43538
|
+
}
|
|
43539
|
+
function checkRegistry(cacheFile, now) {
|
|
43540
|
+
if (checking) return;
|
|
43541
|
+
checking = true;
|
|
43542
|
+
const done = () => {
|
|
43543
|
+
checking = false;
|
|
43544
|
+
};
|
|
43545
|
+
const req = request(REGISTRY_URL, { headers: { accept: "application/json" } });
|
|
43546
|
+
const timer = setTimeout(() => req.destroy(), TIMEOUT_MS);
|
|
43547
|
+
timer.unref();
|
|
43548
|
+
req.on("socket", (socket) => socket.unref());
|
|
43549
|
+
req.on("error", () => void 0);
|
|
43550
|
+
req.on("close", () => {
|
|
43551
|
+
clearTimeout(timer);
|
|
43552
|
+
done();
|
|
43553
|
+
});
|
|
43554
|
+
req.on("response", (response) => {
|
|
43555
|
+
const chunks = [];
|
|
43556
|
+
let size = 0;
|
|
43557
|
+
response.on("error", () => void 0);
|
|
43558
|
+
response.on("data", (chunk) => {
|
|
43559
|
+
size += chunk.length;
|
|
43560
|
+
if (size > MAX_BYTES) req.destroy();
|
|
43561
|
+
else chunks.push(chunk);
|
|
43562
|
+
});
|
|
43563
|
+
response.on("end", () => {
|
|
43564
|
+
if (response.statusCode !== 200 || size > MAX_BYTES) return;
|
|
43565
|
+
readJson(new Response(Buffer.concat(chunks)), "The npm registry").then((body) => {
|
|
43566
|
+
const cache = {
|
|
43567
|
+
checkedAt: now,
|
|
43568
|
+
latest: body.version
|
|
43569
|
+
};
|
|
43570
|
+
if (isLatestCache(cache)) writeJsonAtomic(cacheFile, cache, 384);
|
|
43571
|
+
}).catch(() => void 0);
|
|
43572
|
+
});
|
|
43573
|
+
});
|
|
43574
|
+
req.end();
|
|
43575
|
+
}
|
|
43576
|
+
function updateNotice(options = {}) {
|
|
43577
|
+
try {
|
|
43578
|
+
const env = options.env ?? process.env;
|
|
43579
|
+
const argv = options.argv ?? process.argv;
|
|
43580
|
+
if (env.NO_UPDATE_NOTIFIER || argv.includes("--no-update-notifier") || isTelemetryCi(env)) return;
|
|
43581
|
+
const cacheFile = options.cacheFile ?? join(getSeoCliPaths().cacheDir, "latest-version.json");
|
|
43582
|
+
const now = (options.now ?? /* @__PURE__ */ new Date()).getTime();
|
|
43583
|
+
const stored = readJsonFile(cacheFile);
|
|
43584
|
+
const cache = isLatestCache(stored) ? stored : void 0;
|
|
43585
|
+
if (!cache || now - cache.checkedAt >= CHECK_EVERY_MS) checkRegistry(cacheFile, now);
|
|
43586
|
+
const current = options.version ?? SEO_VERSION;
|
|
43587
|
+
if (!cache || !isNewerVersion(current, cache.latest)) return void 0;
|
|
43588
|
+
return runByNpx(env, options.cliPath ?? argv[1]) ? `seo ${cache.latest} is out (you have ${current}). Run npx -y seo@latest; in an MCP config use seo@latest, then restart the client.` : `seo ${cache.latest} is out (you have ${current}). Update with: npm i -g seo`;
|
|
43589
|
+
} catch {
|
|
43590
|
+
return;
|
|
43591
|
+
}
|
|
43592
|
+
}
|
|
43593
|
+
//#endregion
|
|
42762
43594
|
//#region packages/mcp/src/agent-output-budget.ts
|
|
42763
43595
|
const AGENT_STRUCTURED_OUTPUT_MAX_BYTES = 96 * 1024;
|
|
42764
43596
|
function jsonBytes(value) {
|
|
@@ -45305,7 +46137,7 @@ function text(value) {
|
|
|
45305
46137
|
text: value
|
|
45306
46138
|
}] };
|
|
45307
46139
|
}
|
|
45308
|
-
async function
|
|
46140
|
+
async function countedRun(command, args, telemetry, run) {
|
|
45309
46141
|
const report = telemetry ? telemetryReportFor(command, {
|
|
45310
46142
|
section: typeof args.section === "string" ? args.section : void 0,
|
|
45311
46143
|
action: typeof args.action === "string" ? args.action : void 0
|
|
@@ -45317,10 +46149,22 @@ async function counted(command, args, telemetry, run) {
|
|
|
45317
46149
|
else trackTelemetryReportComplete(report, telemetry());
|
|
45318
46150
|
return result;
|
|
45319
46151
|
} catch (error) {
|
|
45320
|
-
|
|
45321
|
-
|
|
46152
|
+
const failure = localError(error);
|
|
46153
|
+
if (report && telemetry) trackTelemetryReportFailed(report, failure, telemetry());
|
|
46154
|
+
return toolError(failure);
|
|
45322
46155
|
}
|
|
45323
46156
|
}
|
|
46157
|
+
async function counted(command, args, telemetry, run) {
|
|
46158
|
+
const result = await countedRun(command, args, telemetry, run);
|
|
46159
|
+
const notice = updateNotice();
|
|
46160
|
+
return notice ? {
|
|
46161
|
+
...result,
|
|
46162
|
+
content: [...result.content, {
|
|
46163
|
+
type: "text",
|
|
46164
|
+
text: notice
|
|
46165
|
+
}]
|
|
46166
|
+
} : result;
|
|
46167
|
+
}
|
|
45324
46168
|
function registerSharedTools(server, options = {}) {
|
|
45325
46169
|
const surface = {
|
|
45326
46170
|
deps: nodeCommandDeps({ call: mcpCall }),
|
|
@@ -45335,10 +46179,10 @@ function registerSharedTools(server, options = {}) {
|
|
|
45335
46179
|
for (const shared of SHARED_TOOLS) server.registerTool(shared.name, {
|
|
45336
46180
|
title: shared.title,
|
|
45337
46181
|
description: shared.description,
|
|
45338
|
-
inputSchema: shared.input.shape,
|
|
46182
|
+
inputSchema: (shared.localInput ?? shared.input).shape,
|
|
45339
46183
|
annotations: readOnly
|
|
45340
46184
|
}, async (input) => counted(shared.command, input, options.telemetry, async () => {
|
|
45341
|
-
return text((await runSharedTool(shared, surface, input)).text);
|
|
46185
|
+
return text((await runSharedTool(shared, surface, input, true)).text);
|
|
45342
46186
|
}));
|
|
45343
46187
|
server.registerTool("seo_crawl", {
|
|
45344
46188
|
title: "Crawl a site",
|
|
@@ -45433,6 +46277,7 @@ async function startMcpServer(opts = {}) {
|
|
|
45433
46277
|
}
|
|
45434
46278
|
const transport = new StdioServerTransport();
|
|
45435
46279
|
await server.connect(transport);
|
|
46280
|
+
updateNotice();
|
|
45436
46281
|
}
|
|
45437
46282
|
//#endregion
|
|
45438
46283
|
export { AGENT_STRUCTURED_OUTPUT_MAX_BYTES, REPORT_CATEGORIES, compactAgentWorkflowOutput, createServer, describeReport, executeReport, listReports, runReport, startMcpServer };
|