seo 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +14 -7
- package/README.md +1 -1
- package/dist/cli.js +78 -78
- package/dist/cli.js.map +1 -1
- package/dist/{dist-BL_yf1fw.js → dist-B1bivk1S.js} +1 -1
- package/dist/{dist-DqfdgVVU.js → dist-Ban8uP1U.js} +81 -65
- package/dist/dist-Ban8uP1U.js.map +1 -0
- package/dist/{esm-ZQwQYL0U.js → esm-CQJgJ8TD.js} +2 -2
- package/dist/{esm-ZQwQYL0U.js.map → esm-CQJgJ8TD.js.map} +1 -1
- package/dist/index.d.ts +85 -45
- package/dist/index.js +195 -71
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +823 -123
- package/dist/mcp.js.map +1 -1
- package/package.json +1 -1
- package/skills/seo/SKILL.md +15 -1
- package/dist/dist-DqfdgVVU.js.map +0 -1
package/dist/mcp.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-
|
|
2
|
+
import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-CQJgJ8TD.js";
|
|
3
3
|
import { createHash, randomUUID, sign } from "node:crypto";
|
|
4
4
|
import { ZodOptional, z } from "zod";
|
|
5
5
|
import { arch, homedir, platform, tmpdir, totalmem } from "node:os";
|
|
@@ -13790,7 +13790,7 @@ async function googleJson(gfetch, url, input) {
|
|
|
13790
13790
|
return readJson(response, input.label, { maxBytes: input.maxBytes });
|
|
13791
13791
|
}
|
|
13792
13792
|
const GSC = "https://www.googleapis.com/webmasters/v3";
|
|
13793
|
-
const PAGE_ROWS = 25e3;
|
|
13793
|
+
const PAGE_ROWS$1 = 25e3;
|
|
13794
13794
|
async function searchAnalyticsRows(gfetch, site, body, options) {
|
|
13795
13795
|
const rows = [];
|
|
13796
13796
|
const { maxRows: requested, ...apiBody } = body;
|
|
@@ -13801,7 +13801,7 @@ async function searchAnalyticsRows(gfetch, site, body, options) {
|
|
|
13801
13801
|
while (true) {
|
|
13802
13802
|
const remaining = typeof maxRows === "number" ? maxRows - rows.length : void 0;
|
|
13803
13803
|
if (typeof remaining === "number" && remaining <= 0) break;
|
|
13804
|
-
const rowLimit = Math.min(apiBody.rowLimit ?? PAGE_ROWS, remaining ?? PAGE_ROWS, PAGE_ROWS);
|
|
13804
|
+
const rowLimit = Math.min(apiBody.rowLimit ?? PAGE_ROWS$1, remaining ?? PAGE_ROWS$1, PAGE_ROWS$1);
|
|
13805
13805
|
const json = await googleJson(gfetch, url, {
|
|
13806
13806
|
...options,
|
|
13807
13807
|
label: "Search Console",
|
|
@@ -13878,7 +13878,7 @@ async function runAnalyticsReport(gfetch, propertyId, body, options) {
|
|
|
13878
13878
|
//#endregion
|
|
13879
13879
|
//#region packages/core/dist/bing/api.js
|
|
13880
13880
|
const BASE = "https://ssl.bing.com/webmaster/api.svc/json";
|
|
13881
|
-
const MAX_BYTES$
|
|
13881
|
+
const MAX_BYTES$4 = 8e6;
|
|
13882
13882
|
function bingCall(auth, options) {
|
|
13883
13883
|
return async (method, params = {}) => {
|
|
13884
13884
|
const url = new URL(`${BASE}/${method}`);
|
|
@@ -13903,7 +13903,7 @@ function bingCall(auth, options) {
|
|
|
13903
13903
|
if (response.status === 429 || /throttle/i.test(detail)) throw new SeoError("RATE_LIMITED", "Bing Webmaster is limiting requests. Try again in a few minutes.");
|
|
13904
13904
|
throw new SeoError("OPTIONAL_PROVIDER_UNAVAILABLE", `Bing Webmaster answered ${response.status}. ${detail}`.trim());
|
|
13905
13905
|
}
|
|
13906
|
-
return (await readJson(response, "Bing Webmaster", { maxBytes: MAX_BYTES$
|
|
13906
|
+
return (await readJson(response, "Bing Webmaster", { maxBytes: MAX_BYTES$4 })).d;
|
|
13907
13907
|
};
|
|
13908
13908
|
}
|
|
13909
13909
|
function day(value) {
|
|
@@ -16412,6 +16412,7 @@ const MEBIBYTE$2 = 1024 * 1024;
|
|
|
16412
16412
|
const CACHE_MAX_AGE_MS = 720 * 60 * 60 * 1e3;
|
|
16413
16413
|
const CACHE_LIMITS = {
|
|
16414
16414
|
gsc_cache: 48 * MEBIBYTE$2,
|
|
16415
|
+
url_inspection_cache: 16 * MEBIBYTE$2,
|
|
16415
16416
|
ga4_cache: 24 * MEBIBYTE$2,
|
|
16416
16417
|
provider_cache: 32 * MEBIBYTE$2,
|
|
16417
16418
|
http_cache: 128 * MEBIBYTE$2,
|
|
@@ -16429,6 +16430,16 @@ const CACHE_POLICIES = [
|
|
|
16429
16430
|
"response_json"
|
|
16430
16431
|
]
|
|
16431
16432
|
},
|
|
16433
|
+
{
|
|
16434
|
+
table: "url_inspection_cache",
|
|
16435
|
+
keys: ["site_url", "url"],
|
|
16436
|
+
timestamp: "fetched_at",
|
|
16437
|
+
sizeColumns: [
|
|
16438
|
+
"site_url",
|
|
16439
|
+
"url",
|
|
16440
|
+
"status_json"
|
|
16441
|
+
]
|
|
16442
|
+
},
|
|
16432
16443
|
{
|
|
16433
16444
|
table: "ga4_cache",
|
|
16434
16445
|
keys: ["property_id", "query_hash"],
|
|
@@ -16732,6 +16743,17 @@ CREATE INDEX IF NOT EXISTS idx_url_inspection_reservations_minute
|
|
|
16732
16743
|
credential_key, property_site_url, created_at
|
|
16733
16744
|
);
|
|
16734
16745
|
|
|
16746
|
+
CREATE TABLE IF NOT EXISTS url_inspection_cache (
|
|
16747
|
+
site_url TEXT NOT NULL,
|
|
16748
|
+
url TEXT NOT NULL,
|
|
16749
|
+
status_json TEXT NOT NULL,
|
|
16750
|
+
fetched_at INTEGER NOT NULL,
|
|
16751
|
+
expires_at INTEGER NOT NULL,
|
|
16752
|
+
PRIMARY KEY(site_url, url)
|
|
16753
|
+
) WITHOUT ROWID;
|
|
16754
|
+
CREATE INDEX IF NOT EXISTS idx_url_inspection_cache_expires
|
|
16755
|
+
ON url_inspection_cache(expires_at);
|
|
16756
|
+
|
|
16735
16757
|
CREATE TABLE IF NOT EXISTS performance_reports (
|
|
16736
16758
|
id TEXT PRIMARY KEY,
|
|
16737
16759
|
url TEXT NOT NULL,
|
|
@@ -20230,6 +20252,19 @@ function reserveUrlInspectionQuota(input) {
|
|
|
20230
20252
|
if ("error" in result) throw result.error;
|
|
20231
20253
|
return result.reservation;
|
|
20232
20254
|
}
|
|
20255
|
+
function readUrlInspectionQuota(input) {
|
|
20256
|
+
const database = input.database ?? getDb();
|
|
20257
|
+
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
20258
|
+
const bucket = database.prepare(`SELECT limit_count, used_count, reserved_count, blocked_until
|
|
20259
|
+
FROM url_inspection_quota_buckets
|
|
20260
|
+
WHERE credential_key = ? AND property_site_url = ? AND quota_date = ?`).get(input.credentialKey, input.property, urlInspectionQuotaDate(now));
|
|
20261
|
+
const limit = bucket?.limit_count ?? 2e3;
|
|
20262
|
+
return {
|
|
20263
|
+
used: (bucket?.blocked_until ?? 0) > now.getTime() ? limit : Math.min(limit, (bucket?.used_count ?? 0) + (bucket?.reserved_count ?? 0)),
|
|
20264
|
+
limit,
|
|
20265
|
+
resetAt: urlInspectionQuotaResetAt(now).toISOString()
|
|
20266
|
+
};
|
|
20267
|
+
}
|
|
20233
20268
|
function finalizeUrlInspectionQuota(input) {
|
|
20234
20269
|
const database = input.database ?? getDb();
|
|
20235
20270
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
@@ -20274,6 +20309,13 @@ function inspectionResponseError(response, input) {
|
|
|
20274
20309
|
if (response.status >= 500) return new SeoError("PROVIDER_UNAVAILABLE", `Google URL Inspection is unavailable (${response.status}). Try again later.`);
|
|
20275
20310
|
return new SeoError("PROVIDER_UNAVAILABLE", `URL Inspection failed with ${response.status}.`);
|
|
20276
20311
|
}
|
|
20312
|
+
async function urlInspectionQuota(input) {
|
|
20313
|
+
const { quotaIdentity } = await getAuthorized(input.accountEmail ?? await cloudAccountFor(input.siteUrl));
|
|
20314
|
+
return readUrlInspectionQuota({
|
|
20315
|
+
credentialKey: credentialKey(quotaIdentity),
|
|
20316
|
+
property: input.siteUrl
|
|
20317
|
+
});
|
|
20318
|
+
}
|
|
20277
20319
|
async function inspectUrl(input) {
|
|
20278
20320
|
const inspectionUrl = assertUrlMatchesGscProperty(input.siteUrl, input.inspectionUrl);
|
|
20279
20321
|
const { client, quotaIdentity } = await getAuthorized(input.accountEmail ?? await cloudAccountFor(input.siteUrl));
|
|
@@ -22120,7 +22162,7 @@ const actionPriority = new Map([
|
|
|
22120
22162
|
"unverified",
|
|
22121
22163
|
"covered"
|
|
22122
22164
|
].map((type, index) => [type, index]));
|
|
22123
|
-
function plural$
|
|
22165
|
+
function plural$7(count, singular, pluralLabel) {
|
|
22124
22166
|
return count === 1 ? singular : pluralLabel;
|
|
22125
22167
|
}
|
|
22126
22168
|
function actionable(item) {
|
|
@@ -22154,7 +22196,7 @@ function pageOpportunityVerdict(input) {
|
|
|
22154
22196
|
if (focus === "unverified") return "GSC visibility exists, but the page was not verified and no specific change is justified yet.";
|
|
22155
22197
|
if (focus === "covered") return "The verified page covers the sampled queries and CTR is not materially below its directional benchmark.";
|
|
22156
22198
|
const count = analysis.summary.opportunities;
|
|
22157
|
-
const prefix = `${count} ${plural$
|
|
22199
|
+
const prefix = `${count} ${plural$7(count, "actionable observation", "actionable observations")} found.`;
|
|
22158
22200
|
if (focus === "content-gap") return `${prefix} Verified body coverage is the main issue; confirm intent before expanding the page.`;
|
|
22159
22201
|
if (focus === "serp-framing") return `${prefix} The body is relevant, but its title, meta description, or H1 framing is weak.`;
|
|
22160
22202
|
if (focus === "ranking") return `${prefix} Most upside requires stronger relevance or authority; no CTR-only lift is claimed for page-two queries.`;
|
|
@@ -22169,7 +22211,7 @@ function pageOpportunityRecommendations(analysis) {
|
|
|
22169
22211
|
}
|
|
22170
22212
|
const recommendations = [`Start with "${top.query}". ${top.recommendation}`];
|
|
22171
22213
|
const technical = actions.filter((item) => item.opportunityType === "technical-check").length;
|
|
22172
|
-
if (technical) recommendations.push(`Resolve the technical evidence affecting ${technical} ${plural$
|
|
22214
|
+
if (technical) recommendations.push(`Resolve the technical evidence affecting ${technical} ${plural$7(technical, "row", "rows")} before content work.`);
|
|
22173
22215
|
const gaps = actions.filter((item) => item.opportunityType === "content-gap").length;
|
|
22174
22216
|
if (gaps >= 3) recommendations.push(`Group the ${gaps} verified content gaps into useful sections rather than adding one thin paragraph per query.`);
|
|
22175
22217
|
return recommendations;
|
|
@@ -25244,7 +25286,7 @@ function availabilityOf(status) {
|
|
|
25244
25286
|
function errorText(error) {
|
|
25245
25287
|
return error instanceof Error ? error.message : String(error);
|
|
25246
25288
|
}
|
|
25247
|
-
async function get(input, url, headers, readBody = true) {
|
|
25289
|
+
async function get$1(input, url, headers, readBody = true) {
|
|
25248
25290
|
const controller = abortController({
|
|
25249
25291
|
timeoutMs: input.timeoutMs,
|
|
25250
25292
|
signal: input.signal
|
|
@@ -25301,7 +25343,7 @@ function llmsLinks(text, base) {
|
|
|
25301
25343
|
return links;
|
|
25302
25344
|
}
|
|
25303
25345
|
async function checkLlmsTxt(input) {
|
|
25304
|
-
const result = await get(input, new URL("/llms.txt", input.url).toString());
|
|
25346
|
+
const result = await get$1(input, new URL("/llms.txt", input.url).toString());
|
|
25305
25347
|
const availability = result.availability === "available" && isHtml(result.contentType) ? "absent" : result.availability;
|
|
25306
25348
|
const exists = availability === "available";
|
|
25307
25349
|
const base = {
|
|
@@ -25318,7 +25360,7 @@ async function checkLlmsTxt(input) {
|
|
|
25318
25360
|
for (const url of found.slice(0, LLMS_LINK_LIMIT)) {
|
|
25319
25361
|
if (input.signal?.aborted) break;
|
|
25320
25362
|
checked += 1;
|
|
25321
|
-
const link = brokenLink(await get(input, url, void 0, false), false);
|
|
25363
|
+
const link = brokenLink(await get$1(input, url, void 0, false), false);
|
|
25322
25364
|
if (link) broken.push(link);
|
|
25323
25365
|
}
|
|
25324
25366
|
return {
|
|
@@ -25362,7 +25404,7 @@ function sitemapUrlsFromRobots(text) {
|
|
|
25362
25404
|
return [...urls];
|
|
25363
25405
|
}
|
|
25364
25406
|
async function checkRobotsAiAccess(input) {
|
|
25365
|
-
const result = await get(input, new URL("/robots.txt", input.url).toString());
|
|
25407
|
+
const result = await get$1(input, new URL("/robots.txt", input.url).toString());
|
|
25366
25408
|
const exists = result.availability === "available";
|
|
25367
25409
|
const unknown = result.availability === "rate-limited" || result.availability === "unreachable";
|
|
25368
25410
|
const declared = exists ? declaredUserAgents(result.text) : /* @__PURE__ */ new Set();
|
|
@@ -25391,7 +25433,7 @@ async function checkAgentResources(input) {
|
|
|
25391
25433
|
const resources = [];
|
|
25392
25434
|
for (const path of AGENT_RESOURCE_PATHS) {
|
|
25393
25435
|
if (input.signal?.aborted) break;
|
|
25394
|
-
const result = await get(input, new URL(path, input.url).toString());
|
|
25436
|
+
const result = await get$1(input, new URL(path, input.url).toString());
|
|
25395
25437
|
const exists = result.availability === "available" && !isHtml(result.contentType);
|
|
25396
25438
|
let validJson;
|
|
25397
25439
|
if (exists && /\bjson\b/i.test(result.contentType)) try {
|
|
@@ -25411,7 +25453,7 @@ async function checkAgentResources(input) {
|
|
|
25411
25453
|
return resources;
|
|
25412
25454
|
}
|
|
25413
25455
|
async function checkMarkdownNegotiation(input) {
|
|
25414
|
-
const result = await get(input, input.url, { accept: "text/markdown" }, false);
|
|
25456
|
+
const result = await get$1(input, input.url, { accept: "text/markdown" }, false);
|
|
25415
25457
|
const vary = result.headers?.get("vary") ?? "";
|
|
25416
25458
|
return {
|
|
25417
25459
|
url: result.url,
|
|
@@ -25458,7 +25500,7 @@ async function checkMarkdownCopies(input) {
|
|
|
25458
25500
|
const url = page.markdownAlternates?.[0];
|
|
25459
25501
|
if (!url || !input.allows(url)) continue;
|
|
25460
25502
|
checked += 1;
|
|
25461
|
-
const result = await get({
|
|
25503
|
+
const result = await get$1({
|
|
25462
25504
|
...input,
|
|
25463
25505
|
url
|
|
25464
25506
|
}, url);
|
|
@@ -27223,7 +27265,7 @@ const installedPackageJsonSchema = z.object({
|
|
|
27223
27265
|
function packageDirectory(packagesDir, packageName) {
|
|
27224
27266
|
return join(packagesDir, "node_modules", ...packageName.split("/"));
|
|
27225
27267
|
}
|
|
27226
|
-
function inside(parent, child) {
|
|
27268
|
+
function inside$1(parent, child) {
|
|
27227
27269
|
const path = relative(parent, child);
|
|
27228
27270
|
return path === "" || !path.startsWith("..") && !isAbsolute(path);
|
|
27229
27271
|
}
|
|
@@ -27255,14 +27297,14 @@ async function loadOne(installed, registry, options) {
|
|
|
27255
27297
|
async function loadProviderPackage(input, options) {
|
|
27256
27298
|
const importModule = options.importModule ?? ((url) => import(url));
|
|
27257
27299
|
const packageDir = realpathSync(packageDirectory(options.packagesDir, input.package));
|
|
27258
|
-
if (!inside(realpathSync(options.packagesDir), packageDir)) throw new Error("Installed provider package resolves outside the provider directory.");
|
|
27300
|
+
if (!inside$1(realpathSync(options.packagesDir), packageDir)) throw new Error("Installed provider package resolves outside the provider directory.");
|
|
27259
27301
|
const packageJsonPath = join(packageDir, "package.json");
|
|
27260
27302
|
const packageJson = installedPackageJsonSchema.parse(JSON.parse(readFileSync(packageJsonPath, "utf8")));
|
|
27261
27303
|
if (packageJson.name !== input.package || packageJson.version !== input.version) throw new Error("Installed provider package does not match its saved record.");
|
|
27262
27304
|
const packageRegistry = new ProviderExtensionRegistry();
|
|
27263
27305
|
for (const entry of packageJson.seo.providers) {
|
|
27264
27306
|
const entryPath = realpathSync(resolve(dirname(packageJsonPath), entry));
|
|
27265
|
-
if (!inside(packageDir, entryPath)) throw new Error("Provider entry point resolves outside its package.");
|
|
27307
|
+
if (!inside$1(packageDir, entryPath)) throw new Error("Provider entry point resolves outside its package.");
|
|
27266
27308
|
await providerActivate((await importModule(`${pathToFileURL(entryPath).href}?seo-provider=${encodeURIComponent(input.version)}`)).default)(packageRegistry.hostFor({
|
|
27267
27309
|
package: input.package,
|
|
27268
27310
|
version: input.version
|
|
@@ -27298,7 +27340,7 @@ async function loadInstalledProviderExtensions(options = {}) {
|
|
|
27298
27340
|
const DEFAULT_MAX_REQUESTS = 8;
|
|
27299
27341
|
const MAX_RESPONSE_BYTES$1 = 10 * 1024 * 1024;
|
|
27300
27342
|
const MAX_REQUEST_BODY_BYTES = 64 * 1024;
|
|
27301
|
-
const TIMEOUT_MS$
|
|
27343
|
+
const TIMEOUT_MS$3 = 15e3;
|
|
27302
27344
|
function normalizeProviderExtensionAccount(provider, account) {
|
|
27303
27345
|
const fields = provider.connection.fields.filter((field) => field.kind === "account");
|
|
27304
27346
|
const allowed = new Set(fields.map((field) => field.id));
|
|
@@ -27345,7 +27387,7 @@ function runtimeForProvider(provider, options = {}) {
|
|
|
27345
27387
|
},
|
|
27346
27388
|
fetch: providerFetch,
|
|
27347
27389
|
maxResponseBytes: MAX_RESPONSE_BYTES$1,
|
|
27348
|
-
timeoutMs: TIMEOUT_MS$
|
|
27390
|
+
timeoutMs: TIMEOUT_MS$3,
|
|
27349
27391
|
retry: (input.method ?? "GET") === "GET" ? "safe" : "never",
|
|
27350
27392
|
schema: z.unknown()
|
|
27351
27393
|
});
|
|
@@ -28908,7 +28950,7 @@ function analyzeCtrUnderperformersFromRows(input) {
|
|
|
28908
28950
|
const MAX_GSC_ROWS$3 = 1e5;
|
|
28909
28951
|
const DEFAULT_DAYS$3 = 28;
|
|
28910
28952
|
const MAX_DAYS$2 = 548;
|
|
28911
|
-
function plural$
|
|
28953
|
+
function plural$6(count, singular, pluralLabel = `${singular}s`) {
|
|
28912
28954
|
return count === 1 ? singular : pluralLabel;
|
|
28913
28955
|
}
|
|
28914
28956
|
async function ctrUnderperformersReportWith(input, dependencies) {
|
|
@@ -29011,11 +29053,11 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
29011
29053
|
minImpressions,
|
|
29012
29054
|
limit,
|
|
29013
29055
|
brandFiltering: input.includeBrand ? "included" : "excluded",
|
|
29014
|
-
verdict: top ? `${selection.eligibleUnderperformers} CTR ${plural$
|
|
29056
|
+
verdict: top ? `${selection.eligibleUnderperformers} CTR ${plural$6(selection.eligibleUnderperformers, "underperformer")} found in retained rows, with a calculated heuristic shortfall of ${totalClickShortfall.toFixed(0)} clicks for this window. Start review with "${top.query}" because it has the largest gap.` : dataStatus === "partial" ? "No material CTR underperformers remained in the validated retained rows, but partial evidence prevents an all-clear." : `No high-impression rows at average position below 4 in the retained rows are materially below the heuristic CTR benchmark.`
|
|
29015
29057
|
},
|
|
29016
29058
|
items,
|
|
29017
29059
|
caveats: [
|
|
29018
|
-
`Date window: ${range.startDate} to ${range.endDate} (${days} ${plural$
|
|
29060
|
+
`Date window: ${range.startDate} to ${range.endDate} (${days} ${plural$6(days, "day")}), using final GSC data where available.`,
|
|
29019
29061
|
`Brand queries: ${input.includeBrand ? "included" : "excluded"}. Brand terms used: ${brand.terms.join(", ") || "none"} (${brand.source === "provided" ? "provided" : "host forms, plus the bare domain stem only when Search Console shows the site is its navigational answer"}).`,
|
|
29020
29062
|
`Only rows at average position below 4 with at least ${minImpressions} impressions were checked. Rows at position 4 to 10 belong to quick-wins, which uses the same site CTR curve.`,
|
|
29021
29063
|
minImpressionsCaveat({
|
|
@@ -29023,13 +29065,13 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
29023
29065
|
belowMinimumRows: selection.belowMinimumRows
|
|
29024
29066
|
}),
|
|
29025
29067
|
"Expected CTR uses the site CTR curve from all page-one rows (75th percentile of qualified URL CTRs per position) when enough peer data exists, otherwise the fallback position curve.",
|
|
29026
|
-
...selection.fragmentRows ? [`${selection.fragmentRows} source ${plural$
|
|
29027
|
-
...selection.flaggedRows ? [`${selection.flaggedRows} eligible ${plural$
|
|
29068
|
+
...selection.fragmentRows ? [`${selection.fragmentRows} source ${plural$6(selection.fragmentRows, "row")} with a #fragment URL ${selection.fragmentRows === 1 ? "was" : "were"} merged into the base URL.`] : [],
|
|
29069
|
+
...selection.flaggedRows ? [`${selection.flaggedRows} eligible ${plural$6(selection.flaggedRows, "row")} carry heuristic query flags (likely-automated or navigational-other) and rank after unflagged rows.`] : [],
|
|
29028
29070
|
"CTR benchmarks and calculated click shortfalls are directional heuristics, not forecasts or proof that clicks are available. Validate the search intent and live SERP before editing.",
|
|
29029
|
-
...selection.invalidRows ? [`${selection.invalidRows} invalid provider ${plural$
|
|
29030
|
-
...selection.duplicateRows ? [`${selection.duplicateRows} repeated query/page ${plural$
|
|
29071
|
+
...selection.invalidRows ? [`${selection.invalidRows} invalid provider ${plural$6(selection.invalidRows, "row")} ${selection.invalidRows === 1 ? "was" : "were"} excluded, so this report is partial.`] : [],
|
|
29072
|
+
...selection.duplicateRows ? [`${selection.duplicateRows} repeated query/page ${plural$6(selection.duplicateRows, "row")} ${selection.duplicateRows === 1 ? "was" : "were"} aggregated before CTR and average position were calculated.`] : [],
|
|
29031
29073
|
...possiblyTruncated ? [`The retained-row limit of ${MAX_GSC_ROWS$3} was reached, so lower-ranked source rows may be absent.`] : [],
|
|
29032
|
-
...selection.limitedUnderperformers ? [`${selection.limitedUnderperformers} eligible ${plural$
|
|
29074
|
+
...selection.limitedUnderperformers ? [`${selection.limitedUnderperformers} eligible ${plural$6(selection.limitedUnderperformers, "underperformer")} ${selection.limitedUnderperformers === 1 ? "was" : "were"} omitted by the output limit.`] : []
|
|
29033
29075
|
],
|
|
29034
29076
|
recommendations: top ? [
|
|
29035
29077
|
`Review the live SERP and displayed snippet for "${top.query}" first. Test the title or meta description only if the observed framing does not match the query intent.`,
|
|
@@ -29049,11 +29091,11 @@ function ctrUnderperformersReport(input, dependencies = defaultDependencies$9) {
|
|
|
29049
29091
|
}
|
|
29050
29092
|
//#endregion
|
|
29051
29093
|
//#region packages/core/dist/phrasing.js
|
|
29052
|
-
function plural$
|
|
29094
|
+
function plural$5(count, singular, pluralLabel = `${singular}s`) {
|
|
29053
29095
|
return count === 1 ? singular : pluralLabel;
|
|
29054
29096
|
}
|
|
29055
29097
|
function countLabel(count, singular, pluralLabel = `${singular}s`) {
|
|
29056
|
-
return `${count.toLocaleString("en-GB")} ${plural$
|
|
29098
|
+
return `${count.toLocaleString("en-GB")} ${plural$5(count, singular, pluralLabel)}`;
|
|
29057
29099
|
}
|
|
29058
29100
|
//#endregion
|
|
29059
29101
|
//#region packages/core/dist/analyze/diagnosis-priorities.js
|
|
@@ -32812,7 +32854,7 @@ function anomalyEvidence(anomaly) {
|
|
|
32812
32854
|
if (anomaly.direction === "normal") return `${formatMetric(anomaly.metric)} changed ${changePhrase} (${anomaly.baselineMean.toLocaleString("en-GB")} -> ${anomaly.comparisonMean.toLocaleString("en-GB")} per observed day${anomaly.zScore === null ? "; the baseline had no observed variance" : `, z ${anomaly.zScore}`}).`;
|
|
32813
32855
|
return `${formatMetric(anomaly.metric)} ${direction} ${changePhrase} (${anomaly.baselineMean.toLocaleString("en-GB")} -> ${anomaly.comparisonMean.toLocaleString("en-GB")} per observed day${anomaly.zScore === null ? "; outside the flat observed baseline" : `, z ${anomaly.zScore}`}).`;
|
|
32814
32856
|
}
|
|
32815
|
-
function plural$
|
|
32857
|
+
function plural$4(count, singular, pluralLabel = `${singular}s`) {
|
|
32816
32858
|
return count === 1 ? singular : pluralLabel;
|
|
32817
32859
|
}
|
|
32818
32860
|
function strongestSignal(anomalies) {
|
|
@@ -32868,13 +32910,13 @@ function interpretUpdateCorrelation(input) {
|
|
|
32868
32910
|
summary,
|
|
32869
32911
|
evidence: [
|
|
32870
32912
|
...input.anomalies.map(anomalyEvidence),
|
|
32871
|
-
hasUpdate ? `${input.overlappingUpdates.length} official Google Ranking update ${plural$
|
|
32913
|
+
hasUpdate ? `${input.overlappingUpdates.length} official Google Ranking update ${plural$4(input.overlappingUpdates.length, "window")} overlapped the change date or comparison window: ${updateNames.join(", ")}.` : "No official Google Ranking update windows overlapped the change date or comparison window.",
|
|
32872
32914
|
largeMovement ? "Both clicks and impressions moved sharply in the same direction." : "Clicks and impressions did not both meet the sharp-movement evidence threshold.",
|
|
32873
32915
|
hasConfounders ? `Known overlapping site changes: ${confounders.map((confounder) => confounder.title).join("; ")}.` : "No overlapping site changes were supplied."
|
|
32874
32916
|
],
|
|
32875
32917
|
caveats: [
|
|
32876
32918
|
`GSC window: ${input.days} days; recent comparison window: ${input.recentDays} days.`,
|
|
32877
|
-
`Update overlap padding: ${input.paddingDays} ${plural$
|
|
32919
|
+
`Update overlap padding: ${input.paddingDays} ${plural$4(input.paddingDays, "day")} either side of the step-change date, or of the recent comparison window when no step change was found.`,
|
|
32878
32920
|
input.refresh ? "Data freshness: cache bypassed for GSC data." : "Data freshness: local GSC cache may have been used.",
|
|
32879
32921
|
"Source: official Google Search Status Dashboard Ranking incidents only; third-party volatility and unconfirmed chatter are not included.",
|
|
32880
32922
|
"An update-window overlap cannot establish causation, even when movement is large and no known site change was recorded.",
|
|
@@ -33042,7 +33084,7 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
33042
33084
|
if (step && changeDate) {
|
|
33043
33085
|
const previousDate = aggregated.rows[step.index - 1]?.keys[0];
|
|
33044
33086
|
const gap = previousDate ? calendarDays(previousDate, changeDate) - 2 : 0;
|
|
33045
|
-
stepCaveats.push(`${formatMetric(metric)} changed level on ${changeDate}, so its baseline is every observed day before that date and its comparison is every observed day from it.`, ...gap > 0 ? [`The ${metric} change date follows ${gap} missing ${plural$
|
|
33087
|
+
stepCaveats.push(`${formatMetric(metric)} changed level on ${changeDate}, so its baseline is every observed day before that date and its comparison is every observed day from it.`, ...gap > 0 ? [`The ${metric} change date follows ${gap} missing ${plural$4(gap, "day")}; the change may have started on one of them.`] : []);
|
|
33046
33088
|
return compareSegments({
|
|
33047
33089
|
site: input.site,
|
|
33048
33090
|
metric,
|
|
@@ -33073,9 +33115,9 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
33073
33115
|
const comparisonMissingDays = input.recentDays - comparisonRows.length;
|
|
33074
33116
|
const missingDays = expectedDays - aggregated.rows.length;
|
|
33075
33117
|
const caveats = [
|
|
33076
|
-
...missingDays > 0 ? [`${missingDays} requested calendar ${plural$
|
|
33077
|
-
...aggregated.invalidRows > 0 ? [`${aggregated.invalidRows} invalid provider ${plural$
|
|
33078
|
-
...aggregated.duplicateRows > 0 ? [`${aggregated.duplicateRows} duplicate date ${plural$
|
|
33118
|
+
...missingDays > 0 ? [`${missingDays} requested calendar ${plural$4(missingDays, "day")} had no returned date aggregate and ${missingDays === 1 ? "was" : "were"} not filled with zeros.`] : [],
|
|
33119
|
+
...aggregated.invalidRows > 0 ? [`${aggregated.invalidRows} invalid provider ${plural$4(aggregated.invalidRows, "row")} ${aggregated.invalidRows === 1 ? "was" : "were"} excluded.`] : [],
|
|
33120
|
+
...aggregated.duplicateRows > 0 ? [`${aggregated.duplicateRows} duplicate date ${plural$4(aggregated.duplicateRows, "row")} ${aggregated.duplicateRows === 1 ? "was" : "were"} aggregated before analysis.`] : [],
|
|
33079
33121
|
...stepCaveats
|
|
33080
33122
|
];
|
|
33081
33123
|
return {
|
|
@@ -34156,7 +34198,7 @@ const VOID = /* @__PURE__ */ new Set([
|
|
|
34156
34198
|
"col",
|
|
34157
34199
|
"embed"
|
|
34158
34200
|
]);
|
|
34159
|
-
const ENTITIES = {
|
|
34201
|
+
const ENTITIES$1 = {
|
|
34160
34202
|
amp: "&",
|
|
34161
34203
|
lt: "<",
|
|
34162
34204
|
gt: ">",
|
|
@@ -34181,7 +34223,7 @@ function decode(text) {
|
|
|
34181
34223
|
const point = Number.parseInt(code.slice(hex ? 2 : 1), hex ? 16 : 10);
|
|
34182
34224
|
return point > 0 && point <= 1114111 ? String.fromCodePoint(point) : "";
|
|
34183
34225
|
}
|
|
34184
|
-
return ENTITIES[code.toLowerCase()] ?? match;
|
|
34226
|
+
return ENTITIES$1[code.toLowerCase()] ?? match;
|
|
34185
34227
|
});
|
|
34186
34228
|
}
|
|
34187
34229
|
const clean$1 = (value) => value.split(A_OPEN).join("").replace(/\s+/g, " ").trim();
|
|
@@ -34421,7 +34463,7 @@ var SentenceStore = class SentenceStore {
|
|
|
34421
34463
|
});
|
|
34422
34464
|
}
|
|
34423
34465
|
};
|
|
34424
|
-
const bareHost = (host) => host.toLowerCase().replace(/^www\./, "");
|
|
34466
|
+
const bareHost$1 = (host) => host.toLowerCase().replace(/^www\./, "");
|
|
34425
34467
|
function crawlUrl(value, base) {
|
|
34426
34468
|
try {
|
|
34427
34469
|
const url = base ? new URL(value, base) : new URL(value);
|
|
@@ -34487,10 +34529,10 @@ async function crawlForLinks(input) {
|
|
|
34487
34529
|
async function crawl(input, robots, sentences, renderer, warnings) {
|
|
34488
34530
|
const start = crawlUrl(input.startUrl) ?? input.startUrl;
|
|
34489
34531
|
let origin = new URL(start).origin;
|
|
34490
|
-
const site = bareHost(new URL(start).host);
|
|
34532
|
+
const site = bareHost$1(new URL(start).host);
|
|
34491
34533
|
const sameSite = (url) => {
|
|
34492
34534
|
try {
|
|
34493
|
-
return bareHost(new URL(url).host) === site;
|
|
34535
|
+
return bareHost$1(new URL(url).host) === site;
|
|
34494
34536
|
} catch {
|
|
34495
34537
|
return false;
|
|
34496
34538
|
}
|
|
@@ -34984,15 +35026,15 @@ function internalLinksReportRange(input, fallbackDays, now) {
|
|
|
34984
35026
|
days: Math.round((Date.parse(`${endDate}T00:00:00Z`) - Date.parse(`${startDate}T00:00:00Z`)) / DAY$1) + 1
|
|
34985
35027
|
};
|
|
34986
35028
|
}
|
|
34987
|
-
const plural$
|
|
35029
|
+
const plural$3 = (count, word) => `${count} ${word}${count === 1 ? "" : "s"}`;
|
|
34988
35030
|
function internalLinksVerdict(report) {
|
|
34989
35031
|
const { counts } = report;
|
|
34990
35032
|
if (report.dataStatus === "blocked") return "robots.txt does not allow this crawler to read the site, so nothing was checked.";
|
|
34991
35033
|
if (report.dataStatus === "empty") return "No HTML page could be crawled, so there is nothing to link.";
|
|
34992
35034
|
if (report.dataStatus === "target-not-crawled") return "The target page could not be crawled as its own page (it failed, left the site, or names another page as canonical). Check it with seo page first.";
|
|
34993
35035
|
if (report.dataStatus === "target-not-linkable") return `The target is ${report.target?.noindex ? "noindex" : `a ${report.target?.type?.replaceAll("_", " ")} page`}, so no links to it are suggested.`;
|
|
34994
|
-
const judged = report.judge.status === "used" || report.judge.status === "partial" ? `${counts.strong} strong and ${counts.possible} possible link${counts.strong + counts.possible === 1 ? "" : "s"} judged by ${report.judge.model}${counts.heuristic ? `, plus ${plural$
|
|
34995
|
-
const scope = report.mode === "url" ? `to ${report.target?.url}` : `across ${plural$
|
|
35036
|
+
const judged = report.judge.status === "used" || report.judge.status === "partial" ? `${counts.strong} strong and ${counts.possible} possible link${counts.strong + counts.possible === 1 ? "" : "s"} judged by ${report.judge.model}${counts.heuristic ? `, plus ${plural$3(counts.heuristic, "heuristic link")} where a request failed` : ""}` : `${plural$3(counts.heuristic, "heuristic link")} (no decision model)`;
|
|
35037
|
+
const scope = report.mode === "url" ? `to ${report.target?.url}` : `across ${plural$3(counts.pagesCrawled, "page")}`;
|
|
34996
35038
|
const capped = report.crawl.capReached ? ` The crawl stopped at its limit of ${report.crawl.pageLimit} pages, so pages beyond it were not checked.` : "";
|
|
34997
35039
|
return `${judged} ${scope}${counts.heldBack ? `; ${counts.heldBack} held back by the evidence floors` : ""}.${capped}`;
|
|
34998
35040
|
}
|
|
@@ -37430,14 +37472,14 @@ function* retainedQueryClusterRows(input) {
|
|
|
37430
37472
|
};
|
|
37431
37473
|
}
|
|
37432
37474
|
}
|
|
37433
|
-
function plural$
|
|
37475
|
+
function plural$2(count, singular, pluralLabel = `${singular}s`) {
|
|
37434
37476
|
return count === 1 ? singular : pluralLabel;
|
|
37435
37477
|
}
|
|
37436
37478
|
function reportVerdict(input) {
|
|
37437
37479
|
if (!input.clusters.length) return "No query clusters were generated for this scope.";
|
|
37438
37480
|
if (input.highOpportunityClusters > 0) {
|
|
37439
37481
|
const verb = input.highOpportunityClusters === 1 ? "has" : "have";
|
|
37440
|
-
return `${input.highOpportunityClusters} ${plural$
|
|
37482
|
+
return `${input.highOpportunityClusters} ${plural$2(input.highOpportunityClusters, "cluster")} ${verb} enough demand and weak enough performance to review first.`;
|
|
37441
37483
|
}
|
|
37442
37484
|
return "No obvious high-opportunity clusters were found. Use this report for content structure and intent mapping rather than urgent fixes.";
|
|
37443
37485
|
}
|
|
@@ -37578,7 +37620,7 @@ function movementLine(report) {
|
|
|
37578
37620
|
}
|
|
37579
37621
|
function countPhrase(count, singular, pluralLabel = `${singular}s`) {
|
|
37580
37622
|
if (count === 0) return `no ${pluralLabel}`;
|
|
37581
|
-
return `${count} ${plural(count, singular, pluralLabel)}`;
|
|
37623
|
+
return `${count} ${plural$1(count, singular, pluralLabel)}`;
|
|
37582
37624
|
}
|
|
37583
37625
|
function headlineLine(report) {
|
|
37584
37626
|
const parts = [
|
|
@@ -37633,7 +37675,7 @@ function decayDiagnosisLabel(diagnosis) {
|
|
|
37633
37675
|
if (diagnosis === "lost_clicks") return "clicks dropped without one dominant signal";
|
|
37634
37676
|
return diagnosis.replaceAll("_", " ");
|
|
37635
37677
|
}
|
|
37636
|
-
function plural(count, singular, pluralLabel = `${singular}s`) {
|
|
37678
|
+
function plural$1(count, singular, pluralLabel = `${singular}s`) {
|
|
37637
37679
|
return count === 1 ? singular : pluralLabel;
|
|
37638
37680
|
}
|
|
37639
37681
|
function decayClusterLine(report) {
|
|
@@ -37641,7 +37683,7 @@ function decayClusterLine(report) {
|
|
|
37641
37683
|
if (!group) return "No material decay cluster stood out.";
|
|
37642
37684
|
const diagnosis = decayDiagnosisLabel(group.diagnosis);
|
|
37643
37685
|
const samples = group.sampleQueries.slice(0, 2).join("; ");
|
|
37644
|
-
return `Top decay cluster: ${group.count} ${plural(group.count, "finding")} in the ${group.template.label} template (${diagnosis}), ${group.totalClickLoss.toFixed(0)} fewer observed clicks across rows retained in both windows. Action: ${group.recommendation}${samples ? ` Example queries: ${samples}.` : ""}`;
|
|
37686
|
+
return `Top decay cluster: ${group.count} ${plural$1(group.count, "finding")} in the ${group.template.label} template (${diagnosis}), ${group.totalClickLoss.toFixed(0)} fewer observed clicks across rows retained in both windows. Action: ${group.recommendation}${samples ? ` Example queries: ${samples}.` : ""}`;
|
|
37645
37687
|
}
|
|
37646
37688
|
function cannibalSuppressionLine(report) {
|
|
37647
37689
|
const suppressed = report.cannibalization.selection.suppressedQueries;
|
|
@@ -37659,21 +37701,21 @@ function contentOpportunityBullets(report) {
|
|
|
37659
37701
|
if (unavailableSections.length) bullets.push(`Could not assess ${unavailableSections.join(", ")} opportunities in this run; see report caveats.`);
|
|
37660
37702
|
if (report.summary.decayItems) {
|
|
37661
37703
|
const top = report.decay.items[0];
|
|
37662
|
-
bullets.push(top ? `${report.summary.decayItems} observed retained query/page ${plural(report.summary.decayItems, "decline")} need review. Start with "${top.query}" on ${top.url}; it had ${top.clickLoss.toFixed(0)} fewer clicks.` : `${report.summary.decayItems} observed retained query/page ${plural(report.summary.decayItems, "decline")} need review.`);
|
|
37704
|
+
bullets.push(top ? `${report.summary.decayItems} observed retained query/page ${plural$1(report.summary.decayItems, "decline")} need review. Start with "${top.query}" on ${top.url}; it had ${top.clickLoss.toFixed(0)} fewer clicks.` : `${report.summary.decayItems} observed retained query/page ${plural$1(report.summary.decayItems, "decline")} need review.`);
|
|
37663
37705
|
bullets.push(decayClusterLine(report));
|
|
37664
37706
|
}
|
|
37665
37707
|
if (report.summary.cannibalItems) {
|
|
37666
37708
|
const top = report.cannibalization.items[0];
|
|
37667
|
-
bullets.push(top ? `${report.summary.cannibalItems} multi-URL query ${plural(report.summary.cannibalItems, "candidate")} need intent and technical review. Start with "${top.query}" and decide whether ${top.pages.length} ranking URLs answer the same intent.` : `${report.summary.cannibalItems} multi-URL query ${plural(report.summary.cannibalItems, "candidate")} need intent and technical review.`);
|
|
37709
|
+
bullets.push(top ? `${report.summary.cannibalItems} multi-URL query ${plural$1(report.summary.cannibalItems, "candidate")} need intent and technical review. Start with "${top.query}" and decide whether ${top.pages.length} ranking URLs answer the same intent.` : `${report.summary.cannibalItems} multi-URL query ${plural$1(report.summary.cannibalItems, "candidate")} need intent and technical review.`);
|
|
37668
37710
|
}
|
|
37669
37711
|
if (report.cannibalization.selection.suppressedQueries) bullets.push(cannibalSuppressionLine(report));
|
|
37670
37712
|
if (report.summary.strikingDistanceItems) {
|
|
37671
37713
|
const top = report.strikingDistance.items[0];
|
|
37672
|
-
bullets.push(top ? `${countLabel(report.summary.strikingDistanceItems, "query/page candidate")} ${plural(report.summary.strikingDistanceItems, "has", "have")} an average GSC position above 10 and at most 20. Start by investigating "${top.query}" on ${top.url}; it averages position ${top.position.toFixed(1)} with ${top.impressions.toFixed(0)} impressions.` : `${countLabel(report.summary.strikingDistanceItems, "query/page candidate")} ${plural(report.summary.strikingDistanceItems, "has", "have")} an average GSC position above 10 and at most 20.`);
|
|
37714
|
+
bullets.push(top ? `${countLabel(report.summary.strikingDistanceItems, "query/page candidate")} ${plural$1(report.summary.strikingDistanceItems, "has", "have")} an average GSC position above 10 and at most 20. Start by investigating "${top.query}" on ${top.url}; it averages position ${top.position.toFixed(1)} with ${top.impressions.toFixed(0)} impressions.` : `${countLabel(report.summary.strikingDistanceItems, "query/page candidate")} ${plural$1(report.summary.strikingDistanceItems, "has", "have")} an average GSC position above 10 and at most 20.`);
|
|
37673
37715
|
}
|
|
37674
37716
|
if (!skippedReason(report, "quick-win opportunities") && report.summary.quickWinItems) {
|
|
37675
37717
|
const top = report.quickWins.items[0];
|
|
37676
|
-
bullets.push(top ? `${report.summary.quickWinItems} retained query/page ${plural(report.summary.quickWinItems, "candidate")} fell below a site-peer or versioned heuristic CTR target. Start with "${top.query}" on ${top.url}; it had ${top.impressions.toFixed(0)} impressions at average position ${top.position.toFixed(1)}.` : `${report.summary.quickWinItems} CTR-target ${plural(report.summary.quickWinItems, "candidate")} need review.`);
|
|
37718
|
+
bullets.push(top ? `${report.summary.quickWinItems} retained query/page ${plural$1(report.summary.quickWinItems, "candidate")} fell below a site-peer or versioned heuristic CTR target. Start with "${top.query}" on ${top.url}; it had ${top.impressions.toFixed(0)} impressions at average position ${top.position.toFixed(1)}.` : `${report.summary.quickWinItems} CTR-target ${plural$1(report.summary.quickWinItems, "candidate")} need review.`);
|
|
37677
37719
|
}
|
|
37678
37720
|
const templateLine = templateOpportunityLine(report);
|
|
37679
37721
|
if (!templateLine.startsWith("No dominant")) bullets.push(templateLine);
|
|
@@ -38033,10 +38075,10 @@ function applyVerification(item, coverage) {
|
|
|
38033
38075
|
function verdict(report) {
|
|
38034
38076
|
if (report.dataStatus === "empty") return "Google Search Console returned no retained query/page rows for this window.";
|
|
38035
38077
|
if (report.dataStatus === "filtered") return "No pages met the position, brand, and minimum-impression criteria.";
|
|
38036
|
-
if (report.technical > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.technical, "returned page")} ${plural$
|
|
38037
|
-
if (report.failed > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.failed, "returned page")} could not be verified and ${plural$
|
|
38038
|
-
if (report.content > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.content, "returned page")} ${plural$
|
|
38039
|
-
return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.returned, "page")} ${plural$
|
|
38078
|
+
if (report.technical > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.technical, "returned page")} ${plural$5(report.technical, "has", "have")} verified technical issues to fix before content changes.`;
|
|
38079
|
+
if (report.failed > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.failed, "returned page")} could not be verified and ${plural$5(report.failed, "needs", "need")} a fetch check before content changes.`;
|
|
38080
|
+
if (report.content > 0) return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.content, "returned page")} ${plural$5(report.content, "has", "have")} verified content or search-framing gaps.`;
|
|
38081
|
+
return `${countLabel(report.eligible, "eligible average-position page")} found; ${countLabel(report.returned, "page")} ${plural$5(report.returned, "is", "are")} returned in priority order for investigation.`;
|
|
38040
38082
|
}
|
|
38041
38083
|
function reportRecommendations(items) {
|
|
38042
38084
|
const first = items[0];
|
|
@@ -39097,7 +39139,7 @@ async function updatePostmortemWorkflow(input, dependencies = {}) {
|
|
|
39097
39139
|
//#endregion
|
|
39098
39140
|
//#region packages/core/dist/clicky/api.js
|
|
39099
39141
|
const ENDPOINT$1 = "https://api.clicky.com/api/stats/4";
|
|
39100
|
-
const MAX_BYTES$
|
|
39142
|
+
const MAX_BYTES$3 = 5e6;
|
|
39101
39143
|
var ClickyError = class extends Error {
|
|
39102
39144
|
code;
|
|
39103
39145
|
constructor(code, message) {
|
|
@@ -39124,7 +39166,7 @@ async function clickyStats(site, types, options = {}) {
|
|
|
39124
39166
|
throw new ClickyError("remote", "Could not reach Clicky. Try again.");
|
|
39125
39167
|
});
|
|
39126
39168
|
if (!response.ok) throw new ClickyError("remote", `Clicky returned ${response.status}.`);
|
|
39127
|
-
const groups = await readJson(response, "Clicky", { maxBytes: MAX_BYTES$
|
|
39169
|
+
const groups = await readJson(response, "Clicky", { maxBytes: MAX_BYTES$3 });
|
|
39128
39170
|
if (!Array.isArray(groups)) throw new ClickyError("remote", "Clicky returned invalid data.");
|
|
39129
39171
|
const failed = groups.find((group) => typeof group?.error === "string");
|
|
39130
39172
|
if (failed?.error) throw /sitekey|site id|authenticat|permission/iu.test(failed.error) ? new ClickyError("auth", `Clicky rejected site ID ${site.siteId} or its sitekey.`) : new ClickyError("remote", `Clicky: ${failed.error.slice(0, 200)}`);
|
|
@@ -40879,6 +40921,392 @@ async function withDomainRating(rows, key) {
|
|
|
40879
40921
|
added: true
|
|
40880
40922
|
};
|
|
40881
40923
|
}
|
|
40924
|
+
const MAX_INSPECT_LIMIT = 2e3;
|
|
40925
|
+
const CONCURRENCY = 20;
|
|
40926
|
+
const REFERRING_KEPT = 10;
|
|
40927
|
+
const LABELS = {
|
|
40928
|
+
fix: "the evidence shows a fault on the site",
|
|
40929
|
+
check: "a person should look; the evidence alone does not settle it",
|
|
40930
|
+
expected: "the evidence shows the site works as built, nothing to fix",
|
|
40931
|
+
indexed: "Google indexes it and the live check shows no fault"
|
|
40932
|
+
};
|
|
40933
|
+
const AUTH_SEGMENTS = /* @__PURE__ */ new Set([
|
|
40934
|
+
"account",
|
|
40935
|
+
"admin",
|
|
40936
|
+
"api",
|
|
40937
|
+
"auth",
|
|
40938
|
+
"cart",
|
|
40939
|
+
"checkout",
|
|
40940
|
+
"dashboard",
|
|
40941
|
+
"login",
|
|
40942
|
+
"logout",
|
|
40943
|
+
"my-account",
|
|
40944
|
+
"oauth",
|
|
40945
|
+
"sign-in",
|
|
40946
|
+
"signin",
|
|
40947
|
+
"wp-admin",
|
|
40948
|
+
"wp-login.php"
|
|
40949
|
+
]);
|
|
40950
|
+
const bareHost = (url) => new URL(url).hostname.replace(/^www\./, "");
|
|
40951
|
+
const sameSite = (a, b) => {
|
|
40952
|
+
try {
|
|
40953
|
+
return bareHost(a) === bareHost(b);
|
|
40954
|
+
} catch {
|
|
40955
|
+
return false;
|
|
40956
|
+
}
|
|
40957
|
+
};
|
|
40958
|
+
function variantOf(url, target) {
|
|
40959
|
+
const from = new URL(url);
|
|
40960
|
+
const to = new URL(target);
|
|
40961
|
+
if (from.pathname !== to.pathname || from.search !== to.search) return null;
|
|
40962
|
+
if (!sameSite(url, target)) return null;
|
|
40963
|
+
if (from.protocol !== to.protocol) return "http:// variant";
|
|
40964
|
+
if (from.host !== to.host) return `host variant (${from.host})`;
|
|
40965
|
+
return null;
|
|
40966
|
+
}
|
|
40967
|
+
const samePath = (a, b) => {
|
|
40968
|
+
const [from, to] = [new URL(a), new URL(b)];
|
|
40969
|
+
return from.pathname === to.pathname && from.search === to.search;
|
|
40970
|
+
};
|
|
40971
|
+
const authPath = (url) => AUTH_SEGMENTS.has(new URL(url).pathname.split("/")[1]?.toLowerCase() ?? "");
|
|
40972
|
+
const plural = (count, word) => `${count} ${word}${count === 1 ? "" : "s"}`;
|
|
40973
|
+
function labelOf(e) {
|
|
40974
|
+
const { url, status, live } = e;
|
|
40975
|
+
const { first, hops, last } = live;
|
|
40976
|
+
const indexed = status.verdict === "PASS";
|
|
40977
|
+
const coverage = status.coverage ?? "";
|
|
40978
|
+
const referring = status.referring_urls.filter((item) => item !== url && sameSite(item, url) && !samePath(item, url));
|
|
40979
|
+
const pointers = [e.inSitemap ? "the sitemap lists it" : null, referring.length ? `Google lists ${plural(referring.length, "referring page")} on this site as of its last crawl (${referring[0]}${referring.length > 1 ? ", and more" : ""})` : null].filter(Boolean);
|
|
40980
|
+
if (first.error) return indexed ? {
|
|
40981
|
+
label: "indexed",
|
|
40982
|
+
why: `Google indexes it. The live check failed (${first.error}).`
|
|
40983
|
+
} : {
|
|
40984
|
+
label: "check",
|
|
40985
|
+
why: `The live check failed (${first.error}), so only Google's reason is shown.`
|
|
40986
|
+
};
|
|
40987
|
+
const temporary = hops.find((hop) => hop.kind === "temporary" && new URL(hop.from).host === new URL(hop.to).host);
|
|
40988
|
+
if (temporary) return {
|
|
40989
|
+
label: "fix",
|
|
40990
|
+
why: `${temporary.from} sends a ${temporary.status} (temporary) redirect to ${temporary.to} on the same host; a permanent move uses 301 or 308.`
|
|
40991
|
+
};
|
|
40992
|
+
if (!last) return {
|
|
40993
|
+
label: "fix",
|
|
40994
|
+
why: `The redirects loop or pass 5 hops (${hops.map((hop) => hop.status).join(" > ")}).`
|
|
40995
|
+
};
|
|
40996
|
+
if (last.error) return {
|
|
40997
|
+
label: "check",
|
|
40998
|
+
why: `It redirects to ${last.url}, which could not be fetched (${last.error}).`
|
|
40999
|
+
};
|
|
41000
|
+
const end = last.status ?? 0;
|
|
41001
|
+
const now = hops.length ? `redirects to ${last.url}, which returns ${end}` : `returns ${end} now`;
|
|
41002
|
+
const via = `It ${now}`;
|
|
41003
|
+
if (end === 404 || end === 410) {
|
|
41004
|
+
if (pointers.length) return {
|
|
41005
|
+
label: "fix",
|
|
41006
|
+
why: `${via}, and ${pointers.join(" and ")}.`
|
|
41007
|
+
};
|
|
41008
|
+
if (hops.length) return {
|
|
41009
|
+
label: "fix",
|
|
41010
|
+
why: `${via}: a redirect to a missing page.`
|
|
41011
|
+
};
|
|
41012
|
+
return {
|
|
41013
|
+
label: "expected",
|
|
41014
|
+
why: `${via}, and neither the sitemap nor any referring page Google lists on this site points to it.`
|
|
41015
|
+
};
|
|
41016
|
+
}
|
|
41017
|
+
if ((end === 401 || end === 403) && !hops.length) {
|
|
41018
|
+
if (e.inSitemap) return {
|
|
41019
|
+
label: "fix",
|
|
41020
|
+
why: `${via}, but the sitemap lists it.`
|
|
41021
|
+
};
|
|
41022
|
+
if (end === 401 && first.www_authenticate) return {
|
|
41023
|
+
label: "expected",
|
|
41024
|
+
why: `${via} with a WWW-Authenticate header (${first.www_authenticate.split(/\s/)[0]}), so the server asks for sign-in here.`
|
|
41025
|
+
};
|
|
41026
|
+
if (authPath(url)) return {
|
|
41027
|
+
label: "expected",
|
|
41028
|
+
why: `${via} on a path that looks like a sign-in area (heuristic: from the path only).`
|
|
41029
|
+
};
|
|
41030
|
+
return {
|
|
41031
|
+
label: "check",
|
|
41032
|
+
why: `${via} with no WWW-Authenticate header and no sign-in path, so the evidence does not show whether it should be public.`
|
|
41033
|
+
};
|
|
41034
|
+
}
|
|
41035
|
+
if (hops.length) {
|
|
41036
|
+
const codes = hops.map((hop) => hop.status).join(" > ");
|
|
41037
|
+
if (end !== 200) return {
|
|
41038
|
+
label: "check",
|
|
41039
|
+
why: `${via} (${codes}).`
|
|
41040
|
+
};
|
|
41041
|
+
if (e.inSitemap) return {
|
|
41042
|
+
label: "fix",
|
|
41043
|
+
why: `The sitemap lists it, but it redirects (${codes}) to ${last.url}.`
|
|
41044
|
+
};
|
|
41045
|
+
const declared = last.canonical === last.url ? "declares itself canonical" : e.sitemap.has(last.url) ? "is in the sitemap" : null;
|
|
41046
|
+
const variant = variantOf(url, last.url);
|
|
41047
|
+
const moved = `${variant ? `An ${variant}` : "A permanent redirect"}: ${codes} to ${last.url}`;
|
|
41048
|
+
const stale = indexed ? " Google still indexes this URL as of its last crawl." : "";
|
|
41049
|
+
const links = referring.length ? ` ${pointers.at(-1)}.` : "";
|
|
41050
|
+
if (declared) return {
|
|
41051
|
+
label: "expected",
|
|
41052
|
+
why: `${moved}, which returns 200 and ${declared}.${stale}${links}`
|
|
41053
|
+
};
|
|
41054
|
+
return {
|
|
41055
|
+
label: "check",
|
|
41056
|
+
why: `${moved}, which declares ${last.canonical ? `${last.canonical} as canonical` : "no canonical"}.${stale}${links}`
|
|
41057
|
+
};
|
|
41058
|
+
}
|
|
41059
|
+
if (/^(Crawled|Discovered) - currently not indexed$|^URL is unknown to Google$/i.test(coverage)) return {
|
|
41060
|
+
label: "check",
|
|
41061
|
+
why: `${coverage.startsWith("Crawled") ? "Google crawled it but has not indexed it, and gives no further reason" : coverage.startsWith("Discovered") ? "Google knows the URL but has not crawled it yet" : "Google has no record of this URL"}.${e.inSitemap ? " The sitemap lists it." : ""} It returns ${end} now.`
|
|
41062
|
+
};
|
|
41063
|
+
const robots = last.robots ? ` with robots "${last.robots}"` : " with no robots meta tag or X-Robots-Tag header";
|
|
41064
|
+
if (/noindex/i.test(coverage) && !/noindex/i.test(last.robots ?? "")) return {
|
|
41065
|
+
label: "check",
|
|
41066
|
+
why: `Google saw noindex when it last crawled${status.last_crawl ? ` (${status.last_crawl.slice(0, 10)})` : ""}, but it ${now}${robots}, so Google's reason may be out of date.${e.inSitemap ? " The sitemap lists it." : ""}`
|
|
41067
|
+
};
|
|
41068
|
+
const declaredCanonical = status.user_canonical ?? last.canonical;
|
|
41069
|
+
if (status.google_canonical && declaredCanonical && status.google_canonical !== declaredCanonical) return {
|
|
41070
|
+
label: "check",
|
|
41071
|
+
why: `Google chose ${status.google_canonical} as canonical; the page declares ${declaredCanonical}.`
|
|
41072
|
+
};
|
|
41073
|
+
if (indexed) return end === 200 ? {
|
|
41074
|
+
label: "indexed",
|
|
41075
|
+
why: "Google indexes it and it returns 200 now."
|
|
41076
|
+
} : {
|
|
41077
|
+
label: "check",
|
|
41078
|
+
why: `Google indexes it, but it ${now}.`
|
|
41079
|
+
};
|
|
41080
|
+
if (e.inSitemap) return {
|
|
41081
|
+
label: "fix",
|
|
41082
|
+
why: `The sitemap lists it, but Google does not index it. It ${now}${robots}.`
|
|
41083
|
+
};
|
|
41084
|
+
if (status.google_canonical && status.google_canonical === status.user_canonical && status.google_canonical !== url) return {
|
|
41085
|
+
label: "expected",
|
|
41086
|
+
why: `The page declares ${status.user_canonical} as canonical and Google uses it.`
|
|
41087
|
+
};
|
|
41088
|
+
return {
|
|
41089
|
+
label: "check",
|
|
41090
|
+
why: /noindex/i.test(last.robots ?? "") ? `${via}${robots} and the sitemap does not list it. Whether it should be indexed is the site's call.` : `${via}${robots}. No rule here settles this reason.`
|
|
41091
|
+
};
|
|
41092
|
+
}
|
|
41093
|
+
function indexingRow(e, sources, inspectedAt) {
|
|
41094
|
+
const { status, live } = e;
|
|
41095
|
+
const { label, why } = labelOf(e);
|
|
41096
|
+
return {
|
|
41097
|
+
reason: status.coverage ?? status.verdict ?? "No reason returned",
|
|
41098
|
+
url: e.url,
|
|
41099
|
+
label,
|
|
41100
|
+
why,
|
|
41101
|
+
verdict: status.verdict,
|
|
41102
|
+
indexing: status.indexing,
|
|
41103
|
+
status: live.first.status,
|
|
41104
|
+
redirects: live.hops.length ? live.hops.map((hop) => `${hop.status} ${hop.kind}`).join(" > ") : null,
|
|
41105
|
+
redirects_to: live.first.redirects_to,
|
|
41106
|
+
final_url: live.last?.url ?? null,
|
|
41107
|
+
final_status: live.last?.status ?? null,
|
|
41108
|
+
page_canonical: live.last?.canonical ?? null,
|
|
41109
|
+
user_canonical: status.user_canonical,
|
|
41110
|
+
google_canonical: status.google_canonical,
|
|
41111
|
+
in_sitemap: e.inSitemap,
|
|
41112
|
+
referring_urls: status.referring_urls.slice(0, REFERRING_KEPT),
|
|
41113
|
+
last_crawl: status.last_crawl,
|
|
41114
|
+
inspected_at: inspectedAt,
|
|
41115
|
+
page_fetch: status.page_fetch,
|
|
41116
|
+
robots_txt: status.robots_txt,
|
|
41117
|
+
crawled_as: status.crawled_as,
|
|
41118
|
+
sources,
|
|
41119
|
+
...live.first.error ? { live_error: live.first.error } : {}
|
|
41120
|
+
};
|
|
41121
|
+
}
|
|
41122
|
+
const byText = (a, b) => a < b ? -1 : a > b ? 1 : 0;
|
|
41123
|
+
function sortRows(rows) {
|
|
41124
|
+
return [...rows].sort((a, b) => byText(a.reason, b.reason) || byText(a.url, b.url));
|
|
41125
|
+
}
|
|
41126
|
+
function byReason(rows) {
|
|
41127
|
+
const groups = /* @__PURE__ */ new Map();
|
|
41128
|
+
for (const row of rows) {
|
|
41129
|
+
const counts = groups.get(row.reason) ?? {
|
|
41130
|
+
fix: 0,
|
|
41131
|
+
check: 0,
|
|
41132
|
+
expected: 0,
|
|
41133
|
+
indexed: 0
|
|
41134
|
+
};
|
|
41135
|
+
counts[row.label] += 1;
|
|
41136
|
+
groups.set(row.reason, counts);
|
|
41137
|
+
}
|
|
41138
|
+
return [...groups].sort((a, b) => byText(a[0], b[0])).map(([reason, counts]) => ({
|
|
41139
|
+
reason,
|
|
41140
|
+
urls: counts.fix + counts.check + counts.expected + counts.indexed,
|
|
41141
|
+
...counts
|
|
41142
|
+
}));
|
|
41143
|
+
}
|
|
41144
|
+
const isCode = (error, codes) => codes.includes(String(error?.code));
|
|
41145
|
+
function knownUrls(input) {
|
|
41146
|
+
const found = /* @__PURE__ */ new Map();
|
|
41147
|
+
const add = (url, source) => {
|
|
41148
|
+
const sources = found.get(url) ?? [];
|
|
41149
|
+
if (!sources.includes(source)) sources.push(source);
|
|
41150
|
+
found.set(url, sources);
|
|
41151
|
+
};
|
|
41152
|
+
const pages = input.pages.filter((page) => page.impressions > 0 && !page.url.includes("#")).sort((a, b) => b.clicks - a.clicks || b.impressions - a.impressions || byText(a.url, b.url));
|
|
41153
|
+
for (const url of input.lost) add(url, "lost_clicks");
|
|
41154
|
+
for (const page of pages) add(page.url, "search_console");
|
|
41155
|
+
for (const url of input.sitemap.urls) add(url, "sitemap");
|
|
41156
|
+
if (input.home) add(input.home, "home");
|
|
41157
|
+
return {
|
|
41158
|
+
urls: [...found].map(([url, sources]) => ({
|
|
41159
|
+
url,
|
|
41160
|
+
sources
|
|
41161
|
+
})),
|
|
41162
|
+
sitemap: input.sitemap,
|
|
41163
|
+
searchConsole: pages.length,
|
|
41164
|
+
searchConsoleCapped: input.pagesCapped,
|
|
41165
|
+
lostClicks: input.lost.length
|
|
41166
|
+
};
|
|
41167
|
+
}
|
|
41168
|
+
async function inspectKnown(site, known, deps, asked = 100) {
|
|
41169
|
+
const cap = Math.min(Math.max(1, Math.floor(asked)), MAX_INSPECT_LIMIT);
|
|
41170
|
+
const wanted = known.urls.slice(0, cap);
|
|
41171
|
+
let saved = /* @__PURE__ */ new Map();
|
|
41172
|
+
try {
|
|
41173
|
+
saved = deps.savedInspections?.read(site, wanted.map((item) => item.url)) ?? saved;
|
|
41174
|
+
} catch {}
|
|
41175
|
+
const missing = wanted.filter((item) => !saved.has(item.url));
|
|
41176
|
+
const quota = deps.inspectionQuota ? await deps.inspectionQuota(site).catch(() => null) : null;
|
|
41177
|
+
const left = quota ? Math.max(0, quota.limit - quota.used) : missing.length;
|
|
41178
|
+
const asking = new Set(missing.slice(0, left).map((item) => item.url));
|
|
41179
|
+
const queue = wanted.filter((item) => saved.has(item.url) || asking.has(item.url));
|
|
41180
|
+
const quotaShort = left < missing.length;
|
|
41181
|
+
const statuses = /* @__PURE__ */ new Map();
|
|
41182
|
+
const rows = [];
|
|
41183
|
+
const failed = [];
|
|
41184
|
+
const sitemap = new Set(known.sitemap.urls);
|
|
41185
|
+
const now = () => (deps.now?.() ?? /* @__PURE__ */ new Date()).toISOString();
|
|
41186
|
+
let quotaStop = false;
|
|
41187
|
+
let next = 0;
|
|
41188
|
+
let fatal = null;
|
|
41189
|
+
const worker = async () => {
|
|
41190
|
+
while (fatal === null && next < queue.length) {
|
|
41191
|
+
const item = queue[next++];
|
|
41192
|
+
const stored = saved.get(item.url);
|
|
41193
|
+
let status;
|
|
41194
|
+
let inspectedAt;
|
|
41195
|
+
if (stored) {
|
|
41196
|
+
status = stored.status;
|
|
41197
|
+
inspectedAt = stored.inspectedAt;
|
|
41198
|
+
} else {
|
|
41199
|
+
if (quotaStop) continue;
|
|
41200
|
+
try {
|
|
41201
|
+
status = await deps.inspect(site, item.url);
|
|
41202
|
+
inspectedAt = now();
|
|
41203
|
+
} catch (error) {
|
|
41204
|
+
if (isCode(error, ["RATE_LIMITED"])) quotaStop = true;
|
|
41205
|
+
else if (isCode(error, [
|
|
41206
|
+
"AUTH_EXPIRED",
|
|
41207
|
+
"AUTH_REQUIRED",
|
|
41208
|
+
"NOT_CONNECTED"
|
|
41209
|
+
])) fatal = error;
|
|
41210
|
+
else failed.push({
|
|
41211
|
+
url: item.url,
|
|
41212
|
+
error: error instanceof Error ? error.message : String(error)
|
|
41213
|
+
});
|
|
41214
|
+
continue;
|
|
41215
|
+
}
|
|
41216
|
+
try {
|
|
41217
|
+
deps.savedInspections?.save(site, item.url, status);
|
|
41218
|
+
} catch {}
|
|
41219
|
+
}
|
|
41220
|
+
statuses.set(item.url, status);
|
|
41221
|
+
const live = await deps.liveCheck(item.url);
|
|
41222
|
+
rows.push(indexingRow({
|
|
41223
|
+
url: item.url,
|
|
41224
|
+
status,
|
|
41225
|
+
live,
|
|
41226
|
+
inSitemap: item.sources.includes("sitemap"),
|
|
41227
|
+
sitemap
|
|
41228
|
+
}, item.sources, inspectedAt));
|
|
41229
|
+
}
|
|
41230
|
+
};
|
|
41231
|
+
await Promise.all(Array.from({ length: CONCURRENCY }, worker));
|
|
41232
|
+
if (fatal !== null) throw fatal;
|
|
41233
|
+
const reused = rows.filter((row) => saved.has(row.url));
|
|
41234
|
+
const oldest = reused.reduce((first, row) => first === null || row.inspected_at < first ? row.inspected_at : first, null);
|
|
41235
|
+
return {
|
|
41236
|
+
rows: sortRows(rows),
|
|
41237
|
+
statuses,
|
|
41238
|
+
known,
|
|
41239
|
+
limit: cap,
|
|
41240
|
+
inspected: rows.length,
|
|
41241
|
+
fromSaved: reused.length,
|
|
41242
|
+
oldestSaved: oldest,
|
|
41243
|
+
failed: failed.sort((a, b) => byText(a.url, b.url)),
|
|
41244
|
+
stoppedBy: quotaStop || quotaShort ? "quota" : known.urls.length > wanted.length ? "limit" : null,
|
|
41245
|
+
quota
|
|
41246
|
+
};
|
|
41247
|
+
}
|
|
41248
|
+
const n = (value) => value.toLocaleString("en-US");
|
|
41249
|
+
function liveCell(row) {
|
|
41250
|
+
if (row.live_error) return `error: ${row.live_error}`;
|
|
41251
|
+
if (!row.redirects) return String(row.status ?? "");
|
|
41252
|
+
return `${row.redirects.replace(/ (permanent|temporary)/g, "")} > ${row.final_status ?? "loop"}`;
|
|
41253
|
+
}
|
|
41254
|
+
function indexingText(site, run, page, origin, askMore, now) {
|
|
41255
|
+
const { known } = run;
|
|
41256
|
+
const total = known.urls.length;
|
|
41257
|
+
const notInspected = total - run.inspected - run.failed.length;
|
|
41258
|
+
const sources = [
|
|
41259
|
+
`lost-click pages ${n(known.lostClicks)}`,
|
|
41260
|
+
`Search Console pages with impressions in the last 28 days ${n(known.searchConsole)}${known.searchConsoleCapped ? ", capped" : ""}`,
|
|
41261
|
+
`sitemap ${n(known.sitemap.urls.length)} from ${plural(known.sitemap.files, "file")}${known.sitemap.partial ? ", partial" : ""}`
|
|
41262
|
+
];
|
|
41263
|
+
const lowerBound = known.sitemap.partial || known.searchConsoleCapped ? " at least" : "";
|
|
41264
|
+
const asked = run.inspected - run.fromSaved;
|
|
41265
|
+
const most = Math.min(total, MAX_INSPECT_LIMIT, run.quota ? run.inspected + run.quota.limit - run.quota.used - asked : MAX_INSPECT_LIMIT);
|
|
41266
|
+
const days = run.oldestSaved ? Math.floor((now.getTime() - Date.parse(run.oldestSaved)) / 864e5) : 0;
|
|
41267
|
+
const savedNote = run.fromSaved ? ` (${n(asked)} asked of Google now, ${n(run.fromSaved)} from saved results; the oldest was inspected ${run.oldestSaved?.slice(0, 16).replace("T", " ")} UTC, ${days === 0 ? "under a day" : plural(days, "day")} ago. Saved results are reused for 7 days)` : "";
|
|
41268
|
+
const stop = run.stoppedBy === "quota" ? "The URL Inspection quota stopped it." : run.stoppedBy === "limit" ? `It stopped at ${n(run.limit)}, the number of inspections for this run${askMore && most > run.limit ? `; to inspect more, up to the quota left today: ${askMore(most)}` : ""}.` : "Every known URL has a result.";
|
|
41269
|
+
const lines = [
|
|
41270
|
+
`Google's indexing reasons for ${site}, from URL Inspection. Search Console's Page indexing report has no API, so known URLs are inspected one at a time, in a fixed order (lost-click pages, Search Console pages by clicks then impressions, sitemap URLs, then the rest), and checked live from here.`,
|
|
41271
|
+
`Known URLs:${lowerBound} ${n(total)} (${sources.join("; ")}). Inspected ${n(run.inspected)} of ${n(total)}${savedNote}. ${stop}${notInspected > 0 ? ` The ${n(notInspected)} URLs not inspected have no result here, so this is not an all-clear for them.` : ""}${known.sitemap.notes.length ? ` Sitemap: ${known.sitemap.notes.join("; ")}.` : ""}`,
|
|
41272
|
+
run.quota ? `URL Inspection quota: this machine had used ${n(run.quota.used)} of ${n(run.quota.limit)} today for this property before the run. Google also counts inspections from other tools.` : "URL Inspection quota: not counted on this server; Google stops inspections at 2,000 a day per property."
|
|
41273
|
+
];
|
|
41274
|
+
if (run.failed.length) lines.push(`URL Inspection failed for ${plural(run.failed.length, "URL")}: ${run.failed.slice(0, 5).map((item) => `${item.url} (${item.error})`).join("; ")}${run.failed.length > 5 ? "; and more" : ""}.`);
|
|
41275
|
+
const groups = byReason(run.rows);
|
|
41276
|
+
if (groups.length) lines.push(`By Google's reason, with labels from the rules below (not Google's):\n${table(groups, [
|
|
41277
|
+
"reason",
|
|
41278
|
+
"urls",
|
|
41279
|
+
"fix",
|
|
41280
|
+
"check",
|
|
41281
|
+
"expected",
|
|
41282
|
+
"indexed"
|
|
41283
|
+
], null)}`);
|
|
41284
|
+
const listed = run.rows.filter((row) => row.label !== "indexed");
|
|
41285
|
+
const shown = page.summary ? listed.filter((row) => row.label === "fix" || row.label === "check") : listed;
|
|
41286
|
+
const indexedCount = run.rows.length - listed.length;
|
|
41287
|
+
const expectedCount = listed.length - shown.length;
|
|
41288
|
+
const pageRows = shown.slice(page.offset, page.offset + page.size).map((row) => ({
|
|
41289
|
+
reason: row.reason,
|
|
41290
|
+
label: row.label,
|
|
41291
|
+
url: row.url,
|
|
41292
|
+
live: liveCell(row),
|
|
41293
|
+
final_url: row.final_url === row.url ? "" : row.final_url,
|
|
41294
|
+
in_sitemap: row.in_sitemap ? "yes" : "no",
|
|
41295
|
+
last_crawl: row.last_crawl?.slice(0, 10) ?? "",
|
|
41296
|
+
why: row.why
|
|
41297
|
+
}));
|
|
41298
|
+
const end = page.offset + pageRows.length;
|
|
41299
|
+
const more = shown.length - end;
|
|
41300
|
+
const others = page.summary ? [
|
|
41301
|
+
more ? `${more} more labelled fix or check` : null,
|
|
41302
|
+
expectedCount ? `${expectedCount} expected` : null,
|
|
41303
|
+
indexedCount ? `${indexedCount} indexed with no fault found (in the full result)` : null
|
|
41304
|
+
].filter(Boolean) : [];
|
|
41305
|
+
if (page.summary && (more || expectedCount)) others.push(`every row but the indexed ones: ${page.more(0)}`);
|
|
41306
|
+
lines.push(`${shown.length === 0 ? `No inspected URL is labelled ${page.summary ? "fix or check" : "fix, check or expected"}.` : `${page.summary ? "URLs labelled fix or check" : "URLs that are not indexed or show a fault"}, ${page.offset + 1} to ${end} of ${shown.length}:\n${table(pageRows, Object.keys(pageRows[0] ?? {}), origin, 400)}`}${others.length ? `\nNot listed here: ${others.join("; ")}.` : ""}`, `Labels: ${Object.entries(LABELS).map(([label, text]) => `${label}, ${text}`).join("; ")}. Rules: a same-host 302 or 307 is fix; a 404 or 410 that the sitemap or a referring page on this site points to is fix; a sitemap URL that redirects or that Google does not index is fix; a URL outside the sitemap (such as an http:// or host variant) that permanently redirects to a page declaring itself canonical is expected; a 401 with a WWW-Authenticate header is expected, and a 401 or 403 on a sign-in path is expected by heuristic; "Crawled - currently not indexed", "Discovered - currently not indexed" and a Google canonical that differs from the declared one are check. Referring pages are as of Google's last crawl. Live checks are server responses, so a browser-only HSTS "307 Internal Redirect" never shows here.`);
|
|
41307
|
+
if (!page.summary && more) lines.push(`More rows: ${page.more(end)}.`);
|
|
41308
|
+
return lines;
|
|
41309
|
+
}
|
|
40882
41310
|
//#endregion
|
|
40883
41311
|
//#region packages/core/dist/commands/internal-links.js
|
|
40884
41312
|
const SUMMARY_ROWS = 10;
|
|
@@ -40941,8 +41369,8 @@ function internalLinksText(report, options) {
|
|
|
40941
41369
|
//#endregion
|
|
40942
41370
|
//#region packages/core/dist/commands/page-facts.js
|
|
40943
41371
|
const MAX_PAGES = 20;
|
|
40944
|
-
const TIMEOUT_MS$
|
|
40945
|
-
const MAX_BYTES$
|
|
41372
|
+
const TIMEOUT_MS$2 = 6e3;
|
|
41373
|
+
const MAX_BYTES$2 = 1e6;
|
|
40946
41374
|
const clean = (value) => value?.trim().replace(/\s+/g, " ") || null;
|
|
40947
41375
|
async function boundedText(response) {
|
|
40948
41376
|
if (!response.body) return "";
|
|
@@ -40950,7 +41378,7 @@ async function boundedText(response) {
|
|
|
40950
41378
|
const decoder = new TextDecoder();
|
|
40951
41379
|
let text = "";
|
|
40952
41380
|
let total = 0;
|
|
40953
|
-
while (total < MAX_BYTES$
|
|
41381
|
+
while (total < MAX_BYTES$2) {
|
|
40954
41382
|
const { done, value } = await reader.read();
|
|
40955
41383
|
if (done) return text + decoder.decode();
|
|
40956
41384
|
total += value.byteLength;
|
|
@@ -41030,12 +41458,14 @@ async function facts(url) {
|
|
|
41030
41458
|
"user-agent": SEO_CRAWLER_USER_AGENT,
|
|
41031
41459
|
accept: "text/html"
|
|
41032
41460
|
},
|
|
41033
|
-
signal: AbortSignal.timeout(TIMEOUT_MS$
|
|
41461
|
+
signal: AbortSignal.timeout(TIMEOUT_MS$2)
|
|
41034
41462
|
});
|
|
41035
41463
|
out.status = response.status;
|
|
41036
41464
|
const location = response.headers.get("location");
|
|
41037
41465
|
if (location) out.redirects_to = new URL(location, url).toString();
|
|
41038
41466
|
const headerRobots = response.headers.get("x-robots-tag");
|
|
41467
|
+
const auth = response.headers.get("www-authenticate");
|
|
41468
|
+
if (auth) out.www_authenticate = auth.slice(0, 200);
|
|
41039
41469
|
if (response.status !== 200 || !response.headers.get("content-type")?.includes("html")) {
|
|
41040
41470
|
out.robots = headerRobots;
|
|
41041
41471
|
await response.body?.cancel();
|
|
@@ -41052,6 +41482,37 @@ async function facts(url) {
|
|
|
41052
41482
|
function checkPages(urls) {
|
|
41053
41483
|
return Promise.all(urls.slice(0, MAX_PAGES).map(facts));
|
|
41054
41484
|
}
|
|
41485
|
+
const MAX_HOPS = 5;
|
|
41486
|
+
const redirectKind = (status) => status === 301 || status === 308 ? "permanent" : status === 302 || status === 303 || status === 307 ? "temporary" : null;
|
|
41487
|
+
async function liveCheck(url) {
|
|
41488
|
+
const first = await facts(url);
|
|
41489
|
+
const hops = [];
|
|
41490
|
+
const seen = /* @__PURE__ */ new Set([url]);
|
|
41491
|
+
let last = first;
|
|
41492
|
+
while (last.redirects_to && last.status !== null) {
|
|
41493
|
+
const kind = redirectKind(last.status);
|
|
41494
|
+
if (!kind) break;
|
|
41495
|
+
const to = last.redirects_to;
|
|
41496
|
+
hops.push({
|
|
41497
|
+
from: last.url,
|
|
41498
|
+
status: last.status,
|
|
41499
|
+
to,
|
|
41500
|
+
kind
|
|
41501
|
+
});
|
|
41502
|
+
if (seen.has(to) || hops.length > MAX_HOPS) return {
|
|
41503
|
+
first,
|
|
41504
|
+
hops,
|
|
41505
|
+
last: null
|
|
41506
|
+
};
|
|
41507
|
+
seen.add(to);
|
|
41508
|
+
last = await facts(to);
|
|
41509
|
+
}
|
|
41510
|
+
return {
|
|
41511
|
+
first,
|
|
41512
|
+
hops,
|
|
41513
|
+
last
|
|
41514
|
+
};
|
|
41515
|
+
}
|
|
41055
41516
|
function temporaryRedirectNote(pages) {
|
|
41056
41517
|
const temporary = pages.filter((page) => (page.status === 302 || page.status === 307) && page.redirects_to !== null && new URL(page.redirects_to).host === new URL(page.url).host);
|
|
41057
41518
|
if (!temporary.length) return null;
|
|
@@ -41646,17 +42107,137 @@ async function runResearch(input, keys, options) {
|
|
|
41646
42107
|
};
|
|
41647
42108
|
}
|
|
41648
42109
|
//#endregion
|
|
42110
|
+
//#region packages/core/dist/commands/sitemap-urls.js
|
|
42111
|
+
const MAX_FILES = 20;
|
|
42112
|
+
const MAX_URLS = 5e4;
|
|
42113
|
+
const MAX_BYTES$1 = 2e7;
|
|
42114
|
+
const TIMEOUT_MS$1 = 1e4;
|
|
42115
|
+
async function boundedBytes(body) {
|
|
42116
|
+
if (!body) return /* @__PURE__ */ new Uint8Array();
|
|
42117
|
+
const reader = body.getReader();
|
|
42118
|
+
const chunks = [];
|
|
42119
|
+
let total = 0;
|
|
42120
|
+
while (true) {
|
|
42121
|
+
const { done, value } = await reader.read();
|
|
42122
|
+
if (done) break;
|
|
42123
|
+
total += value.byteLength;
|
|
42124
|
+
if (total > MAX_BYTES$1) {
|
|
42125
|
+
await reader.cancel();
|
|
42126
|
+
return null;
|
|
42127
|
+
}
|
|
42128
|
+
chunks.push(value);
|
|
42129
|
+
}
|
|
42130
|
+
const bytes = new Uint8Array(total);
|
|
42131
|
+
let offset = 0;
|
|
42132
|
+
for (const chunk of chunks) {
|
|
42133
|
+
bytes.set(chunk, offset);
|
|
42134
|
+
offset += chunk.byteLength;
|
|
42135
|
+
}
|
|
42136
|
+
return bytes;
|
|
42137
|
+
}
|
|
42138
|
+
async function get(url) {
|
|
42139
|
+
const response = await fetch(url, {
|
|
42140
|
+
headers: { "user-agent": SEO_CRAWLER_USER_AGENT },
|
|
42141
|
+
signal: AbortSignal.timeout(TIMEOUT_MS$1)
|
|
42142
|
+
});
|
|
42143
|
+
if (!response.ok) {
|
|
42144
|
+
await response.body?.cancel();
|
|
42145
|
+
throw new Error(`answered ${response.status}`);
|
|
42146
|
+
}
|
|
42147
|
+
let bytes = await boundedBytes(response.body);
|
|
42148
|
+
if (bytes && bytes[0] === 31 && bytes[1] === 139) bytes = await boundedBytes(new Response(bytes).body?.pipeThrough(new DecompressionStream("gzip")) ?? null);
|
|
42149
|
+
if (!bytes) throw new Error(`is larger than ${MAX_BYTES$1} bytes`);
|
|
42150
|
+
return new TextDecoder().decode(bytes);
|
|
42151
|
+
}
|
|
42152
|
+
const ENTITIES = {
|
|
42153
|
+
amp: "&",
|
|
42154
|
+
lt: "<",
|
|
42155
|
+
gt: ">",
|
|
42156
|
+
quot: "\"",
|
|
42157
|
+
apos: "'"
|
|
42158
|
+
};
|
|
42159
|
+
function locs(xml) {
|
|
42160
|
+
return [...xml.matchAll(/<loc>\s*(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?\s*<\/loc>/gi)].map((match) => (match[1] ?? "").trim().replace(/&(amp|lt|gt|quot|apos);/g, (_, name) => ENTITIES[name] ?? ""));
|
|
42161
|
+
}
|
|
42162
|
+
const inside = (site, url) => {
|
|
42163
|
+
try {
|
|
42164
|
+
return assertUrlMatchesGscProperty(site, url);
|
|
42165
|
+
} catch {
|
|
42166
|
+
return null;
|
|
42167
|
+
}
|
|
42168
|
+
};
|
|
42169
|
+
async function sitemapUrls(site) {
|
|
42170
|
+
const origin = new URL(originOf(site)).origin;
|
|
42171
|
+
const notes = [];
|
|
42172
|
+
const declared = [...(await get(`${origin}/robots.txt`).catch(() => "")).matchAll(/^\s*sitemap:\s*(\S+)/gim)].map((match) => match[1] ?? "").filter(Boolean);
|
|
42173
|
+
const queue = declared.length ? declared : [`${origin}/sitemap.xml`];
|
|
42174
|
+
const seenFiles = /* @__PURE__ */ new Set();
|
|
42175
|
+
const urls = /* @__PURE__ */ new Set();
|
|
42176
|
+
let files = 0;
|
|
42177
|
+
let partial = false;
|
|
42178
|
+
while (queue.length) {
|
|
42179
|
+
const file = queue.shift();
|
|
42180
|
+
if (seenFiles.has(file)) continue;
|
|
42181
|
+
seenFiles.add(file);
|
|
42182
|
+
if (files >= MAX_FILES) {
|
|
42183
|
+
partial = true;
|
|
42184
|
+
notes.push(`stopped at ${MAX_FILES} sitemap files`);
|
|
42185
|
+
break;
|
|
42186
|
+
}
|
|
42187
|
+
files += 1;
|
|
42188
|
+
let xml;
|
|
42189
|
+
try {
|
|
42190
|
+
xml = await get(file);
|
|
42191
|
+
} catch (error) {
|
|
42192
|
+
if (!declared.length && files === 1) {
|
|
42193
|
+
notes.push(`robots.txt names no sitemap and ${file} ${error instanceof Error ? error.message : "could not be read"}`);
|
|
42194
|
+
continue;
|
|
42195
|
+
}
|
|
42196
|
+
partial = true;
|
|
42197
|
+
notes.push(`${file} ${error instanceof Error ? error.message : "could not be read"}`);
|
|
42198
|
+
continue;
|
|
42199
|
+
}
|
|
42200
|
+
if (/<sitemapindex[\s>]/i.test(xml)) {
|
|
42201
|
+
queue.push(...locs(xml));
|
|
42202
|
+
continue;
|
|
42203
|
+
}
|
|
42204
|
+
for (const loc of locs(xml)) {
|
|
42205
|
+
const url = inside(site, loc);
|
|
42206
|
+
if (!url) continue;
|
|
42207
|
+
if (urls.size >= MAX_URLS) {
|
|
42208
|
+
partial = true;
|
|
42209
|
+
notes.push(`stopped at ${MAX_URLS.toLocaleString("en-US")} URLs`);
|
|
42210
|
+
queue.length = 0;
|
|
42211
|
+
break;
|
|
42212
|
+
}
|
|
42213
|
+
urls.add(url);
|
|
42214
|
+
}
|
|
42215
|
+
}
|
|
42216
|
+
return {
|
|
42217
|
+
urls: [...urls],
|
|
42218
|
+
files,
|
|
42219
|
+
partial,
|
|
42220
|
+
notes: [...new Set(notes)]
|
|
42221
|
+
};
|
|
42222
|
+
}
|
|
42223
|
+
//#endregion
|
|
41649
42224
|
//#region packages/core/dist/commands/site.js
|
|
41650
42225
|
function indexStatusFrom(result) {
|
|
41651
42226
|
const status = result?.inspectionResult?.indexStatusResult ?? result?.indexStatusResult ?? {};
|
|
41652
42227
|
const text = (value) => typeof value === "string" ? value : null;
|
|
42228
|
+
const list = (value) => Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
41653
42229
|
return {
|
|
41654
42230
|
verdict: text(status.verdict),
|
|
41655
42231
|
coverage: text(status.coverageState),
|
|
41656
42232
|
indexing: text(status.indexingState),
|
|
41657
42233
|
last_crawl: text(status.lastCrawlTime),
|
|
41658
42234
|
google_canonical: text(status.googleCanonical),
|
|
41659
|
-
user_canonical: text(status.userCanonical)
|
|
42235
|
+
user_canonical: text(status.userCanonical),
|
|
42236
|
+
page_fetch: text(status.pageFetchState),
|
|
42237
|
+
robots_txt: text(status.robotsTxtState),
|
|
42238
|
+
crawled_as: text(status.crawledAs),
|
|
42239
|
+
referring_urls: list(status.referringUrls),
|
|
42240
|
+
sitemaps: list(status.sitemap)
|
|
41660
42241
|
};
|
|
41661
42242
|
}
|
|
41662
42243
|
async function runPage(input, deps) {
|
|
@@ -41677,13 +42258,22 @@ async function runPage(input, deps) {
|
|
|
41677
42258
|
}]
|
|
41678
42259
|
});
|
|
41679
42260
|
const full = input.detail === "full";
|
|
41680
|
-
const [
|
|
41681
|
-
deps.
|
|
41682
|
-
deps.inspect(input.site, input.url),
|
|
42261
|
+
const [live, indexed, sitemap, now, then, markdown] = await Promise.all([
|
|
42262
|
+
deps.liveCheck(input.url),
|
|
42263
|
+
deps.inspect(input.site, input.url).catch(() => null),
|
|
42264
|
+
sitemapUrls(input.site),
|
|
41683
42265
|
query(range),
|
|
41684
42266
|
query(before),
|
|
41685
42267
|
full && deps.pageMarkdown ? deps.pageMarkdown(input.url).catch((error) => ({ error: error instanceof Error ? error.message : String(error) })) : null
|
|
41686
42268
|
]);
|
|
42269
|
+
const facts = live.first;
|
|
42270
|
+
const reason = indexed ? indexingRow({
|
|
42271
|
+
url: input.url,
|
|
42272
|
+
status: indexed,
|
|
42273
|
+
live,
|
|
42274
|
+
inSitemap: sitemap.urls.includes(input.url),
|
|
42275
|
+
sitemap: new Set(sitemap.urls)
|
|
42276
|
+
}, [], (deps.now?.() ?? /* @__PURE__ */ new Date()).toISOString()) : null;
|
|
41687
42277
|
const old = new Map(then.rows.map((row) => [row.keys[0], row]));
|
|
41688
42278
|
const seen = /* @__PURE__ */ new Set();
|
|
41689
42279
|
const rows = now.rows.map((row) => {
|
|
@@ -41721,8 +42311,9 @@ async function runPage(input, deps) {
|
|
|
41721
42311
|
rowsText({
|
|
41722
42312
|
title: `${input.url} in ${input.site}, ${range.startDate} to ${range.endDate} against ${before.startDate} to ${before.endDate}.`,
|
|
41723
42313
|
notes: [
|
|
41724
|
-
`Live page (fetched now
|
|
42314
|
+
`Live page (fetched now):\n${pageFactsTable([facts], null)}${live.hops.length ? `\nRedirects: ${live.hops.map((hop) => `${hop.status} to ${hop.to}`).join(", ")}${live.last ? `, which returns ${live.last.status ?? live.last.error}` : ", then loop or pass 5 hops"}.` : ""}`,
|
|
41725
42315
|
indexed ? `Google index status:\n${table([indexed], Object.keys(indexed), null)}` : "Google index status: not available for this URL.",
|
|
42316
|
+
...reason ? [`Google's reason: ${reason.reason}. Label: ${reason.label}. ${reason.why} (Labels are rules over this evidence, not Google's; ${deps.call("index")} explains them.)`] : [],
|
|
41726
42317
|
`Query totals: clicks ${sum(now.rows, "clicks")} (was ${sum(then.rows, "clicks")}), impressions ${sum(now.rows, "impressions")} (was ${sum(then.rows, "impressions")}). Search Console hides rare queries, so these can be lower than the page total.`,
|
|
41727
42318
|
deps.pageMarkdown ? `Main content as Markdown: ${deps.call("page", {
|
|
41728
42319
|
url: input.url,
|
|
@@ -41747,8 +42338,9 @@ async function runPage(input, deps) {
|
|
|
41747
42338
|
url: input.url,
|
|
41748
42339
|
range,
|
|
41749
42340
|
comparisonRange: before,
|
|
41750
|
-
page: facts
|
|
42341
|
+
page: facts,
|
|
41751
42342
|
index: indexed,
|
|
42343
|
+
reason,
|
|
41752
42344
|
totalQueries: rows.length,
|
|
41753
42345
|
queries: rows.slice(offset, offset + 50),
|
|
41754
42346
|
...full ? { markdown } : {}
|
|
@@ -41803,6 +42395,7 @@ const MIN_LOST = 3;
|
|
|
41803
42395
|
const CHECKED = 20;
|
|
41804
42396
|
const INSPECTED = 10;
|
|
41805
42397
|
const SITE_WIDE_MIN = 5;
|
|
42398
|
+
const PAGE_ROWS = 25e3;
|
|
41806
42399
|
function indexSummary(inspected, home) {
|
|
41807
42400
|
const known = inspected.filter((item) => item?.verdict != null);
|
|
41808
42401
|
const indexed = known.filter((item) => item.verdict === "PASS").length;
|
|
@@ -41835,6 +42428,7 @@ function indexSummary(inspected, home) {
|
|
|
41835
42428
|
};
|
|
41836
42429
|
}
|
|
41837
42430
|
const LOST_PAGES_RULE = `Pages that had at least ${MIN_LOST} clicks in the previous 28 days and kept ${LOST_SHARE * 100}% or less of them in the last 28 days, with a live check of the top ${CHECKED} and Google’s index status for the top ${INSPECTED} and the home page.`;
|
|
42431
|
+
const INDEXING_RULE = `Google’s indexing reasons for 100 known URLs a run in a fixed order (lost-click pages, Search Console pages by clicks then impressions, sitemap URLs, then the rest), each checked live and labelled fix, check, expected or indexed from the evidence, sorted by reason then URL.`;
|
|
41838
42432
|
async function runLostPages(input, deps) {
|
|
41839
42433
|
const range = dateWindow(input, 28);
|
|
41840
42434
|
const before = earlierRange(range);
|
|
@@ -41842,9 +42436,13 @@ async function runLostPages(input, deps) {
|
|
|
41842
42436
|
...window,
|
|
41843
42437
|
dimensions: ["page"],
|
|
41844
42438
|
type: "web",
|
|
41845
|
-
maxRows:
|
|
42439
|
+
maxRows: PAGE_ROWS
|
|
41846
42440
|
});
|
|
41847
|
-
const [now, then] = await Promise.all([
|
|
42441
|
+
const [now, then, sitemap] = await Promise.all([
|
|
42442
|
+
read(range),
|
|
42443
|
+
read(before),
|
|
42444
|
+
sitemapUrls(input.site)
|
|
42445
|
+
]);
|
|
41848
42446
|
const current = new Map(now.rows.map((row) => [row.keys[0], row.clicks]));
|
|
41849
42447
|
const lost = then.rows.filter((row) => row.clicks >= MIN_LOST && !row.keys[0]?.includes("#")).map((row) => ({
|
|
41850
42448
|
url: row.keys[0] ?? "",
|
|
@@ -41852,48 +42450,107 @@ async function runLostPages(input, deps) {
|
|
|
41852
42450
|
clicks: current.get(row.keys[0]) ?? 0
|
|
41853
42451
|
})).filter((row) => row.clicks <= row.previous_clicks * LOST_SHARE).sort((a, b) => b.previous_clicks - b.clicks - (a.previous_clicks - a.clicks) || a.url.localeCompare(b.url));
|
|
41854
42452
|
const top = lost.slice(0, CHECKED);
|
|
41855
|
-
const
|
|
41856
|
-
const
|
|
41857
|
-
|
|
41858
|
-
|
|
41859
|
-
|
|
41860
|
-
|
|
41861
|
-
|
|
41862
|
-
|
|
42453
|
+
const first = top[0]?.url ?? sitemap.urls[0] ?? now.rows[0]?.keys[0];
|
|
42454
|
+
const homeUrl = first ? `${new URL(first).origin}/` : null;
|
|
42455
|
+
const known = knownUrls({
|
|
42456
|
+
lost: lost.map((row) => row.url),
|
|
42457
|
+
home: homeUrl,
|
|
42458
|
+
sitemap,
|
|
42459
|
+
pages: now.rows.map((row) => ({
|
|
42460
|
+
url: row.keys[0] ?? "",
|
|
42461
|
+
clicks: row.clicks,
|
|
42462
|
+
impressions: row.impressions
|
|
42463
|
+
})),
|
|
42464
|
+
pagesCapped: now.rows.length >= PAGE_ROWS
|
|
42465
|
+
});
|
|
42466
|
+
const [facts, run] = await Promise.all([deps.checkPages(top.map((row) => row.url)), inspectKnown(input.site, known, deps, input.maxUrls)]);
|
|
42467
|
+
const status = (url) => url ? run.statuses.get(url) ?? null : null;
|
|
42468
|
+
const summary = indexSummary(top.slice(0, INSPECTED).filter((row) => row.url !== homeUrl).map((row) => status(row.url)), top.length && homeUrl ? {
|
|
41863
42469
|
url: homeUrl,
|
|
41864
|
-
status:
|
|
42470
|
+
status: status(homeUrl)
|
|
41865
42471
|
} : null);
|
|
41866
|
-
const rows = top.map((row, index) =>
|
|
41867
|
-
|
|
41868
|
-
|
|
41869
|
-
|
|
41870
|
-
|
|
41871
|
-
|
|
41872
|
-
|
|
41873
|
-
|
|
41874
|
-
|
|
41875
|
-
|
|
41876
|
-
|
|
41877
|
-
|
|
41878
|
-
|
|
41879
|
-
|
|
41880
|
-
|
|
41881
|
-
|
|
41882
|
-
|
|
41883
|
-
|
|
41884
|
-
|
|
42472
|
+
const rows = top.map((row, index) => {
|
|
42473
|
+
const indexed = index < INSPECTED ? status(row.url) : null;
|
|
42474
|
+
return {
|
|
42475
|
+
...row,
|
|
42476
|
+
status: facts[index]?.status ?? null,
|
|
42477
|
+
redirects_to: facts[index]?.redirects_to ?? null,
|
|
42478
|
+
robots: facts[index]?.robots ?? null,
|
|
42479
|
+
canonical: facts[index]?.canonical ?? null,
|
|
42480
|
+
verdict: indexed?.verdict ?? null,
|
|
42481
|
+
coverage: indexed?.coverage ?? null,
|
|
42482
|
+
google_canonical: indexed?.google_canonical ?? null
|
|
42483
|
+
};
|
|
42484
|
+
});
|
|
42485
|
+
const detail = input.detail ?? "summary";
|
|
42486
|
+
const offset = input.offset ?? 0;
|
|
42487
|
+
const origin = originOf(input.site);
|
|
42488
|
+
const firstFix = run.rows.find((row) => row.label === "fix");
|
|
42489
|
+
const reasons = indexingText(input.site, run, {
|
|
42490
|
+
summary: detail === "summary",
|
|
42491
|
+
offset,
|
|
42492
|
+
size: detail === "summary" ? 25 : 50,
|
|
42493
|
+
more: (next) => deps.call("index", {
|
|
42494
|
+
site: input.site,
|
|
42495
|
+
max_urls: input.maxUrls,
|
|
42496
|
+
detail: "rows",
|
|
42497
|
+
offset: next
|
|
42498
|
+
})
|
|
42499
|
+
}, origin, deps.inspectionQuota ? (count) => deps.call("index", {
|
|
42500
|
+
site: input.site,
|
|
42501
|
+
max_urls: count
|
|
42502
|
+
}) : null, deps.now?.() ?? /* @__PURE__ */ new Date());
|
|
42503
|
+
const lostText = [
|
|
42504
|
+
`Pages that lost their clicks on ${input.site}: ${range.startDate} to ${range.endDate} against ${before.startDate} to ${before.endDate}.`,
|
|
42505
|
+
`${lost.length} pages kept ${LOST_SHARE * 100}% or less of at least ${MIN_LOST} clicks${lost.length > CHECKED ? `; the top ${CHECKED} by clicks lost are shown` : ""}.`,
|
|
42506
|
+
rows.length ? table(rows, Object.keys(rows[0] ?? {}), origin) : "No pages lost their clicks.",
|
|
42507
|
+
summary.text,
|
|
42508
|
+
"Live checks are fetched now with no redirects followed. Index status is from URL Inspection.",
|
|
42509
|
+
temporaryRedirectNote(rows)
|
|
42510
|
+
];
|
|
42511
|
+
const indexing = {
|
|
42512
|
+
rule: INDEXING_RULE,
|
|
42513
|
+
known: {
|
|
42514
|
+
total: known.urls.length,
|
|
42515
|
+
sitemap: sitemap.urls.length,
|
|
42516
|
+
sitemapFiles: sitemap.files,
|
|
42517
|
+
sitemapPartial: sitemap.partial,
|
|
42518
|
+
searchConsole: known.searchConsole,
|
|
42519
|
+
searchConsoleCapped: known.searchConsoleCapped,
|
|
42520
|
+
lostClicks: known.lostClicks
|
|
42521
|
+
},
|
|
42522
|
+
limit: run.limit,
|
|
42523
|
+
inspected: run.inspected,
|
|
42524
|
+
fromSaved: run.fromSaved,
|
|
42525
|
+
oldestSaved: run.oldestSaved,
|
|
42526
|
+
failed: run.failed,
|
|
42527
|
+
stoppedBy: run.stoppedBy,
|
|
42528
|
+
quota: run.quota,
|
|
42529
|
+
byReason: byReason(run.rows),
|
|
42530
|
+
rows: run.rows
|
|
42531
|
+
};
|
|
42532
|
+
return {
|
|
42533
|
+
text: [...detail === "full" ? [`Google's indexing reasons for ${input.site}, every inspected URL as JSON, sorted by reason then URL:`, JSON.stringify(indexing)] : [
|
|
42534
|
+
...detail === "summary" ? lostText : [],
|
|
42535
|
+
...reasons,
|
|
42536
|
+
`To see what changed, keep this run's full result (${deps.call("index", {
|
|
42537
|
+
site: input.site,
|
|
42538
|
+
max_urls: input.maxUrls,
|
|
42539
|
+
detail: "full"
|
|
42540
|
+
})}) and compare it with the next run by url and reason. Inspect the same number each week: the order is fixed, so the same first URLs come back. Where results are saved (the CLI and the local MCP), a URL inspected in the last 7 days is reused; any other URL is inspected again and uses URL Inspection quota.`,
|
|
42541
|
+
`Next: ${firstFix ? deps.call("page", { url: firstFix.url }) : deps.call("page")} for one URL, including a URL from a Search Console email that is not in this set; ${deps.call("report", {
|
|
41885
42542
|
site: input.site,
|
|
41886
42543
|
section: "content-decay"
|
|
41887
|
-
})} for pages that lost part of their clicks
|
|
41888
|
-
|
|
41889
|
-
].filter(Boolean).join("\n\n"),
|
|
42544
|
+
})} for pages that lost part of their clicks.`
|
|
42545
|
+
], FACTS_ONLY].filter(Boolean).join("\n\n"),
|
|
41890
42546
|
data: {
|
|
41891
42547
|
range,
|
|
41892
42548
|
comparisonRange: before,
|
|
41893
42549
|
rule: LOST_PAGES_RULE,
|
|
41894
42550
|
lostPages: lost.length,
|
|
41895
42551
|
index: summary.data,
|
|
41896
|
-
pages: rows
|
|
42552
|
+
pages: rows,
|
|
42553
|
+
indexing
|
|
41897
42554
|
}
|
|
41898
42555
|
};
|
|
41899
42556
|
}
|
|
@@ -42036,16 +42693,26 @@ const SHARED_TOOLS = [
|
|
|
42036
42693
|
tool({
|
|
42037
42694
|
name: "seo_index",
|
|
42038
42695
|
command: "index",
|
|
42039
|
-
title: "
|
|
42040
|
-
description: `${
|
|
42696
|
+
title: "Google indexing reasons and pages that lost their clicks",
|
|
42697
|
+
description: `${INDEXING_RULE} Use it for a first baseline and then weekly: keep the full result and compare by URL and reason to find new reasons. Also: ${LOST_PAGES_RULE}`,
|
|
42041
42698
|
input: z.object({
|
|
42042
42699
|
site,
|
|
42043
|
-
...dates
|
|
42700
|
+
...dates,
|
|
42701
|
+
...detail
|
|
42702
|
+
}),
|
|
42703
|
+
localInput: z.object({
|
|
42704
|
+
site,
|
|
42705
|
+
...dates,
|
|
42706
|
+
max_urls: z.number().int().min(1).max(MAX_INSPECT_LIMIT).optional().describe(`URLs to inspect with Google, 100 by default, up to the URL Inspection quota left today.`),
|
|
42707
|
+
...detail
|
|
42044
42708
|
}),
|
|
42045
42709
|
run: async (surface, input) => runLostPages({
|
|
42046
42710
|
site: await surface.resolveSite(input.site),
|
|
42047
42711
|
startDate: input.start_date,
|
|
42048
|
-
endDate: input.end_date
|
|
42712
|
+
endDate: input.end_date,
|
|
42713
|
+
detail: input.detail,
|
|
42714
|
+
offset: input.offset,
|
|
42715
|
+
maxUrls: input.max_urls
|
|
42049
42716
|
}, surface.deps)
|
|
42050
42717
|
}),
|
|
42051
42718
|
tool({
|
|
@@ -42142,15 +42809,15 @@ const SHARED_TOOLS = [
|
|
|
42142
42809
|
}
|
|
42143
42810
|
})
|
|
42144
42811
|
];
|
|
42145
|
-
async function runSharedTool(tool, surface, input) {
|
|
42146
|
-
const parsed = tool.input.safeParse(input ?? {});
|
|
42812
|
+
async function runSharedTool(tool, surface, input, local = false) {
|
|
42813
|
+
const parsed = (local && tool.localInput || tool.input).safeParse(input ?? {});
|
|
42147
42814
|
if (!parsed.success) throw new CommandError("INVALID_INPUT", parsed.error.issues.map((issue) => `${issue.path.join(".") || "input"}: ${issue.message}`).join("; "));
|
|
42148
42815
|
return tool.run(surface, parsed.data);
|
|
42149
42816
|
}
|
|
42150
42817
|
const MCP_INSTRUCTIONS = `SEO Skill reads the person's own Search Console, Google Analytics, Bing Webmaster Tools, Clicky and optional keyword accounts, and returns facts with their dates and limits. It gives no advice.
|
|
42151
42818
|
1. seo_sites: the properties the connected accounts can read.
|
|
42152
42819
|
2. seo_report: the main report. Run it without a section first; it shows the overview and every section's count and top rows. Then ask for the one section you need.
|
|
42153
|
-
3. seo_page for one URL, seo_compare for before and after a date, seo_index for pages that lost their clicks, seo_research for keyword, competitor and backlink data, and whether AI answers cite the site, from the DataForSEO or Ahrefs key the person saved, seo_data for raw rows, including Google Analytics, Bing and Clicky.`;
|
|
42820
|
+
3. seo_page for one URL, seo_compare for before and after a date, seo_index for Google's indexing reasons and pages that lost their clicks, seo_research for keyword, competitor and backlink data, and whether AI answers cite the site, from the DataForSEO or Ahrefs key the person saved, seo_data for raw rows, including Google Analytics, Bing and Clicky.`;
|
|
42154
42821
|
const mcpCall = (command, args) => {
|
|
42155
42822
|
const name = `seo_${command.replace("-", "_")}`;
|
|
42156
42823
|
const shown = Object.entries(args ?? {}).filter(([, value]) => value !== void 0);
|
|
@@ -42164,6 +42831,36 @@ const SKILL_RESOURCE = {
|
|
|
42164
42831
|
mimeType: "text/markdown"
|
|
42165
42832
|
};
|
|
42166
42833
|
//#endregion
|
|
42834
|
+
//#region packages/core/dist/gsc/client/inspection-cache.js
|
|
42835
|
+
const INSPECTION_FRESH_MS = 7 * 864e5;
|
|
42836
|
+
function readSavedInspections(site, urls, now = Date.now()) {
|
|
42837
|
+
const found = /* @__PURE__ */ new Map();
|
|
42838
|
+
try {
|
|
42839
|
+
const statement = cacheDb().prepare(`SELECT status_json, fetched_at FROM url_inspection_cache
|
|
42840
|
+
WHERE site_url = ? AND url = ? AND fetched_at > ?`);
|
|
42841
|
+
for (const url of urls) {
|
|
42842
|
+
const row = statement.get(site, url, now - INSPECTION_FRESH_MS);
|
|
42843
|
+
if (!row) continue;
|
|
42844
|
+
found.set(url, {
|
|
42845
|
+
status: JSON.parse(row.status_json),
|
|
42846
|
+
inspectedAt: new Date(row.fetched_at).toISOString()
|
|
42847
|
+
});
|
|
42848
|
+
}
|
|
42849
|
+
} catch {
|
|
42850
|
+
return /* @__PURE__ */ new Map();
|
|
42851
|
+
}
|
|
42852
|
+
return found;
|
|
42853
|
+
}
|
|
42854
|
+
function saveInspection(site, url, status, now = Date.now()) {
|
|
42855
|
+
const json = JSON.stringify(status);
|
|
42856
|
+
writeToCache((database) => {
|
|
42857
|
+
database.prepare(`INSERT OR REPLACE INTO url_inspection_cache
|
|
42858
|
+
(site_url, url, status_json, fetched_at, expires_at)
|
|
42859
|
+
VALUES (?, ?, ?, ?, ?)`).run(site, url, json, now, now + INSPECTION_FRESH_MS);
|
|
42860
|
+
noteCacheWrite(Buffer.byteLength(json) + url.length, database);
|
|
42861
|
+
});
|
|
42862
|
+
}
|
|
42863
|
+
//#endregion
|
|
42167
42864
|
//#region packages/core/dist/providers/api-key-credentials.js
|
|
42168
42865
|
function createApiKeyCredentialStore(input) {
|
|
42169
42866
|
const normalize = (value) => {
|
|
@@ -42343,16 +43040,19 @@ function nodeCommandDeps(options) {
|
|
|
42343
43040
|
accountEmail: options.account
|
|
42344
43041
|
}),
|
|
42345
43042
|
checkPages,
|
|
42346
|
-
|
|
42347
|
-
|
|
42348
|
-
|
|
42349
|
-
|
|
42350
|
-
|
|
42351
|
-
|
|
42352
|
-
|
|
42353
|
-
|
|
42354
|
-
|
|
42355
|
-
|
|
43043
|
+
liveCheck,
|
|
43044
|
+
inspect: async (site, url) => indexStatusFrom(await inspectUrl({
|
|
43045
|
+
siteUrl: site,
|
|
43046
|
+
inspectionUrl: url,
|
|
43047
|
+
accountEmail: options.account
|
|
43048
|
+
})),
|
|
43049
|
+
inspectionQuota: (site) => urlInspectionQuota({
|
|
43050
|
+
siteUrl: site,
|
|
43051
|
+
accountEmail: options.account
|
|
43052
|
+
}),
|
|
43053
|
+
savedInspections: {
|
|
43054
|
+
read: (site, urls) => options.refresh ? /* @__PURE__ */ new Map() : readSavedInspections(site, urls),
|
|
43055
|
+
save: saveInspection
|
|
42356
43056
|
},
|
|
42357
43057
|
pageMarkdown: async (url) => extractMainMarkdown(await fetchPage(url, { refresh: options.refresh })),
|
|
42358
43058
|
call: options.call,
|
|
@@ -45479,10 +46179,10 @@ function registerSharedTools(server, options = {}) {
|
|
|
45479
46179
|
for (const shared of SHARED_TOOLS) server.registerTool(shared.name, {
|
|
45480
46180
|
title: shared.title,
|
|
45481
46181
|
description: shared.description,
|
|
45482
|
-
inputSchema: shared.input.shape,
|
|
46182
|
+
inputSchema: (shared.localInput ?? shared.input).shape,
|
|
45483
46183
|
annotations: readOnly
|
|
45484
46184
|
}, async (input) => counted(shared.command, input, options.telemetry, async () => {
|
|
45485
|
-
return text((await runSharedTool(shared, surface, input)).text);
|
|
46185
|
+
return text((await runSharedTool(shared, surface, input, true)).text);
|
|
45486
46186
|
}));
|
|
45487
46187
|
server.registerTool("seo_crawl", {
|
|
45488
46188
|
title: "Crawl a site",
|