seo 0.15.2 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +84 -84
- package/dist/cli.js.map +1 -1
- package/dist/{dist-DOJoZZ9W.js → dist-BZ5-V3qL.js} +1 -1
- package/dist/{dist-Dpl5hSMM.js → dist-CqzyPI3J.js} +140 -136
- package/dist/dist-CqzyPI3J.js.map +1 -0
- package/dist/{esm-Dt4JJZ0P.js → esm-Cvmg0qhZ.js} +2 -2
- package/dist/{esm-Dt4JJZ0P.js.map → esm-Cvmg0qhZ.js.map} +1 -1
- package/dist/index.d.ts +108 -46
- package/dist/index.js +1664 -807
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +1223 -372
- package/dist/mcp.js.map +1 -1
- package/package.json +1 -1
- package/skills/seo/SKILL.md +10 -1
- package/dist/dist-Dpl5hSMM.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { n as version, t as load } from "./esm-
|
|
2
|
+
import { n as version, t as load } from "./esm-Cvmg0qhZ.js";
|
|
3
3
|
import { SEO_PROVIDER_API_VERSION } from "./provider-sdk.js";
|
|
4
4
|
import crypto, { createHash, randomUUID, sign } from "node:crypto";
|
|
5
5
|
import { z } from "zod";
|
|
@@ -343,6 +343,24 @@ const RULE_DEFINITIONS = [
|
|
|
343
343
|
suggestedCommands: ["seo crawl <url> --max-pages 1 --json"]
|
|
344
344
|
}
|
|
345
345
|
},
|
|
346
|
+
{
|
|
347
|
+
id: "crawl_single_page_reach",
|
|
348
|
+
title: "Crawl could reach only the start page",
|
|
349
|
+
category: "links",
|
|
350
|
+
defaultSeverity: "high",
|
|
351
|
+
whyItMatters: "The start page had no crawlable internal links and no sitemap returned URLs, so the crawl checked one page. Every other finding in this report covers that page only. The page probably builds its links with JavaScript.",
|
|
352
|
+
howToFix: "Rerun with `--rendering on` so the crawler reads the links a browser sees, or pass the pages to check with `--mode list`. For crawlers that do not run JavaScript, put plain `<a href>` links in the initial HTML and publish a sitemap.",
|
|
353
|
+
impactIfIgnored: "Crawlers and audits that do not run JavaScript can find only this page, so the rest of the site stays unchecked and harder to discover.",
|
|
354
|
+
howToVerify: "Rerun the crawl and confirm it retains more than one page and the start page reports internal links.",
|
|
355
|
+
agentHints: {
|
|
356
|
+
evidenceFields: [
|
|
357
|
+
"page.outgoingInternalCount",
|
|
358
|
+
"report.sitemapDiscovery.urlsReturned",
|
|
359
|
+
"page.fetchDiagnostics.rendering"
|
|
360
|
+
],
|
|
361
|
+
suggestedCommands: ["seo crawl <url> --rendering on --json", "seo crawl --mode list --urls <url>,<url> --json"]
|
|
362
|
+
}
|
|
363
|
+
},
|
|
346
364
|
{
|
|
347
365
|
id: "soft_404",
|
|
348
366
|
title: "Unknown URLs return a successful response",
|
|
@@ -351,7 +369,7 @@ const RULE_DEFINITIONS = [
|
|
|
351
369
|
whyItMatters: "Known-nonexistent URLs returned a successful response or redirected to another successful page. Search engines and users cannot reliably distinguish missing content from a real page.",
|
|
352
370
|
howToFix: "Return 404 or 410 for missing URLs. If a replacement exists, redirect only the specific retired URL to its closest relevant replacement rather than sending every unknown path to a generic page.",
|
|
353
371
|
impactIfIgnored: "Missing URLs can remain in crawl queues, waste crawl attention, and create confusing search or monitoring evidence.",
|
|
354
|
-
howToVerify: "Request several
|
|
372
|
+
howToVerify: "Request several made-up paths, including one at the site root and one nested path, and confirm each returns 404 or 410 without redirecting to a generic successful page.",
|
|
355
373
|
agentHints: {
|
|
356
374
|
evidenceFields: ["report.siteChecks.soft404"],
|
|
357
375
|
suggestedCommands: ["seo crawl <url> --json"]
|
|
@@ -1803,7 +1821,7 @@ function day$3(value) {
|
|
|
1803
1821
|
const date = legacy ? new Date(Number(legacy[1])) : new Date(value);
|
|
1804
1822
|
return Number.isNaN(date.getTime()) ? null : date.toISOString().slice(0, 10);
|
|
1805
1823
|
}
|
|
1806
|
-
const count$
|
|
1824
|
+
const count$4 = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
|
|
1807
1825
|
function list$2(data, label) {
|
|
1808
1826
|
if (!Array.isArray(data)) throw new SeoError("OPTIONAL_PROVIDER_UNAVAILABLE", `Bing Webmaster returned an invalid ${label}.`);
|
|
1809
1827
|
return data.filter((item) => Boolean(item) && typeof item === "object");
|
|
@@ -1819,8 +1837,8 @@ async function bingTraffic(call, site) {
|
|
|
1819
1837
|
let invalidRows = 0;
|
|
1820
1838
|
for (const row of list$2(await call("GetRankAndTrafficStats", { siteUrl: site }), "traffic list")) {
|
|
1821
1839
|
const date = day$3(row.Date);
|
|
1822
|
-
const clicks = count$
|
|
1823
|
-
const impressions = count$
|
|
1840
|
+
const clicks = count$4(row.Clicks);
|
|
1841
|
+
const impressions = count$4(row.Impressions);
|
|
1824
1842
|
if (!date || clicks === null || impressions === null) {
|
|
1825
1843
|
invalidRows += 1;
|
|
1826
1844
|
continue;
|
|
@@ -1843,9 +1861,9 @@ async function keyed(call, method, params) {
|
|
|
1843
1861
|
for (const row of list$2(await call(method, params), "row list")) {
|
|
1844
1862
|
const date = day$3(row.Date);
|
|
1845
1863
|
const key = typeof row.Query === "string" ? row.Query.trim() : "";
|
|
1846
|
-
const clicks = count$
|
|
1847
|
-
const impressions = count$
|
|
1848
|
-
const position = count$
|
|
1864
|
+
const clicks = count$4(row.Clicks);
|
|
1865
|
+
const impressions = count$4(row.Impressions);
|
|
1866
|
+
const position = count$4(row.AvgImpressionPosition);
|
|
1849
1867
|
if (!date || !key || clicks === null || impressions === null) {
|
|
1850
1868
|
invalidRows += 1;
|
|
1851
1869
|
continue;
|
|
@@ -2377,6 +2395,14 @@ function sharedReader(read, redirects = /* @__PURE__ */ new Map()) {
|
|
|
2377
2395
|
};
|
|
2378
2396
|
}
|
|
2379
2397
|
const MAX_STATUS_BYTES = 10485760;
|
|
2398
|
+
async function rankingUpdates() {
|
|
2399
|
+
const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? readJson(response, "Google Search Status", { maxBytes: MAX_STATUS_BYTES }) : null).catch(() => null);
|
|
2400
|
+
if (!Array.isArray(incidents)) return "Google Search Status could not be read.";
|
|
2401
|
+
return searchUpdatesFromIncidents(withRankingHistory(incidents), {
|
|
2402
|
+
product: "Ranking",
|
|
2403
|
+
limit: Number.POSITIVE_INFINITY
|
|
2404
|
+
});
|
|
2405
|
+
}
|
|
2380
2406
|
async function googleUpdates(window) {
|
|
2381
2407
|
if (!window) return [];
|
|
2382
2408
|
const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? readJson(response, "Google Search Status", { maxBytes: MAX_STATUS_BYTES }) : null).catch(() => null);
|
|
@@ -7702,17 +7728,15 @@ function recordHostFetch(input) {
|
|
|
7702
7728
|
const cfg = input.rate.backpressure;
|
|
7703
7729
|
health.durations.push(input.durationMs);
|
|
7704
7730
|
if (health.durations.length > 20) health.durations.shift();
|
|
7705
|
-
const
|
|
7706
|
-
|
|
7707
|
-
|
|
7708
|
-
429
|
|
7709
|
-
].includes(input.status);
|
|
7710
|
-
const errored = input.status >= 500;
|
|
7731
|
+
const rateLimited = input.status === 429 || input.status === 503 && input.retryAfterMs !== void 0;
|
|
7732
|
+
const blocked = [401, 403].includes(input.status);
|
|
7733
|
+
const errored = input.status >= 500 && !rateLimited;
|
|
7711
7734
|
const slow = input.durationMs >= cfg.slowMs;
|
|
7712
7735
|
health.consecutiveBlocked = blocked ? health.consecutiveBlocked + 1 : 0;
|
|
7713
7736
|
health.consecutiveErrors = errored ? health.consecutiveErrors + 1 : 0;
|
|
7714
7737
|
health.consecutiveSlow = slow ? health.consecutiveSlow + 1 : 0;
|
|
7715
|
-
if (!blocked && !errored && !slow) health.reason = void 0;
|
|
7738
|
+
if (!blocked && !errored && !slow && !rateLimited) health.reason = void 0;
|
|
7739
|
+
if (rateLimited) health.reason = `origin rate limited the crawler (HTTP ${input.status})`;
|
|
7716
7740
|
const retryDelay = input.retryAfterMs === void 0 ? 0 : Math.min(input.retryAfterMs, cfg.retryAfterCapMs);
|
|
7717
7741
|
if (retryDelay > 0) {
|
|
7718
7742
|
health.reason = `origin sent Retry-After (${Math.round(retryDelay / 1e3)}s)`;
|
|
@@ -8113,6 +8137,56 @@ function systemBrowserCandidates(input) {
|
|
|
8113
8137
|
product: path.includes("google-chrome") ? "Google Chrome" : "Chromium"
|
|
8114
8138
|
}));
|
|
8115
8139
|
}
|
|
8140
|
+
function listDirectorySafely(path) {
|
|
8141
|
+
try {
|
|
8142
|
+
return readdirSync(path);
|
|
8143
|
+
} catch {
|
|
8144
|
+
return [];
|
|
8145
|
+
}
|
|
8146
|
+
}
|
|
8147
|
+
function versionParts(name) {
|
|
8148
|
+
return (name.match(/\d+/g) ?? []).map(Number);
|
|
8149
|
+
}
|
|
8150
|
+
function newestFirst(left, right) {
|
|
8151
|
+
const a = versionParts(left);
|
|
8152
|
+
const b = versionParts(right);
|
|
8153
|
+
for (let index = 0; index < Math.max(a.length, b.length); index += 1) {
|
|
8154
|
+
const difference = (b[index] ?? -1) - (a[index] ?? -1);
|
|
8155
|
+
if (difference) return difference;
|
|
8156
|
+
}
|
|
8157
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
8158
|
+
}
|
|
8159
|
+
const MAX_CACHED_BROWSER_VERSIONS = 20;
|
|
8160
|
+
function versionDirectories(root, prefix, listDirectory) {
|
|
8161
|
+
return listDirectory(root).filter((name) => name.startsWith(prefix)).sort(newestFirst).slice(0, MAX_CACHED_BROWSER_VERSIONS).map((name) => join(root, name));
|
|
8162
|
+
}
|
|
8163
|
+
const CHROME_FOR_TESTING_APP = join("Google Chrome for Testing.app", "Contents", "MacOS", "Google Chrome for Testing");
|
|
8164
|
+
/**
|
|
8165
|
+
* Browsers that other local tools already downloaded: agent-browser keeps
|
|
8166
|
+
* Chrome for Testing builds and Playwright keeps one Chromium per revision.
|
|
8167
|
+
*/
|
|
8168
|
+
function cachedBrowserCandidates(input) {
|
|
8169
|
+
const agentBrowserExecutables = input.platform === "darwin" ? [CHROME_FOR_TESTING_APP] : input.platform === "win32" ? ["chrome.exe"] : ["chrome"];
|
|
8170
|
+
const agentBrowser = versionDirectories(join(input.homeDirectory, ".agent-browser", "browsers"), "chrome-", input.listDirectory).flatMap((directory) => agentBrowserExecutables.map((executable) => ({
|
|
8171
|
+
path: join(directory, executable),
|
|
8172
|
+
source: "agent-browser-cache",
|
|
8173
|
+
product: "agent-browser Chrome for Testing"
|
|
8174
|
+
})));
|
|
8175
|
+
const configuredRoot = input.environment.PLAYWRIGHT_BROWSERS_PATH?.trim();
|
|
8176
|
+
const playwrightRoot = configuredRoot && configuredRoot !== "0" ? configuredRoot : input.platform === "darwin" ? join(input.homeDirectory, "Library", "Caches", "ms-playwright") : input.platform === "win32" ? join(input.environment.LOCALAPPDATA ?? join(input.homeDirectory, "AppData", "Local"), "ms-playwright") : join(input.environment.XDG_CACHE_HOME ?? join(input.homeDirectory, ".cache"), "ms-playwright");
|
|
8177
|
+
const playwrightExecutables = input.platform === "darwin" ? [
|
|
8178
|
+
join("chrome-mac-arm64", CHROME_FOR_TESTING_APP),
|
|
8179
|
+
join("chrome-mac-x64", CHROME_FOR_TESTING_APP),
|
|
8180
|
+
join("chrome-mac", CHROME_FOR_TESTING_APP),
|
|
8181
|
+
join("chrome-mac", "Chromium.app", "Contents", "MacOS", "Chromium")
|
|
8182
|
+
] : input.platform === "win32" ? [join("chrome-win64", "chrome.exe"), join("chrome-win", "chrome.exe")] : [join("chrome-linux64", "chrome"), join("chrome-linux", "chrome")];
|
|
8183
|
+
const playwright = versionDirectories(playwrightRoot, "chromium-", input.listDirectory).flatMap((directory) => playwrightExecutables.map((executable) => ({
|
|
8184
|
+
path: join(directory, executable),
|
|
8185
|
+
source: "playwright-cache",
|
|
8186
|
+
product: "Playwright Chromium"
|
|
8187
|
+
})));
|
|
8188
|
+
return [...agentBrowser, ...playwright];
|
|
8189
|
+
}
|
|
8116
8190
|
/**
|
|
8117
8191
|
* Resolve a browser without downloading one during normal package installation.
|
|
8118
8192
|
* The explicit path wins so CI and managed workstations can select their own
|
|
@@ -8144,11 +8218,23 @@ function resolveBrowserExecutable(playwrightExecutablePath, dependencies = {}) {
|
|
|
8144
8218
|
product: "Playwright Chromium"
|
|
8145
8219
|
}
|
|
8146
8220
|
};
|
|
8147
|
-
const
|
|
8148
|
-
|
|
8149
|
-
|
|
8221
|
+
const platform = dependencies.platform ?? process.platform;
|
|
8222
|
+
const homeDirectory = dependencies.homeDirectory ?? homedir();
|
|
8223
|
+
const systemBrowser = systemBrowserCandidates({
|
|
8224
|
+
platform,
|
|
8225
|
+
homeDirectory,
|
|
8150
8226
|
environment
|
|
8151
8227
|
}).find((candidate) => fileExists(candidate.path));
|
|
8228
|
+
if (systemBrowser) return {
|
|
8229
|
+
status: "available",
|
|
8230
|
+
browser: systemBrowser
|
|
8231
|
+
};
|
|
8232
|
+
const browser = cachedBrowserCandidates({
|
|
8233
|
+
platform,
|
|
8234
|
+
homeDirectory,
|
|
8235
|
+
environment,
|
|
8236
|
+
listDirectory: dependencies.listDirectory ?? listDirectorySafely
|
|
8237
|
+
}).find((candidate) => fileExists(candidate.path));
|
|
8152
8238
|
return browser ? {
|
|
8153
8239
|
status: "available",
|
|
8154
8240
|
browser
|
|
@@ -11200,6 +11286,17 @@ function isAuditableHtmlPage(page) {
|
|
|
11200
11286
|
const contentAuditAllowed = page.contentAuditAllowed ?? (!page.blocked && page.robotsTxt?.allowed !== false);
|
|
11201
11287
|
return isSuccessfulResponse(page) && isHtmlPage(page) && !isRedirectedPage(page) && contentAuditAllowed;
|
|
11202
11288
|
}
|
|
11289
|
+
/**
|
|
11290
|
+
* Cloudflare serves its own endpoints under `/cdn-cgi/`, such as the email
|
|
11291
|
+
* obfuscation link `/cdn-cgi/l/email-protection`. They are not site pages.
|
|
11292
|
+
*/
|
|
11293
|
+
function isCdnUtilityUrl(value) {
|
|
11294
|
+
try {
|
|
11295
|
+
return new URL(value).pathname.startsWith("/cdn-cgi/");
|
|
11296
|
+
} catch {
|
|
11297
|
+
return false;
|
|
11298
|
+
}
|
|
11299
|
+
}
|
|
11203
11300
|
// packages/core/src/analyze/crawler/audit.ts
|
|
11204
11301
|
function sameUrl(a, b) {
|
|
11205
11302
|
if (!a || !b) return false;
|
|
@@ -11312,38 +11409,61 @@ function isValuablePage(page) {
|
|
|
11312
11409
|
function normalizedMetadata(value) {
|
|
11313
11410
|
return value?.trim().replace(/\s+/g, " ").toLowerCase() || void 0;
|
|
11314
11411
|
}
|
|
11315
|
-
|
|
11316
|
-
|
|
11412
|
+
/**
|
|
11413
|
+
* A page whose canonical names another crawled page in its group is already
|
|
11414
|
+
* handled. A group counts only when two or more of its pages are
|
|
11415
|
+
* self-canonical or declare no canonical, and only those pages are reported.
|
|
11416
|
+
*/
|
|
11417
|
+
function duplicateGroups(pages, read) {
|
|
11418
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
11317
11419
|
for (const page of pages) {
|
|
11318
11420
|
if (!isAuditableHtmlPage(page)) continue;
|
|
11319
|
-
const
|
|
11320
|
-
|
|
11321
|
-
|
|
11322
|
-
|
|
11323
|
-
|
|
11324
|
-
sampleUrls: [],
|
|
11325
|
-
value: raw
|
|
11421
|
+
const entry = read(page);
|
|
11422
|
+
if (!entry) continue;
|
|
11423
|
+
const group = grouped.get(entry.key) ?? {
|
|
11424
|
+
value: entry.value,
|
|
11425
|
+
pages: []
|
|
11326
11426
|
};
|
|
11327
|
-
|
|
11328
|
-
|
|
11329
|
-
|
|
11427
|
+
group.pages.push(page);
|
|
11428
|
+
grouped.set(entry.key, group);
|
|
11429
|
+
}
|
|
11430
|
+
const duplicates = /* @__PURE__ */ new Map();
|
|
11431
|
+
for (const [key, group] of grouped) {
|
|
11432
|
+
if (group.pages.length < 2) continue;
|
|
11433
|
+
const groupUrls = /* @__PURE__ */ new Set();
|
|
11434
|
+
for (const page of group.pages) for (const value of [page.url, page.finalUrl]) {
|
|
11435
|
+
const url = urlKey(value);
|
|
11436
|
+
if (url) groupUrls.add(url);
|
|
11437
|
+
}
|
|
11438
|
+
const counted = group.pages.filter((page) => {
|
|
11439
|
+
const canonical = urlKey(page.canonical);
|
|
11440
|
+
return !canonical || canonical === urlKey(page.url) || canonical === urlKey(page.finalUrl) || !groupUrls.has(canonical);
|
|
11441
|
+
});
|
|
11442
|
+
if (counted.length < 2) continue;
|
|
11443
|
+
duplicates.set(key, {
|
|
11444
|
+
count: counted.length,
|
|
11445
|
+
sampleUrls: counted.slice(0, 10).map((page) => page.url),
|
|
11446
|
+
value: group.value,
|
|
11447
|
+
urls: new Set(counted.map((page) => page.url))
|
|
11448
|
+
});
|
|
11330
11449
|
}
|
|
11331
|
-
return
|
|
11450
|
+
return duplicates;
|
|
11451
|
+
}
|
|
11452
|
+
function metadataDuplicates(pages, read) {
|
|
11453
|
+
return duplicateGroups(pages, (page) => {
|
|
11454
|
+
const value = read(page)?.trim();
|
|
11455
|
+
const key = normalizedMetadata(value);
|
|
11456
|
+
return key && value ? {
|
|
11457
|
+
key,
|
|
11458
|
+
value
|
|
11459
|
+
} : void 0;
|
|
11460
|
+
});
|
|
11332
11461
|
}
|
|
11333
11462
|
function contentDuplicates(pages) {
|
|
11334
|
-
|
|
11335
|
-
|
|
11336
|
-
|
|
11337
|
-
|
|
11338
|
-
const existing = counts.get(page.mainContentHash) ?? {
|
|
11339
|
-
count: 0,
|
|
11340
|
-
sampleUrls: []
|
|
11341
|
-
};
|
|
11342
|
-
existing.count += 1;
|
|
11343
|
-
if (existing.sampleUrls.length < 10) existing.sampleUrls.push(page.url);
|
|
11344
|
-
counts.set(page.mainContentHash, existing);
|
|
11345
|
-
}
|
|
11346
|
-
return new Map([...counts].filter(([, value]) => value.count > 1));
|
|
11463
|
+
return duplicateGroups(pages, (page) => page.wordCount >= 20 && page.mainContentHash ? {
|
|
11464
|
+
key: page.mainContentHash,
|
|
11465
|
+
value: page.mainContentHash
|
|
11466
|
+
} : void 0);
|
|
11347
11467
|
}
|
|
11348
11468
|
const QUERY_STOP_WORDS = /* @__PURE__ */ new Set([
|
|
11349
11469
|
"a",
|
|
@@ -11446,7 +11566,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
11446
11566
|
if (page.accessBlock) continue;
|
|
11447
11567
|
if (page.status >= 500) {
|
|
11448
11568
|
issues.push(issue$1("server_error", page, String(page.status), { status: page.status }));
|
|
11449
|
-
if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
11569
|
+
if ((page.internalInlinkCount ?? 0) > 0 && !isCdnUtilityUrl(page.url)) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
11450
11570
|
status: page.status,
|
|
11451
11571
|
internalInlinkCount: page.internalInlinkCount
|
|
11452
11572
|
}));
|
|
@@ -11454,7 +11574,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
11454
11574
|
}
|
|
11455
11575
|
if (page.status >= 400) {
|
|
11456
11576
|
issues.push(issue$1("client_error", page, String(page.status), { status: page.status }));
|
|
11457
|
-
if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
11577
|
+
if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0 && !isCdnUtilityUrl(page.url)) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
11458
11578
|
status: page.status,
|
|
11459
11579
|
internalInlinkCount: page.internalInlinkCount
|
|
11460
11580
|
}));
|
|
@@ -11553,7 +11673,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
11553
11673
|
title,
|
|
11554
11674
|
...titleWidth
|
|
11555
11675
|
}));
|
|
11556
|
-
if (title && duplicateTitle) issues.push(issue$1("title_duplicate", page, duplicateTitle.value, {
|
|
11676
|
+
if (title && duplicateTitle?.urls.has(page.url)) issues.push(issue$1("title_duplicate", page, duplicateTitle.value, {
|
|
11557
11677
|
title,
|
|
11558
11678
|
duplicateCount: duplicateTitle.count,
|
|
11559
11679
|
sampleUrls: duplicateTitle.sampleUrls
|
|
@@ -11561,7 +11681,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
11561
11681
|
const metaDescription = page.metaDescription?.trim();
|
|
11562
11682
|
const duplicateDescription = duplicateDescriptions.get(normalizedMetadata(metaDescription) ?? "");
|
|
11563
11683
|
if (!metaDescription) issues.push(issue$1("missing_meta_description", page));
|
|
11564
|
-
if (metaDescription && duplicateDescription) issues.push(issue$1("meta_description_duplicate", page, duplicateDescription.value, {
|
|
11684
|
+
if (metaDescription && duplicateDescription?.urls.has(page.url)) issues.push(issue$1("meta_description_duplicate", page, duplicateDescription.value, {
|
|
11565
11685
|
metaDescription,
|
|
11566
11686
|
duplicateCount: duplicateDescription.count,
|
|
11567
11687
|
sampleUrls: duplicateDescription.sampleUrls
|
|
@@ -11609,7 +11729,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
11609
11729
|
finalUrl: page.finalUrl
|
|
11610
11730
|
}));
|
|
11611
11731
|
const contentDuplicate = page.mainContentHash ? duplicateContent.get(page.mainContentHash) : void 0;
|
|
11612
|
-
if (contentDuplicate) issues.push(issue$1("duplicate_content", page, "Duplicate main content", {
|
|
11732
|
+
if (contentDuplicate?.urls.has(page.url)) issues.push(issue$1("duplicate_content", page, "Duplicate main content", {
|
|
11613
11733
|
mainContentHash: page.mainContentHash,
|
|
11614
11734
|
duplicateCount: contentDuplicate.count,
|
|
11615
11735
|
sampleUrls: contentDuplicate.sampleUrls
|
|
@@ -11677,6 +11797,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
11677
11797
|
// packages/core/src/analyze/crawler/crawl-skip-reasons.ts
|
|
11678
11798
|
const CRAWL_SKIP_REASONS = [
|
|
11679
11799
|
"asset-url",
|
|
11800
|
+
"cdn-utility-url",
|
|
11680
11801
|
"configured-exclusion",
|
|
11681
11802
|
"duplicate-normalization",
|
|
11682
11803
|
"invalid-url",
|
|
@@ -11684,12 +11805,14 @@ const CRAWL_SKIP_REASONS = [
|
|
|
11684
11805
|
"off-origin",
|
|
11685
11806
|
"origin-backpressure",
|
|
11686
11807
|
"queue-safety-limit",
|
|
11808
|
+
"query-variant-limit",
|
|
11687
11809
|
"robots-disallowed",
|
|
11688
11810
|
"robots-uncertain",
|
|
11689
11811
|
"legacy-unclassified"
|
|
11690
11812
|
];
|
|
11691
11813
|
const SKIP_IMPACT = {
|
|
11692
11814
|
"asset-url": "non-impacting",
|
|
11815
|
+
"cdn-utility-url": "non-impacting",
|
|
11693
11816
|
"configured-exclusion": "non-impacting",
|
|
11694
11817
|
"duplicate-normalization": "non-impacting",
|
|
11695
11818
|
"invalid-url": "non-impacting",
|
|
@@ -11697,6 +11820,7 @@ const SKIP_IMPACT = {
|
|
|
11697
11820
|
"off-origin": "non-impacting",
|
|
11698
11821
|
"origin-backpressure": "coverage-affecting",
|
|
11699
11822
|
"queue-safety-limit": "coverage-affecting",
|
|
11823
|
+
"query-variant-limit": "coverage-affecting",
|
|
11700
11824
|
"robots-disallowed": "coverage-affecting",
|
|
11701
11825
|
"robots-uncertain": "coverage-affecting",
|
|
11702
11826
|
"legacy-unclassified": "coverage-affecting"
|
|
@@ -12315,64 +12439,169 @@ function refreshPrioritiesCsvFiles(report) {
|
|
|
12315
12439
|
...prefixCsvFiles("diagnosis", diagnoseCsvFiles(report.output.diagnosis))
|
|
12316
12440
|
]);
|
|
12317
12441
|
}
|
|
12318
|
-
// packages/core/src/analyze/crawler/
|
|
12319
|
-
|
|
12320
|
-
|
|
12321
|
-
medium: 1200,
|
|
12322
|
-
low: 100
|
|
12323
|
-
};
|
|
12324
|
-
const EFFORT_BY_CATEGORY = {
|
|
12325
|
-
canonical: "medium",
|
|
12326
|
-
content: "high",
|
|
12327
|
-
agents: "low",
|
|
12328
|
-
headings: "low",
|
|
12329
|
-
images: "medium",
|
|
12330
|
-
indexability: "medium",
|
|
12331
|
-
international: "low",
|
|
12332
|
-
metadata: "low",
|
|
12333
|
-
mobile: "low",
|
|
12334
|
-
response: "medium",
|
|
12335
|
-
security: "medium",
|
|
12336
|
-
social: "low",
|
|
12337
|
-
"structured-data": "medium"
|
|
12338
|
-
};
|
|
12339
|
-
const EFFORT_SCORE = {
|
|
12340
|
-
low: 60,
|
|
12341
|
-
medium: 30,
|
|
12342
|
-
high: 0
|
|
12343
|
-
};
|
|
12344
|
-
const AFFECTED_URL_WEIGHT = {
|
|
12345
|
-
high: 25,
|
|
12346
|
-
medium: 15,
|
|
12347
|
-
low: 2
|
|
12348
|
-
};
|
|
12349
|
-
const VALUE_WEIGHT = {
|
|
12350
|
-
high: 1,
|
|
12351
|
-
medium: .75,
|
|
12352
|
-
low: .15
|
|
12353
|
-
};
|
|
12354
|
-
function matchesPattern(pattern, value) {
|
|
12355
|
-
if (!pattern.includes("*")) return value.includes(pattern);
|
|
12356
|
-
const parts = pattern.split("*");
|
|
12357
|
-
let position = 0;
|
|
12358
|
-
for (const part of parts) {
|
|
12359
|
-
if (!part) continue;
|
|
12360
|
-
const found = value.slice(position).indexOf(part);
|
|
12361
|
-
if (found === -1) return false;
|
|
12362
|
-
position += found + part.length;
|
|
12363
|
-
}
|
|
12364
|
-
return true;
|
|
12442
|
+
// packages/core/src/analyze/crawler/crawl-url-queue.ts
|
|
12443
|
+
function before(left, right) {
|
|
12444
|
+
return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
|
|
12365
12445
|
}
|
|
12366
|
-
|
|
12367
|
-
|
|
12368
|
-
|
|
12369
|
-
|
|
12370
|
-
|
|
12371
|
-
|
|
12446
|
+
var CrawlUrlQueue = class {
|
|
12447
|
+
#items = [];
|
|
12448
|
+
#positions = /* @__PURE__ */ new Map();
|
|
12449
|
+
get size() {
|
|
12450
|
+
return this.#items.length;
|
|
12451
|
+
}
|
|
12452
|
+
has(url) {
|
|
12453
|
+
return this.#positions.has(url);
|
|
12454
|
+
}
|
|
12455
|
+
push(item) {
|
|
12456
|
+
if (this.has(item.url)) throw new Error(`URL is already queued: ${item.url}`);
|
|
12457
|
+
const index = this.#items.length;
|
|
12458
|
+
this.#items.push(item);
|
|
12459
|
+
this.#positions.set(item.url, index);
|
|
12460
|
+
this.#bubbleUp(index);
|
|
12461
|
+
}
|
|
12462
|
+
decreaseDepth(url, depth) {
|
|
12463
|
+
const index = this.#positions.get(url);
|
|
12464
|
+
if (index === void 0) return false;
|
|
12465
|
+
const item = this.#items[index];
|
|
12466
|
+
if (!item || depth >= item.depth) return false;
|
|
12467
|
+
item.depth = depth;
|
|
12468
|
+
this.#bubbleUp(index);
|
|
12372
12469
|
return true;
|
|
12373
|
-
}
|
|
12470
|
+
}
|
|
12471
|
+
take() {
|
|
12472
|
+
const first = this.#items[0];
|
|
12473
|
+
if (!first) return void 0;
|
|
12474
|
+
const last = this.#items.pop();
|
|
12475
|
+
this.#positions.delete(first.url);
|
|
12476
|
+
if (last && last !== first) {
|
|
12477
|
+
this.#items[0] = last;
|
|
12478
|
+
this.#positions.set(last.url, 0);
|
|
12479
|
+
this.#bubbleDown(0);
|
|
12480
|
+
}
|
|
12481
|
+
return first;
|
|
12482
|
+
}
|
|
12483
|
+
#swap(left, right) {
|
|
12484
|
+
const leftItem = this.#items[left];
|
|
12485
|
+
const rightItem = this.#items[right];
|
|
12486
|
+
if (!leftItem || !rightItem) return;
|
|
12487
|
+
this.#items[left] = rightItem;
|
|
12488
|
+
this.#items[right] = leftItem;
|
|
12489
|
+
this.#positions.set(rightItem.url, left);
|
|
12490
|
+
this.#positions.set(leftItem.url, right);
|
|
12491
|
+
}
|
|
12492
|
+
#bubbleUp(start) {
|
|
12493
|
+
let index = start;
|
|
12494
|
+
while (index > 0) {
|
|
12495
|
+
const parent = Math.floor((index - 1) / 2);
|
|
12496
|
+
const item = this.#items[index];
|
|
12497
|
+
const parentItem = this.#items[parent];
|
|
12498
|
+
if (!item || !parentItem || !before(item, parentItem)) break;
|
|
12499
|
+
this.#swap(index, parent);
|
|
12500
|
+
index = parent;
|
|
12501
|
+
}
|
|
12502
|
+
}
|
|
12503
|
+
#bubbleDown(start) {
|
|
12504
|
+
let index = start;
|
|
12505
|
+
while (true) {
|
|
12506
|
+
const left = index * 2 + 1;
|
|
12507
|
+
const right = left + 1;
|
|
12508
|
+
let next = index;
|
|
12509
|
+
const item = this.#items[next];
|
|
12510
|
+
const leftItem = this.#items[left];
|
|
12511
|
+
const rightItem = this.#items[right];
|
|
12512
|
+
if (leftItem && item && before(leftItem, item)) next = left;
|
|
12513
|
+
const nextItem = this.#items[next];
|
|
12514
|
+
if (rightItem && nextItem && before(rightItem, nextItem)) next = right;
|
|
12515
|
+
if (next === index) return;
|
|
12516
|
+
this.#swap(index, next);
|
|
12517
|
+
index = next;
|
|
12518
|
+
}
|
|
12519
|
+
}
|
|
12520
|
+
};
|
|
12521
|
+
// packages/core/src/analyze/crawler/report.ts
|
|
12522
|
+
const MAX_CRAWL_PAGES = 1e4;
|
|
12523
|
+
const MAX_CRAWL_DEPTH = 64;
|
|
12524
|
+
const MAX_CRAWL_CONCURRENCY = 16;
|
|
12525
|
+
const MAX_CRAWL_TIMEOUT_MS = 12e4;
|
|
12526
|
+
const SENSITIVE_FIELD_NAMES = /* @__PURE__ */ new Set([
|
|
12527
|
+
"authorization",
|
|
12528
|
+
"cookie",
|
|
12529
|
+
"proxy-authorization",
|
|
12530
|
+
"set-cookie",
|
|
12531
|
+
"www-authenticate",
|
|
12532
|
+
"x-api-key",
|
|
12533
|
+
"api-key",
|
|
12534
|
+
"access-token",
|
|
12535
|
+
"refresh-token"
|
|
12536
|
+
]);
|
|
12537
|
+
const SENSITIVE_PARAM_PATTERN = /^(access_?token|auth|api_?key|client_?secret|code|jwt|key|password|refresh_?token|secret|session|sig|signature|token)$/i;
|
|
12538
|
+
function uniqueSorted(values = []) {
|
|
12539
|
+
return [...new Set(values.filter(Boolean))].sort();
|
|
12374
12540
|
}
|
|
12375
|
-
function
|
|
12541
|
+
function normalizeFetchRate(input, strategy) {
|
|
12542
|
+
const defaultConcurrency = strategy === "health" ? 4 : 8;
|
|
12543
|
+
return {
|
|
12544
|
+
concurrency: input.fetchRate?.concurrency ?? input.concurrency ?? defaultConcurrency,
|
|
12545
|
+
...input.fetchRate?.intervalCap !== void 0 ? { intervalCap: input.fetchRate.intervalCap } : {},
|
|
12546
|
+
...input.fetchRate?.intervalMs !== void 0 ? { intervalMs: input.fetchRate.intervalMs } : {},
|
|
12547
|
+
...input.fetchRate?.backpressure ? { backpressure: input.fetchRate.backpressure } : {}
|
|
12548
|
+
};
|
|
12549
|
+
}
|
|
12550
|
+
function normalizeCrawlConfig(input) {
|
|
12551
|
+
const strategy = input.strategy ?? "full";
|
|
12552
|
+
const concurrency = input.concurrency ?? (strategy === "health" ? 4 : 8);
|
|
12553
|
+
return {
|
|
12554
|
+
url: new URL(input.url).toString(),
|
|
12555
|
+
mode: input.mode ?? (strategy === "health" ? "sitemap" : "site"),
|
|
12556
|
+
strategy,
|
|
12557
|
+
...input.sitemapUrl ? { sitemapUrl: new URL(input.sitemapUrl).toString() } : {},
|
|
12558
|
+
urls: uniqueSorted(input.urls ?? []),
|
|
12559
|
+
maxPages: input.maxPages ?? 500,
|
|
12560
|
+
maxDepth: input.maxDepth ?? 16,
|
|
12561
|
+
concurrency,
|
|
12562
|
+
timeoutMs: input.timeoutMs ?? 2e4,
|
|
12563
|
+
include: uniqueSorted(input.include ?? []),
|
|
12564
|
+
exclude: uniqueSorted(input.exclude ?? []),
|
|
12565
|
+
respectRobots: input.respectRobots ?? true,
|
|
12566
|
+
useSitemap: input.useSitemap ?? true,
|
|
12567
|
+
checkExternal: strategy === "health" ? false : input.checkExternal ?? true,
|
|
12568
|
+
js: strategy === "health" ? "off" : normalizeJavaScriptRenderingMode(input.js),
|
|
12569
|
+
refresh: strategy === "health" ? true : input.refresh ?? false,
|
|
12570
|
+
fetchRate: normalizeFetchRate(input, strategy)
|
|
12571
|
+
};
|
|
12572
|
+
}
|
|
12573
|
+
function assertIntegerRange(value, label, minimum, maximum) {
|
|
12574
|
+
if (!Number.isInteger(value) || value < minimum || value > maximum) throw new RangeError(`${label} must be an integer from ${minimum} to ${maximum}.`);
|
|
12575
|
+
}
|
|
12576
|
+
function assertCrawlConfigLimits(config) {
|
|
12577
|
+
if (config.strategy === "health" && config.mode !== "sitemap") throw new RangeError("The health strategy requires sitemap mode.");
|
|
12578
|
+
if (config.strategy === "health" && !config.useSitemap) throw new RangeError("The health strategy requires sitemap discovery.");
|
|
12579
|
+
assertIntegerRange(config.maxPages, "maxPages", 1, MAX_CRAWL_PAGES);
|
|
12580
|
+
assertIntegerRange(config.maxDepth, "maxDepth", 0, 64);
|
|
12581
|
+
assertIntegerRange(config.concurrency, "concurrency", 1, 16);
|
|
12582
|
+
assertIntegerRange(config.timeoutMs, "timeoutMs", 1, MAX_CRAWL_TIMEOUT_MS);
|
|
12583
|
+
if (config.fetchRate.concurrency !== void 0) assertIntegerRange(config.fetchRate.concurrency, "fetchRate.concurrency", 1, 16);
|
|
12584
|
+
if (config.strategy === "health" && config.mode !== "sitemap") throw new RangeError("Health strategy requires sitemap mode.");
|
|
12585
|
+
if (config.sitemapUrl && config.mode !== "sitemap") throw new RangeError("sitemapUrl requires sitemap mode.");
|
|
12586
|
+
return config;
|
|
12587
|
+
}
|
|
12588
|
+
function crawlConfigHash(config) {
|
|
12589
|
+
const normalized = normalizeCrawlConfig(config);
|
|
12590
|
+
return createHash("sha256").update(JSON.stringify(normalized)).digest("hex").slice(0, 16);
|
|
12591
|
+
}
|
|
12592
|
+
function crawlDefinitionId(input) {
|
|
12593
|
+
const normalized = {
|
|
12594
|
+
config: normalizeCrawlConfig(input.config),
|
|
12595
|
+
site: input.site ?? null,
|
|
12596
|
+
googleAnalyticsPropertyId: input.googleAnalyticsPropertyId ?? null,
|
|
12597
|
+
...input.analyticsConnection ? { analyticsConnection: input.analyticsConnection } : {}
|
|
12598
|
+
};
|
|
12599
|
+
return `crawl_def_${createHash("sha256").update(JSON.stringify(normalized)).digest("hex").slice(0, 20)}`;
|
|
12600
|
+
}
|
|
12601
|
+
function crawlRunId() {
|
|
12602
|
+
return `crawl_${randomUUID().replaceAll("-", "")}`;
|
|
12603
|
+
}
|
|
12604
|
+
function groupCrawlIssues(issues) {
|
|
12376
12605
|
const groups = /* @__PURE__ */ new Map();
|
|
12377
12606
|
for (const issue of issues) {
|
|
12378
12607
|
const existing = groups.get(issue.ruleId) ?? {
|
|
@@ -12387,540 +12616,51 @@ function groupIssues(issues) {
|
|
|
12387
12616
|
if (existing.sampleUrls.length < 10) existing.sampleUrls.push(issue.url);
|
|
12388
12617
|
groups.set(issue.ruleId, existing);
|
|
12389
12618
|
}
|
|
12390
|
-
|
|
12391
|
-
|
|
12392
|
-
|
|
12393
|
-
|
|
12394
|
-
const pages = report.pages.filter((page) => wanted.has(page.url) || wanted.has(page.finalUrl));
|
|
12395
|
-
const metrics = pages.map((page) => page.searchMetrics).filter((item) => Boolean(item));
|
|
12396
|
-
const clicks = metrics.reduce((sum, item) => sum + item.clicks, 0);
|
|
12397
|
-
const impressions = metrics.reduce((sum, item) => sum + item.impressions, 0);
|
|
12398
|
-
const sessions = pages.reduce((sum, page) => sum + (page.analytics?.sessions ?? 0), 0);
|
|
12399
|
-
const observedMetrics = new Set(report.dataSources?.analytics.observedMetrics ?? [
|
|
12400
|
-
"sessions",
|
|
12401
|
-
"totalUsers",
|
|
12402
|
-
"conversions"
|
|
12403
|
-
]);
|
|
12404
|
-
const totalUsers = observedMetrics.has("totalUsers") ? pages.reduce((sum, page) => sum + (page.analytics?.totalUsers ?? 0), 0) : void 0;
|
|
12405
|
-
const conversions = observedMetrics.has("conversions") ? pages.reduce((sum, page) => sum + (page.analytics?.conversions ?? 0), 0) : void 0;
|
|
12406
|
-
const avgPosition = metrics.length ? metrics.reduce((sum, item) => sum + item.position, 0) / metrics.length : void 0;
|
|
12407
|
-
return {
|
|
12408
|
-
searchVisibleUrls: metrics.length,
|
|
12409
|
-
clicks,
|
|
12410
|
-
impressions,
|
|
12411
|
-
sessions,
|
|
12412
|
-
totalUsers,
|
|
12413
|
-
conversions,
|
|
12414
|
-
avgPosition
|
|
12619
|
+
const severityRank = {
|
|
12620
|
+
high: 3,
|
|
12621
|
+
medium: 2,
|
|
12622
|
+
low: 1
|
|
12415
12623
|
};
|
|
12624
|
+
return [...groups.values()].sort((a, b) => severityRank[b.severity] - severityRank[a.severity] || b.count - a.count);
|
|
12416
12625
|
}
|
|
12417
|
-
function
|
|
12418
|
-
const
|
|
12419
|
-
|
|
12420
|
-
const analyticsLabel = analyticsDetails.label;
|
|
12421
|
-
const sessionLabel = analyticsDetails.landingMetricLabel;
|
|
12422
|
-
return `${visibility}${input.sessions ? ` ${analyticsLabel} adds ${input.sessions} ${sessionLabel}${input.conversions === void 0 ? "" : ` and ${input.conversions} conversions`} from affected pages.` : ""} ${recommendation === "review" ? "This is a review observation. Confirm it before scheduling implementation work." : `This ranks as a fix. Severity is ${input.severity}; ${input.affectedUrls} URL${input.affectedUrls === 1 ? " is" : "s are"} affected; estimated effort is ${input.effort}.`}`;
|
|
12423
|
-
}
|
|
12424
|
-
function verificationCommand(report, severity) {
|
|
12425
|
-
return `seo crawl ${report.config.url} --severity ${severity} --max-pages ${Math.min(report.config.maxPages, 100)}`;
|
|
12426
|
-
}
|
|
12427
|
-
function topFixes(report, filters = {}) {
|
|
12428
|
-
return groupIssues(filteredIssues(report, filters)).map((group) => {
|
|
12429
|
-
const search = searchValueForGroup(report, group.sampleUrls);
|
|
12430
|
-
const effort = EFFORT_BY_CATEGORY[group.category] ?? "medium";
|
|
12431
|
-
const scoreFactors = {
|
|
12432
|
-
severity: SEVERITY_SCORE[group.severity],
|
|
12433
|
-
affectedUrls: group.count,
|
|
12434
|
-
searchVisibleUrls: search.searchVisibleUrls,
|
|
12435
|
-
clicks: search.clicks,
|
|
12436
|
-
impressions: search.impressions,
|
|
12437
|
-
sessions: search.sessions,
|
|
12438
|
-
totalUsers: search.totalUsers,
|
|
12439
|
-
conversions: search.conversions,
|
|
12440
|
-
avgPosition: search.avgPosition,
|
|
12441
|
-
effort,
|
|
12442
|
-
effortScore: EFFORT_SCORE[effort]
|
|
12443
|
-
};
|
|
12444
|
-
const valueWeight = VALUE_WEIGHT[group.severity];
|
|
12445
|
-
const score = scoreFactors.severity + scoreFactors.affectedUrls * AFFECTED_URL_WEIGHT[group.severity] + scoreFactors.searchVisibleUrls * 100 * valueWeight + scoreFactors.clicks * 20 * valueWeight + Math.min(scoreFactors.impressions, 1e4) / 25 * valueWeight + scoreFactors.sessions * 2 * valueWeight + (scoreFactors.conversions ?? 0) * 100 * valueWeight + scoreFactors.effortScore;
|
|
12446
|
-
const rule = explainRule(group.ruleId);
|
|
12447
|
-
const recommendation = recommendationForRule(group.ruleId);
|
|
12448
|
-
return {
|
|
12449
|
-
...group,
|
|
12450
|
-
recommendation,
|
|
12451
|
-
score: Math.round(score),
|
|
12452
|
-
scoreFactors,
|
|
12453
|
-
whyThisRanks: whyThisRanks(scoreFactors, recommendation, report.dataSources?.searchConsole.status, report.dataSources?.analytics.provider),
|
|
12454
|
-
howToFix: rule?.howToFix ?? "",
|
|
12455
|
-
howToVerify: rule?.howToVerify ?? "",
|
|
12456
|
-
verification: {
|
|
12457
|
-
command: verificationCommand(report, group.severity),
|
|
12458
|
-
expected: rule?.howToVerify ?? ""
|
|
12459
|
-
}
|
|
12460
|
-
};
|
|
12461
|
-
}).sort((a, b) => b.score - a.score || SEVERITY_SCORE[b.severity] - SEVERITY_SCORE[a.severity] || b.count - a.count || a.ruleId.localeCompare(b.ruleId)).slice(0, filters.limit ?? 10);
|
|
12462
|
-
}
|
|
12463
|
-
function reviewObservations(report, filters = {}) {
|
|
12464
|
-
return topFixes(report, {
|
|
12465
|
-
...filters,
|
|
12466
|
-
recommendation: "review"
|
|
12467
|
-
});
|
|
12626
|
+
function isSensitiveFieldName(key) {
|
|
12627
|
+
const lower = key.toLowerCase();
|
|
12628
|
+
return SENSITIVE_FIELD_NAMES.has(lower) || SENSITIVE_PARAM_PATTERN.test(lower.replace(/-/g, "_"));
|
|
12468
12629
|
}
|
|
12469
|
-
|
|
12470
|
-
|
|
12471
|
-
|
|
12472
|
-
|
|
12473
|
-
|
|
12474
|
-
|
|
12475
|
-
|
|
12476
|
-
|
|
12477
|
-
"clicks",
|
|
12478
|
-
"impressions",
|
|
12479
|
-
"position"
|
|
12480
|
-
];
|
|
12481
|
-
const pageHeaders = [
|
|
12482
|
-
"url",
|
|
12483
|
-
"final_url",
|
|
12484
|
-
"status",
|
|
12485
|
-
"indexable",
|
|
12486
|
-
"title",
|
|
12487
|
-
"meta_description",
|
|
12488
|
-
"canonical",
|
|
12489
|
-
"word_count",
|
|
12490
|
-
"content_extractor",
|
|
12491
|
-
"content_extractor_type",
|
|
12492
|
-
"content_extraction_fallback",
|
|
12493
|
-
"word_count_source",
|
|
12494
|
-
"internal_links",
|
|
12495
|
-
"internal_inlinks",
|
|
12496
|
-
"internal_authority_score",
|
|
12497
|
-
"external_links",
|
|
12498
|
-
"images_total",
|
|
12499
|
-
"images_missing_alt",
|
|
12500
|
-
"oversized_image_candidates",
|
|
12501
|
-
"schema_types",
|
|
12502
|
-
"clicks",
|
|
12503
|
-
"impressions",
|
|
12504
|
-
"sessions"
|
|
12505
|
-
];
|
|
12506
|
-
function renderCrawlCsv(report) {
|
|
12507
|
-
return renderCsv(report.issues.map((issue) => ({
|
|
12508
|
-
rule_id: issue.ruleId,
|
|
12509
|
-
title: issue.title,
|
|
12510
|
-
category: issue.category,
|
|
12511
|
-
severity: issue.severity,
|
|
12512
|
-
url: issue.url,
|
|
12513
|
-
detail: issue.detail,
|
|
12514
|
-
clicks: issue.searchMetrics?.clicks,
|
|
12515
|
-
impressions: issue.searchMetrics?.impressions,
|
|
12516
|
-
position: issue.searchMetrics?.position
|
|
12517
|
-
})), issueHeaders);
|
|
12630
|
+
function sanitizeUrlString(value) {
|
|
12631
|
+
try {
|
|
12632
|
+
const url = new URL(value);
|
|
12633
|
+
for (const key of [...url.searchParams.keys()]) if (SENSITIVE_PARAM_PATTERN.test(key)) url.searchParams.set(key, "[redacted]");
|
|
12634
|
+
return url.toString();
|
|
12635
|
+
} catch {
|
|
12636
|
+
return value;
|
|
12637
|
+
}
|
|
12518
12638
|
}
|
|
12519
|
-
function
|
|
12520
|
-
return
|
|
12521
|
-
url: page.url,
|
|
12522
|
-
final_url: page.finalUrl,
|
|
12523
|
-
status: page.status,
|
|
12524
|
-
indexable: page.indexable,
|
|
12525
|
-
title: page.title,
|
|
12526
|
-
meta_description: page.metaDescription,
|
|
12527
|
-
canonical: page.canonical,
|
|
12528
|
-
word_count: page.wordCount,
|
|
12529
|
-
content_extractor: page.contentExtraction?.used,
|
|
12530
|
-
content_extractor_type: page.contentExtraction?.extractorType,
|
|
12531
|
-
content_extraction_fallback: page.contentExtraction?.fallback,
|
|
12532
|
-
word_count_source: page.contentExtraction?.wordCountSource,
|
|
12533
|
-
internal_links: page.outgoingInternalCount,
|
|
12534
|
-
internal_inlinks: page.internalInlinkCount,
|
|
12535
|
-
internal_authority_score: page.internalLinkAuthorityScore,
|
|
12536
|
-
external_links: page.outgoingExternalCount,
|
|
12537
|
-
images_total: page.imagesTotal,
|
|
12538
|
-
images_missing_alt: page.imagesMissingAlt,
|
|
12539
|
-
oversized_image_candidates: page.oversizedImageCandidates?.length ?? 0,
|
|
12540
|
-
schema_types: page.schemaTypes?.join("|"),
|
|
12541
|
-
clicks: page.searchMetrics?.clicks,
|
|
12542
|
-
impressions: page.searchMetrics?.impressions,
|
|
12543
|
-
sessions: page.analytics?.sessions
|
|
12544
|
-
})), pageHeaders);
|
|
12639
|
+
function sanitizeTenantString(value) {
|
|
12640
|
+
return value.replace(/https?:\/\/[^\s"'<>`]+/g, (match) => sanitizeUrlString(match)).replace(/\b(access_?token|api_?key|auth|client_?secret|jwt|password|refresh_?token|secret|session|signature|token)(\s*[=:]\s*)[^\s,;&]+/gi, (_match, key, separator) => `${key}${separator}[redacted]`).replace(/\b[A-Za-z]:\\[^\s"'<>`]+/g, "[local-path]").replace(/\/(?:Users|home|tmp|var\/folders)\/[^\s"'<>`]+/g, "[local-path]");
|
|
12545
12641
|
}
|
|
12546
|
-
function
|
|
12547
|
-
|
|
12548
|
-
|
|
12549
|
-
|
|
12550
|
-
|
|
12551
|
-
|
|
12552
|
-
|
|
12553
|
-
|
|
12554
|
-
|
|
12555
|
-
`Documents: ${report.summary.totalPages} retained; ${requestSummary}, ${report.summary.discoveredUrls} URLs discovered, ${report.summary.failedUrls} failed, ${report.summary.skippedUrls} skipped`,
|
|
12556
|
-
`Issues: ${report.issues.length} (${report.summary.highIssues} high, ${report.summary.mediumIssues} medium, ${report.summary.lowIssues} low)`
|
|
12557
|
-
];
|
|
12558
|
-
if (report.externalLinkVerification) {
|
|
12559
|
-
const external = report.externalLinkVerification;
|
|
12560
|
-
lines.push(`External links: ${external.fetchedUrls} fetched, ${external.failedUrls} request failure${external.failedUrls === 1 ? "" : "s"}, ${external.deferredUrls} deferred (${external.dataStatus})`);
|
|
12561
|
-
}
|
|
12562
|
-
if (report.access.blockedRequests > 0) {
|
|
12563
|
-
lines.push("", `Access blocks: ${report.access.blockedRequests}`);
|
|
12564
|
-
for (const sample of report.access.samples.slice(0, 3)) {
|
|
12565
|
-
const requestId = sample.evidence.requestId ? `, request ID ${sample.evidence.requestId}` : "";
|
|
12566
|
-
lines.push(`- ${sample.url}: ${sample.evidence.guidance.summary} (${sample.evidence.provider}${requestId})`, ` Action: ${sample.evidence.guidance.recommendedAction}`, ` Security: ${sample.evidence.guidance.securityNote}`);
|
|
12567
|
-
if (sample.evidence.guidance.documentationUrl) lines.push(` Provider docs: ${sample.evidence.guidance.documentationUrl}`);
|
|
12642
|
+
function sanitizeTenantValue(value, key) {
|
|
12643
|
+
if (key && isSensitiveFieldName(key)) return value === "[redacted]" ? value : "[redacted]";
|
|
12644
|
+
if (typeof value === "string") return sanitizeTenantString(value);
|
|
12645
|
+
if (Array.isArray(value)) {
|
|
12646
|
+
let sanitized;
|
|
12647
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
12648
|
+
const item = sanitizeTenantValue(value[index]);
|
|
12649
|
+
if (sanitized) sanitized.push(item);
|
|
12650
|
+
else if (item !== value[index]) sanitized = [...value.slice(0, index), item];
|
|
12568
12651
|
}
|
|
12652
|
+
return sanitized ?? value;
|
|
12569
12653
|
}
|
|
12570
|
-
if (
|
|
12571
|
-
|
|
12572
|
-
|
|
12573
|
-
|
|
12574
|
-
|
|
12575
|
-
|
|
12576
|
-
|
|
12577
|
-
|
|
12578
|
-
lines.push("", "Review observations (check before scheduling work)");
|
|
12579
|
-
for (const observation of reviews.slice(0, 5)) {
|
|
12580
|
-
lines.push(`- ${observation.title} (${observation.severity}, ${observation.count} URL${observation.count === 1 ? "" : "s"}): ${observation.howToFix}`);
|
|
12581
|
-
if (observation.sampleUrls[0]) lines.push(` First URL: ${observation.sampleUrls[0]}`);
|
|
12582
|
-
}
|
|
12654
|
+
if (!value || typeof value !== "object") return value;
|
|
12655
|
+
const record = value;
|
|
12656
|
+
let sanitized;
|
|
12657
|
+
for (const [entryKey, entry] of Object.entries(record)) {
|
|
12658
|
+
const safeEntry = sanitizeTenantValue(entry, entryKey);
|
|
12659
|
+
if (safeEntry === entry) continue;
|
|
12660
|
+
sanitized ??= { ...record };
|
|
12661
|
+
sanitized[entryKey] = safeEntry;
|
|
12583
12662
|
}
|
|
12584
|
-
|
|
12585
|
-
for (const command of [...new Set(fixes.slice(0, 3).map((fix) => fix.verification.command))]) lines.push(`- ${command}`);
|
|
12586
|
-
if (report.warnings.length) lines.push("", "Warnings", ...report.warnings.map((item) => `- ${item}`));
|
|
12587
|
-
if (report.caveats.length) lines.push("", "Caveats", ...report.caveats.map((item) => `- ${item}`));
|
|
12588
|
-
return `${lines.join("\n")}\n`;
|
|
12589
|
-
}
|
|
12590
|
-
function renderCrawlJunit(report) {
|
|
12591
|
-
const issuesByUrl = /* @__PURE__ */ new Map();
|
|
12592
|
-
for (const issue of report.issues) {
|
|
12593
|
-
const existing = issuesByUrl.get(issue.url) ?? [];
|
|
12594
|
-
existing.push(issue);
|
|
12595
|
-
issuesByUrl.set(issue.url, existing);
|
|
12596
|
-
}
|
|
12597
|
-
const sitemapCases = (report.sitemapDiscovery?.roots ?? []).flatMap((root) => {
|
|
12598
|
-
return (root.documents.length ? root.documents : [{
|
|
12599
|
-
url: root.url,
|
|
12600
|
-
dataStatus: root.dataStatus,
|
|
12601
|
-
compression: "none",
|
|
12602
|
-
warning: root.warnings.join(" ") || void 0
|
|
12603
|
-
}]).map((document) => {
|
|
12604
|
-
const failures = [
|
|
12605
|
-
...document.dataStatus !== "complete" ? [document.warning ?? `Sitemap evidence is ${document.dataStatus}.`] : [],
|
|
12606
|
-
...document.redirected ? [`Sitemap redirected to ${document.finalUrl ?? "an unknown URL"}.`] : [],
|
|
12607
|
-
...document.status !== void 0 && (document.status < 200 || document.status >= 300) ? [`Sitemap returned HTTP ${document.status}.`] : []
|
|
12608
|
-
];
|
|
12609
|
-
const output = [
|
|
12610
|
-
`dataStatus=${document.dataStatus}`,
|
|
12611
|
-
...document.status !== void 0 ? [`status=${document.status}`] : [],
|
|
12612
|
-
...document.finalUrl ? [`finalUrl=${document.finalUrl}`] : [],
|
|
12613
|
-
...document.root ? [`root=${document.root}`] : [],
|
|
12614
|
-
...document.warning ? [`warning=${document.warning}`] : []
|
|
12615
|
-
].join("\n");
|
|
12616
|
-
return {
|
|
12617
|
-
xml: [
|
|
12618
|
-
` <testcase classname="seo.sitemap-document" name="${escapeXml(document.url)}" time="0.000">`,
|
|
12619
|
-
...failures.length ? [` <failure message="${escapeXml(failures[0] ?? "Sitemap check failed")}">${escapeXml(failures.join("\n"))}</failure>`] : [],
|
|
12620
|
-
` <system-out>${escapeXml(output)}</system-out>`,
|
|
12621
|
-
" </testcase>"
|
|
12622
|
-
].join("\n"),
|
|
12623
|
-
failed: failures.length > 0
|
|
12624
|
-
};
|
|
12625
|
-
});
|
|
12626
|
-
});
|
|
12627
|
-
const requestCases = report.requests.map((request) => {
|
|
12628
|
-
const failures = [
|
|
12629
|
-
...(issuesByUrl.get(request.requestedUrl) ?? []).map((issue) => `${issue.severity}: ${issue.title}${issue.detail ? `: ${issue.detail}` : ""}`),
|
|
12630
|
-
...request.outcome === "failure" ? [`${request.failureKind}: ${request.error}`] : [],
|
|
12631
|
-
...request.outcome === "skipped" ? [request.reason === "origin-backpressure" ? request.error : `${request.reason}: ${request.robotsTxt.matchedLine ?? request.robotsTxt.availability}`] : []
|
|
12632
|
-
];
|
|
12633
|
-
const output = request.outcome === "response" ? [
|
|
12634
|
-
`status=${request.status}`,
|
|
12635
|
-
`finalUrl=${request.finalUrl}`,
|
|
12636
|
-
...request.robotsTxt ? [
|
|
12637
|
-
`robotsUrl=${request.robotsTxt.url}`,
|
|
12638
|
-
`robotsAllowed=${request.robotsTxt.allowed ?? "unknown"}`,
|
|
12639
|
-
`robotsAvailability=${request.robotsTxt.availability}`,
|
|
12640
|
-
...request.robotsTxt.status !== void 0 ? [`robotsStatus=${request.robotsTxt.status}`] : [],
|
|
12641
|
-
...request.robotsTxt.matchedLine ? [`robotsMatchedLine=${request.robotsTxt.matchedLine}`] : [],
|
|
12642
|
-
...request.robotsTxt.error ? [`robotsError=${request.robotsTxt.error}`] : []
|
|
12643
|
-
] : [],
|
|
12644
|
-
...request.accessBlock ? [
|
|
12645
|
-
`accessProvider=${request.accessBlock.provider}`,
|
|
12646
|
-
`crawler=${request.accessBlock.crawler.userAgent}`,
|
|
12647
|
-
...request.accessBlock.requestId ? [`requestId=${request.accessBlock.requestId}`] : []
|
|
12648
|
-
] : []
|
|
12649
|
-
].join("\n") : request.outcome === "failure" ? request.error : request.reason === "origin-backpressure" ? `reason=${request.reason}\nerror=${request.error}` : [
|
|
12650
|
-
`reason=${request.reason}`,
|
|
12651
|
-
`robotsUrl=${request.robotsTxt.url}`,
|
|
12652
|
-
`robotsAllowed=${request.robotsTxt.allowed ?? "unknown"}`,
|
|
12653
|
-
`robotsAvailability=${request.robotsTxt.availability}`,
|
|
12654
|
-
...request.robotsTxt.status !== void 0 ? [`robotsStatus=${request.robotsTxt.status}`] : [],
|
|
12655
|
-
...request.robotsTxt.matchedLine ? [`robotsMatchedLine=${request.robotsTxt.matchedLine}`] : [],
|
|
12656
|
-
...request.robotsTxt.error ? [`robotsError=${request.robotsTxt.error}`] : []
|
|
12657
|
-
].join("\n");
|
|
12658
|
-
const duration = ((request.durationMs ?? 0) / 1e3).toFixed(3);
|
|
12659
|
-
return {
|
|
12660
|
-
xml: [
|
|
12661
|
-
` <testcase classname="seo.sitemap-health" name="${escapeXml(request.requestedUrl)}" time="${duration}">`,
|
|
12662
|
-
...failures.length ? [` <failure message="${escapeXml(failures[0] ?? "Health check failed")}">${escapeXml(failures.join("\n"))}</failure>`] : [],
|
|
12663
|
-
` <system-out>${escapeXml(output)}</system-out>`,
|
|
12664
|
-
" </testcase>"
|
|
12665
|
-
].join("\n"),
|
|
12666
|
-
failed: failures.length > 0
|
|
12667
|
-
};
|
|
12668
|
-
});
|
|
12669
|
-
const cases = [...sitemapCases, ...requestCases];
|
|
12670
|
-
const failures = cases.filter((item) => item.failed).length;
|
|
12671
|
-
const elapsed = report.requests.reduce((sum, request) => sum + (request.durationMs ?? 0), 0);
|
|
12672
|
-
return [
|
|
12673
|
-
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>",
|
|
12674
|
-
`<testsuite name="seo sitemap health" tests="${cases.length}" failures="${failures}" errors="0" skipped="0" time="${(elapsed / 1e3).toFixed(3)}">`,
|
|
12675
|
-
" <properties>",
|
|
12676
|
-
` <property name="report.id" value="${escapeXml(report.id)}"/>`,
|
|
12677
|
-
` <property name="report.status" value="${escapeXml(report.status)}"/>`,
|
|
12678
|
-
` <property name="crawl.strategy" value="${escapeXml(report.config.strategy)}"/>`,
|
|
12679
|
-
` <property name="crawler.userAgent" value="${escapeXml(report.access.crawler.userAgent)}"/>`,
|
|
12680
|
-
" </properties>",
|
|
12681
|
-
...cases.map((item) => item.xml),
|
|
12682
|
-
"</testsuite>",
|
|
12683
|
-
""
|
|
12684
|
-
].join("\n");
|
|
12685
|
-
}
|
|
12686
|
-
function renderCrawlHtml(report, fixes = topFixes(report)) {
|
|
12687
|
-
const reviews = reviewObservations(report);
|
|
12688
|
-
const issueRows = report.issueGroups.map((group) => {
|
|
12689
|
-
const rule = explainRule(group.ruleId);
|
|
12690
|
-
return `<tr><td>${escapeHtml(group.severity)}</td><td>${escapeHtml(group.title)}</td><td>${group.count}</td><td>${escapeHtml(rule?.howToFix ?? "")}</td><td>${escapeHtml(group.sampleUrls[0] ?? "")}</td></tr>`;
|
|
12691
|
-
}).join("\n");
|
|
12692
|
-
const fixItems = fixes.slice(0, 10).map((fix) => `<li><strong>${escapeHtml(fix.title)}</strong><br>${escapeHtml(fix.howToFix)}<br><span>Verify: ${escapeHtml(fix.howToVerify)}</span><br><code>${escapeHtml(fix.verification.command)}</code></li>`).join("\n");
|
|
12693
|
-
const reviewItems = reviews.slice(0, 10).map((observation) => `<li><strong>${escapeHtml(observation.title)}</strong><br>${escapeHtml(observation.howToFix)}<br><span>Check before scheduling work. ${escapeHtml(observation.howToVerify)}</span></li>`).join("\n");
|
|
12694
|
-
const caveatItems = report.caveats.map((caveat) => `<li>${escapeHtml(caveat)}</li>`).join("\n");
|
|
12695
|
-
const warningItems = report.warnings.map((warning) => `<li>${escapeHtml(warning)}</li>`).join("\n");
|
|
12696
|
-
const accessItems = report.access.samples.map((sample) => `<li><strong>${escapeHtml(sample.url)}</strong><br>${escapeHtml(sample.evidence.guidance.summary)}<br>${escapeHtml(sample.evidence.guidance.recommendedAction)}<br><span>${escapeHtml(sample.evidence.guidance.securityNote)}</span></li>`).join("\n");
|
|
12697
|
-
return `<!doctype html>
|
|
12698
|
-
<html lang="en">
|
|
12699
|
-
<head>
|
|
12700
|
-
<meta charset="utf-8">
|
|
12701
|
-
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
12702
|
-
<title>Crawl report for ${escapeHtml(report.config.url)}</title>
|
|
12703
|
-
<style>
|
|
12704
|
-
body { color: #17202a; font: 15px/1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; margin: 32px; max-width: 1120px; }
|
|
12705
|
-
h1, h2 { line-height: 1.2; }
|
|
12706
|
-
.summary { display: grid; gap: 12px; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); margin: 24px 0; }
|
|
12707
|
-
.metric { border: 1px solid #d8dee4; border-radius: 8px; padding: 12px; }
|
|
12708
|
-
.metric strong { display: block; font-size: 24px; }
|
|
12709
|
-
table { border-collapse: collapse; width: 100%; }
|
|
12710
|
-
th, td { border-bottom: 1px solid #d8dee4; padding: 10px; text-align: left; vertical-align: top; }
|
|
12711
|
-
th { background: #f6f8fa; }
|
|
12712
|
-
li { margin: 0 0 12px; }
|
|
12713
|
-
span { color: #57606a; }
|
|
12714
|
-
</style>
|
|
12715
|
-
</head>
|
|
12716
|
-
<body>
|
|
12717
|
-
<h1>Crawl report</h1>
|
|
12718
|
-
<p>${escapeHtml(report.config.url)}</p>
|
|
12719
|
-
<p>Strategy: ${escapeHtml(report.config.strategy === "health" ? "sitemap health pass" : "full audit")}<br>Crawler: <code>${escapeHtml(report.access.crawler.userAgent)}</code></p>
|
|
12720
|
-
<section class="summary">
|
|
12721
|
-
<div class="metric"><span>Status</span><strong>${escapeHtml(report.status)}</strong></div>
|
|
12722
|
-
<div class="metric"><span>Documents</span><strong>${report.summary.totalPages}</strong></div>
|
|
12723
|
-
<div class="metric"><span>Requests</span><strong>${report.requestEvidenceStatus === "unavailable" ? "n/a" : report.requests.length}${report.requestEvidenceStatus === "partial" ? " (partial)" : ""}</strong></div>
|
|
12724
|
-
<div class="metric"><span>Discovered</span><strong>${report.summary.discoveredUrls}</strong></div>
|
|
12725
|
-
<div class="metric"><span>Queued</span><strong>${report.summary.queuedUrls}</strong></div>
|
|
12726
|
-
<div class="metric"><span>Skipped</span><strong>${report.summary.skippedUrls}</strong></div>
|
|
12727
|
-
<div class="metric"><span>Failed fetches</span><strong>${report.summary.failedUrls}</strong></div>
|
|
12728
|
-
<div class="metric"><span>Observed internal links</span><strong>${report.summary.observedInternalLinks}</strong></div>
|
|
12729
|
-
<div class="metric"><span>Issues</span><strong>${report.issues.length}</strong></div>
|
|
12730
|
-
<div class="metric"><span>High</span><strong>${report.summary.highIssues}</strong></div>
|
|
12731
|
-
</section>
|
|
12732
|
-
${accessItems ? `<h2>Access blocks</h2><ul>${accessItems}</ul>` : ""}
|
|
12733
|
-
<h2>Prioritised fixes</h2>
|
|
12734
|
-
${fixItems ? `<ol>${fixItems}</ol>` : "<p>No prioritised fixes found.</p>"}
|
|
12735
|
-
${reviewItems ? `<h2>Review observations</h2>
|
|
12736
|
-
<p>Check these before scheduling implementation work.</p>
|
|
12737
|
-
<ul>${reviewItems}</ul>` : ""}
|
|
12738
|
-
${report.caveats.length ? `<h2>Caveats</h2>
|
|
12739
|
-
<ul>${caveatItems}</ul>` : ""}
|
|
12740
|
-
${report.warnings.length ? `<h2>Warnings</h2>
|
|
12741
|
-
<ul>${warningItems}</ul>` : ""}
|
|
12742
|
-
<h2>Issue groups</h2>
|
|
12743
|
-
<table>
|
|
12744
|
-
<thead><tr><th>Severity</th><th>Issue</th><th>URLs</th><th>Fix</th><th>Sample URL</th></tr></thead>
|
|
12745
|
-
<tbody>${issueRows}</tbody>
|
|
12746
|
-
</table>
|
|
12747
|
-
</body>
|
|
12748
|
-
</html>
|
|
12749
|
-
`;
|
|
12750
|
-
}
|
|
12751
|
-
function renderCrawlMarkdownTickets(report, fixes = topFixes(report)) {
|
|
12752
|
-
const reviews = reviewObservations(report);
|
|
12753
|
-
const lines = [
|
|
12754
|
-
`# Crawl Implementation Tickets`,
|
|
12755
|
-
"",
|
|
12756
|
-
`Source: ${report.config.url}`,
|
|
12757
|
-
`Report: ${report.id}`,
|
|
12758
|
-
`Status: ${report.status}`,
|
|
12759
|
-
`Strategy: ${report.config.strategy}`,
|
|
12760
|
-
`Crawler: ${report.access.crawler.userAgent}`,
|
|
12761
|
-
""
|
|
12762
|
-
];
|
|
12763
|
-
if (report.caveats.length) lines.push("## Caveats", "", ...report.caveats.map((item) => `- ${item}`), "");
|
|
12764
|
-
if (report.warnings.length) lines.push("## Warnings", "", ...report.warnings.map((item) => `- ${item}`), "");
|
|
12765
|
-
if (report.access.blockedRequests > 0) lines.push("## Access blocks", "", ...report.access.samples.flatMap((sample) => [
|
|
12766
|
-
`- ${sample.url}: ${sample.evidence.guidance.summary}`,
|
|
12767
|
-
` - Action: ${sample.evidence.guidance.recommendedAction}`,
|
|
12768
|
-
` - Security: ${sample.evidence.guidance.securityNote}`
|
|
12769
|
-
]), "");
|
|
12770
|
-
for (const [index, fix] of fixes.entries()) lines.push(`## ${index + 1}. ${fix.title}`, "", `- [ ] Fix ${fix.count} affected URL${fix.count === 1 ? "" : "s"}`, `- Severity: ${fix.severity}`, `- Rule: ${fix.ruleId}`, `- Why this ranks: ${fix.whyThisRanks}`, `- Plain-English fix: ${fix.howToFix}`, `- Verify: ${fix.howToVerify}`, `- Command: ${fix.verification.command}`, "", `Affected URLs:`, ...fix.sampleUrls.slice(0, 10).map((url) => `- ${url}`), "");
|
|
12771
|
-
if (!fixes.length) lines.push("No implementation tickets were generated.");
|
|
12772
|
-
if (reviews.length) lines.push("## Review observations", "", "These are not implementation tickets. Check the evidence before deciding whether to work on them.", "", ...reviews.slice(0, 10).flatMap((observation) => [`- ${observation.title} (${observation.severity}, ${observation.count} URL${observation.count === 1 ? "" : "s"}): ${observation.howToFix}`, ` - First URL: ${observation.sampleUrls[0] ?? "No sample URL recorded"}`]), "");
|
|
12773
|
-
return `${lines.join("\n")}\n`;
|
|
12774
|
-
}
|
|
12775
|
-
function escapeHtml(value) {
|
|
12776
|
-
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll("\"", """).replaceAll("'", "'");
|
|
12777
|
-
}
|
|
12778
|
-
function escapeXml(value) {
|
|
12779
|
-
return escapeHtml(value);
|
|
12780
|
-
}
|
|
12781
|
-
// packages/core/src/analyze/crawler/report.ts
|
|
12782
|
-
const MAX_CRAWL_PAGES = 1e4;
|
|
12783
|
-
const MAX_CRAWL_DEPTH = 64;
|
|
12784
|
-
const MAX_CRAWL_CONCURRENCY = 16;
|
|
12785
|
-
const MAX_CRAWL_TIMEOUT_MS = 12e4;
|
|
12786
|
-
const SENSITIVE_FIELD_NAMES = /* @__PURE__ */ new Set([
|
|
12787
|
-
"authorization",
|
|
12788
|
-
"cookie",
|
|
12789
|
-
"proxy-authorization",
|
|
12790
|
-
"set-cookie",
|
|
12791
|
-
"www-authenticate",
|
|
12792
|
-
"x-api-key",
|
|
12793
|
-
"api-key",
|
|
12794
|
-
"access-token",
|
|
12795
|
-
"refresh-token"
|
|
12796
|
-
]);
|
|
12797
|
-
const SENSITIVE_PARAM_PATTERN = /^(access_?token|auth|api_?key|client_?secret|code|jwt|key|password|refresh_?token|secret|session|sig|signature|token)$/i;
|
|
12798
|
-
function uniqueSorted(values = []) {
|
|
12799
|
-
return [...new Set(values.filter(Boolean))].sort();
|
|
12800
|
-
}
|
|
12801
|
-
function normalizeFetchRate(input, strategy) {
|
|
12802
|
-
const defaultConcurrency = strategy === "health" ? 4 : 8;
|
|
12803
|
-
return {
|
|
12804
|
-
concurrency: input.fetchRate?.concurrency ?? input.concurrency ?? defaultConcurrency,
|
|
12805
|
-
...input.fetchRate?.intervalCap !== void 0 ? { intervalCap: input.fetchRate.intervalCap } : {},
|
|
12806
|
-
...input.fetchRate?.intervalMs !== void 0 ? { intervalMs: input.fetchRate.intervalMs } : {},
|
|
12807
|
-
...input.fetchRate?.backpressure ? { backpressure: input.fetchRate.backpressure } : {}
|
|
12808
|
-
};
|
|
12809
|
-
}
|
|
12810
|
-
function normalizeCrawlConfig(input) {
|
|
12811
|
-
const strategy = input.strategy ?? "full";
|
|
12812
|
-
const concurrency = input.concurrency ?? (strategy === "health" ? 4 : 8);
|
|
12813
|
-
return {
|
|
12814
|
-
url: new URL(input.url).toString(),
|
|
12815
|
-
mode: input.mode ?? (strategy === "health" ? "sitemap" : "site"),
|
|
12816
|
-
strategy,
|
|
12817
|
-
...input.sitemapUrl ? { sitemapUrl: new URL(input.sitemapUrl).toString() } : {},
|
|
12818
|
-
urls: uniqueSorted(input.urls ?? []),
|
|
12819
|
-
maxPages: input.maxPages ?? 500,
|
|
12820
|
-
maxDepth: input.maxDepth ?? 16,
|
|
12821
|
-
concurrency,
|
|
12822
|
-
timeoutMs: input.timeoutMs ?? 2e4,
|
|
12823
|
-
include: uniqueSorted(input.include ?? []),
|
|
12824
|
-
exclude: uniqueSorted(input.exclude ?? []),
|
|
12825
|
-
respectRobots: input.respectRobots ?? true,
|
|
12826
|
-
useSitemap: input.useSitemap ?? true,
|
|
12827
|
-
checkExternal: strategy === "health" ? false : input.checkExternal ?? true,
|
|
12828
|
-
js: strategy === "health" ? "off" : normalizeJavaScriptRenderingMode(input.js),
|
|
12829
|
-
refresh: strategy === "health" ? true : input.refresh ?? false,
|
|
12830
|
-
fetchRate: normalizeFetchRate(input, strategy)
|
|
12831
|
-
};
|
|
12832
|
-
}
|
|
12833
|
-
function assertIntegerRange(value, label, minimum, maximum) {
|
|
12834
|
-
if (!Number.isInteger(value) || value < minimum || value > maximum) throw new RangeError(`${label} must be an integer from ${minimum} to ${maximum}.`);
|
|
12835
|
-
}
|
|
12836
|
-
function assertCrawlConfigLimits(config) {
|
|
12837
|
-
if (config.strategy === "health" && config.mode !== "sitemap") throw new RangeError("The health strategy requires sitemap mode.");
|
|
12838
|
-
if (config.strategy === "health" && !config.useSitemap) throw new RangeError("The health strategy requires sitemap discovery.");
|
|
12839
|
-
assertIntegerRange(config.maxPages, "maxPages", 1, MAX_CRAWL_PAGES);
|
|
12840
|
-
assertIntegerRange(config.maxDepth, "maxDepth", 0, 64);
|
|
12841
|
-
assertIntegerRange(config.concurrency, "concurrency", 1, 16);
|
|
12842
|
-
assertIntegerRange(config.timeoutMs, "timeoutMs", 1, MAX_CRAWL_TIMEOUT_MS);
|
|
12843
|
-
if (config.fetchRate.concurrency !== void 0) assertIntegerRange(config.fetchRate.concurrency, "fetchRate.concurrency", 1, 16);
|
|
12844
|
-
if (config.strategy === "health" && config.mode !== "sitemap") throw new RangeError("Health strategy requires sitemap mode.");
|
|
12845
|
-
if (config.sitemapUrl && config.mode !== "sitemap") throw new RangeError("sitemapUrl requires sitemap mode.");
|
|
12846
|
-
return config;
|
|
12847
|
-
}
|
|
12848
|
-
function crawlConfigHash(config) {
|
|
12849
|
-
const normalized = normalizeCrawlConfig(config);
|
|
12850
|
-
return createHash("sha256").update(JSON.stringify(normalized)).digest("hex").slice(0, 16);
|
|
12851
|
-
}
|
|
12852
|
-
function crawlDefinitionId(input) {
|
|
12853
|
-
const normalized = {
|
|
12854
|
-
config: normalizeCrawlConfig(input.config),
|
|
12855
|
-
site: input.site ?? null,
|
|
12856
|
-
googleAnalyticsPropertyId: input.googleAnalyticsPropertyId ?? null,
|
|
12857
|
-
...input.analyticsConnection ? { analyticsConnection: input.analyticsConnection } : {}
|
|
12858
|
-
};
|
|
12859
|
-
return `crawl_def_${createHash("sha256").update(JSON.stringify(normalized)).digest("hex").slice(0, 20)}`;
|
|
12860
|
-
}
|
|
12861
|
-
function crawlRunId() {
|
|
12862
|
-
return `crawl_${randomUUID().replaceAll("-", "")}`;
|
|
12863
|
-
}
|
|
12864
|
-
function groupCrawlIssues(issues) {
|
|
12865
|
-
const groups = /* @__PURE__ */ new Map();
|
|
12866
|
-
for (const issue of issues) {
|
|
12867
|
-
const existing = groups.get(issue.ruleId) ?? {
|
|
12868
|
-
ruleId: issue.ruleId,
|
|
12869
|
-
title: issue.title,
|
|
12870
|
-
category: issue.category,
|
|
12871
|
-
severity: issue.severity,
|
|
12872
|
-
count: 0,
|
|
12873
|
-
sampleUrls: []
|
|
12874
|
-
};
|
|
12875
|
-
existing.count += 1;
|
|
12876
|
-
if (existing.sampleUrls.length < 10) existing.sampleUrls.push(issue.url);
|
|
12877
|
-
groups.set(issue.ruleId, existing);
|
|
12878
|
-
}
|
|
12879
|
-
const severityRank = {
|
|
12880
|
-
high: 3,
|
|
12881
|
-
medium: 2,
|
|
12882
|
-
low: 1
|
|
12883
|
-
};
|
|
12884
|
-
return [...groups.values()].sort((a, b) => severityRank[b.severity] - severityRank[a.severity] || b.count - a.count);
|
|
12885
|
-
}
|
|
12886
|
-
function isSensitiveFieldName(key) {
|
|
12887
|
-
const lower = key.toLowerCase();
|
|
12888
|
-
return SENSITIVE_FIELD_NAMES.has(lower) || SENSITIVE_PARAM_PATTERN.test(lower.replace(/-/g, "_"));
|
|
12889
|
-
}
|
|
12890
|
-
function sanitizeUrlString(value) {
|
|
12891
|
-
try {
|
|
12892
|
-
const url = new URL(value);
|
|
12893
|
-
for (const key of [...url.searchParams.keys()]) if (SENSITIVE_PARAM_PATTERN.test(key)) url.searchParams.set(key, "[redacted]");
|
|
12894
|
-
return url.toString();
|
|
12895
|
-
} catch {
|
|
12896
|
-
return value;
|
|
12897
|
-
}
|
|
12898
|
-
}
|
|
12899
|
-
function sanitizeTenantString(value) {
|
|
12900
|
-
return value.replace(/https?:\/\/[^\s"'<>`]+/g, (match) => sanitizeUrlString(match)).replace(/\b(access_?token|api_?key|auth|client_?secret|jwt|password|refresh_?token|secret|session|signature|token)(\s*[=:]\s*)[^\s,;&]+/gi, (_match, key, separator) => `${key}${separator}[redacted]`).replace(/\b[A-Za-z]:\\[^\s"'<>`]+/g, "[local-path]").replace(/\/(?:Users|home|tmp|var\/folders)\/[^\s"'<>`]+/g, "[local-path]");
|
|
12901
|
-
}
|
|
12902
|
-
function sanitizeTenantValue(value, key) {
|
|
12903
|
-
if (key && isSensitiveFieldName(key)) return value === "[redacted]" ? value : "[redacted]";
|
|
12904
|
-
if (typeof value === "string") return sanitizeTenantString(value);
|
|
12905
|
-
if (Array.isArray(value)) {
|
|
12906
|
-
let sanitized;
|
|
12907
|
-
for (let index = 0; index < value.length; index += 1) {
|
|
12908
|
-
const item = sanitizeTenantValue(value[index]);
|
|
12909
|
-
if (sanitized) sanitized.push(item);
|
|
12910
|
-
else if (item !== value[index]) sanitized = [...value.slice(0, index), item];
|
|
12911
|
-
}
|
|
12912
|
-
return sanitized ?? value;
|
|
12913
|
-
}
|
|
12914
|
-
if (!value || typeof value !== "object") return value;
|
|
12915
|
-
const record = value;
|
|
12916
|
-
let sanitized;
|
|
12917
|
-
for (const [entryKey, entry] of Object.entries(record)) {
|
|
12918
|
-
const safeEntry = sanitizeTenantValue(entry, entryKey);
|
|
12919
|
-
if (safeEntry === entry) continue;
|
|
12920
|
-
sanitized ??= { ...record };
|
|
12921
|
-
sanitized[entryKey] = safeEntry;
|
|
12922
|
-
}
|
|
12923
|
-
return sanitized ?? value;
|
|
12663
|
+
return sanitized ?? value;
|
|
12924
12664
|
}
|
|
12925
12665
|
function sanitizeCrawlConfig(config) {
|
|
12926
12666
|
return sanitizeTenantValue(config);
|
|
@@ -12948,6 +12688,10 @@ function sanitizeMessages(values = []) {
|
|
|
12948
12688
|
function compareText$17(left, right) {
|
|
12949
12689
|
return left < right ? -1 : left > right ? 1 : 0;
|
|
12950
12690
|
}
|
|
12691
|
+
function compareCaveats(left, right) {
|
|
12692
|
+
const rank = (value) => value.startsWith("Coverage incomplete:") ? 0 : 1;
|
|
12693
|
+
return rank(left) - rank(right) || compareText$17(left, right);
|
|
12694
|
+
}
|
|
12951
12695
|
function sortPages(pages) {
|
|
12952
12696
|
return [...pages].sort((left, right) => compareText$17(left.url, right.url));
|
|
12953
12697
|
}
|
|
@@ -13182,7 +12926,7 @@ function createCrawlReport(input) {
|
|
|
13182
12926
|
...input.externalLinkVerification ? { externalLinkVerification: sanitizeTenantValue(input.externalLinkVerification) } : {},
|
|
13183
12927
|
...input.siteChecks ? { siteChecks: sanitizeTenantValue(input.siteChecks) } : {},
|
|
13184
12928
|
warnings: sanitizeMessages(input.warnings).sort(compareText$17),
|
|
13185
|
-
caveats: sanitizeMessages(input.caveats).sort(
|
|
12929
|
+
caveats: sanitizeMessages(input.caveats).sort(compareCaveats)
|
|
13186
12930
|
};
|
|
13187
12931
|
}
|
|
13188
12932
|
function normalizeLoadedCrawlReport(report) {
|
|
@@ -13214,9 +12958,547 @@ function normalizeLoadedCrawlReport(report) {
|
|
|
13214
12958
|
issues,
|
|
13215
12959
|
issueGroups: groupCrawlIssues(issues),
|
|
13216
12960
|
warnings: sanitizeMessages(report.warnings).sort(compareText$17),
|
|
13217
|
-
caveats: sanitizeMessages(report.caveats).sort(
|
|
12961
|
+
caveats: sanitizeMessages(report.caveats).sort(compareCaveats)
|
|
12962
|
+
};
|
|
12963
|
+
}
|
|
12964
|
+
// packages/core/src/analyze/crawler/crawl-caveats.ts
|
|
12965
|
+
/** Reports rank a caveat with this prefix first and lead with it. */
|
|
12966
|
+
const CRAWL_COVERAGE_CAVEAT_PREFIX = "Coverage incomplete:";
|
|
12967
|
+
function rateLimitRerunCommand(config, intervalMs) {
|
|
12968
|
+
const scope = config.strategy === "health" ? " --health" : config.mode === "site" ? "" : ` --mode ${config.mode}`;
|
|
12969
|
+
const sitemap = config.sitemapUrl ? ` --sitemap-url ${config.sitemapUrl}` : "";
|
|
12970
|
+
const slowMs = Math.min(8e3, Math.max(2e3, intervalMs * 2));
|
|
12971
|
+
return `seo crawl ${config.url}${scope}${sitemap} --max-pages ${config.maxPages} --concurrency 1 --fetch-interval-cap 1 --fetch-interval-ms ${slowMs}`;
|
|
12972
|
+
}
|
|
12973
|
+
/** Set when the sitemaps list more URLs than the page limit let the crawl read. */
|
|
12974
|
+
function sitemapLimitCaveatInput(config, discovery, defaulted) {
|
|
12975
|
+
if (!discovery || discovery.urlsReturned < config.maxPages || !discovery.roots.some((root) => root.possiblyTruncated)) return;
|
|
12976
|
+
return {
|
|
12977
|
+
maxPages: config.maxPages,
|
|
12978
|
+
defaulted,
|
|
12979
|
+
ceiling: MAX_CRAWL_PAGES,
|
|
12980
|
+
health: config.strategy === "health",
|
|
12981
|
+
url: config.url
|
|
13218
12982
|
};
|
|
13219
12983
|
}
|
|
12984
|
+
function crawlCaveats(input) {
|
|
12985
|
+
const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
|
|
12986
|
+
if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
|
|
12987
|
+
if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
|
|
12988
|
+
if (input.rateLimit) {
|
|
12989
|
+
const { readPages, skippedUrls, rerunCommand } = input.rateLimit;
|
|
12990
|
+
caveats.push(`${CRAWL_COVERAGE_CAVEAT_PREFIX} read ${readPages} of ${readPages + skippedUrls} pages because the origin kept rate limiting the crawl (HTTP 429) after it backed off. Rerun slower: ${rerunCommand}`);
|
|
12991
|
+
}
|
|
12992
|
+
if (input.sitemapLimit) {
|
|
12993
|
+
const { maxPages, defaulted, ceiling, health, url } = input.sitemapLimit;
|
|
12994
|
+
caveats.push(`Checked only the first ${maxPages} sitemap URLs because the page limit is ${maxPages}${defaulted ? " by default" : ""}, and the sitemaps list more. Raise --max-pages, up to ${ceiling.toLocaleString("en-US")}, to check more: seo crawl ${url}${health ? " --health" : ""} --max-pages ${ceiling}`);
|
|
12995
|
+
}
|
|
12996
|
+
if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
|
|
12997
|
+
const renderingFallbackUrls = input.renderingFallbackUrls ?? 0;
|
|
12998
|
+
if (renderingFallbackUrls > 0) caveats.push(`JavaScript rendering could not run for ${renderingFallbackUrls} page${renderingFallbackUrls === 1 ? "" : "s"}, so ${renderingFallbackUrls === 1 ? "it was" : "they were"} audited from raw HTML. Client-rendered content, metadata and links may be missing. Install Chromium or set SEO_BROWSER_EXECUTABLE_PATH, then rerun.`);
|
|
12999
|
+
const queryVariantSkippedUrls = input.queryVariantSkippedUrls ?? 0;
|
|
13000
|
+
if (queryVariantSkippedUrls > 0) caveats.push(`Left ${queryVariantSkippedUrls} query-string variant${queryVariantSkippedUrls === 1 ? "" : "s"} unqueued after 5 variants of the same path, so tracking parameters do not fill the crawl.`);
|
|
13001
|
+
for (const issue of input.reachIssues ?? []) if (issue.detail) caveats.push(issue.detail);
|
|
13002
|
+
return caveats;
|
|
13003
|
+
}
|
|
13004
|
+
/**
|
|
13005
|
+
* A site crawl that found no internal links on the start page and no sitemap
|
|
13006
|
+
* URLs checked one page, so its other findings cannot stand for the site.
|
|
13007
|
+
*/
|
|
13008
|
+
function crawlReachIssues(input) {
|
|
13009
|
+
const [page] = input.pages;
|
|
13010
|
+
if (input.crawledPages !== 1 || input.pages.length !== 1 || !page || page.outgoingInternalCount > 0 || input.sitemapUrlsReturned > 0) return [];
|
|
13011
|
+
const rule = explainRule("crawl_single_page_reach");
|
|
13012
|
+
if (!rule) throw new Error("Missing rule guidance for crawl_single_page_reach");
|
|
13013
|
+
return [{
|
|
13014
|
+
ruleId: "crawl_single_page_reach",
|
|
13015
|
+
title: rule.title,
|
|
13016
|
+
category: rule.category,
|
|
13017
|
+
severity: rule.defaultSeverity,
|
|
13018
|
+
url: page.url,
|
|
13019
|
+
detail: page.usedJs ? "The rendered start page had 0 internal links and no sitemap returned URLs, so the crawl checked only this page. List the pages to check with --mode list." : "The start page had 0 internal links in its HTML and no sitemap returned URLs, so the crawl checked only this page. It probably builds its links with JavaScript. Rerun with --rendering on, or list the pages with --mode list.",
|
|
13020
|
+
evidence: {
|
|
13021
|
+
startUrl: input.startUrl,
|
|
13022
|
+
crawledPages: 1,
|
|
13023
|
+
outgoingInternalCount: 0,
|
|
13024
|
+
sitemapUrlsReturned: 0,
|
|
13025
|
+
rendered: Boolean(page.usedJs)
|
|
13026
|
+
}
|
|
13027
|
+
}];
|
|
13028
|
+
}
|
|
13029
|
+
// packages/core/src/analyze/crawler/top-fixes.ts
|
|
13030
|
+
const SEVERITY_SCORE = {
|
|
13031
|
+
high: 5e3,
|
|
13032
|
+
medium: 1200,
|
|
13033
|
+
low: 100
|
|
13034
|
+
};
|
|
13035
|
+
const EFFORT_BY_CATEGORY = {
|
|
13036
|
+
canonical: "medium",
|
|
13037
|
+
content: "high",
|
|
13038
|
+
agents: "low",
|
|
13039
|
+
headings: "low",
|
|
13040
|
+
images: "medium",
|
|
13041
|
+
indexability: "medium",
|
|
13042
|
+
international: "low",
|
|
13043
|
+
metadata: "low",
|
|
13044
|
+
mobile: "low",
|
|
13045
|
+
response: "medium",
|
|
13046
|
+
security: "medium",
|
|
13047
|
+
social: "low",
|
|
13048
|
+
"structured-data": "medium"
|
|
13049
|
+
};
|
|
13050
|
+
const EFFORT_SCORE = {
|
|
13051
|
+
low: 60,
|
|
13052
|
+
medium: 30,
|
|
13053
|
+
high: 0
|
|
13054
|
+
};
|
|
13055
|
+
const AFFECTED_URL_WEIGHT = {
|
|
13056
|
+
high: 25,
|
|
13057
|
+
medium: 15,
|
|
13058
|
+
low: 2
|
|
13059
|
+
};
|
|
13060
|
+
const VALUE_WEIGHT = {
|
|
13061
|
+
high: 1,
|
|
13062
|
+
medium: .75,
|
|
13063
|
+
low: .15
|
|
13064
|
+
};
|
|
13065
|
+
const CRAWL_REACH_RULES = /* @__PURE__ */ new Set(["crawl_single_page_reach"]);
|
|
13066
|
+
function reachRank(ruleId) {
|
|
13067
|
+
return CRAWL_REACH_RULES.has(ruleId) ? 0 : 1;
|
|
13068
|
+
}
|
|
13069
|
+
function matchesPattern(pattern, value) {
|
|
13070
|
+
if (!pattern.includes("*")) return value.includes(pattern);
|
|
13071
|
+
const parts = pattern.split("*");
|
|
13072
|
+
let position = 0;
|
|
13073
|
+
for (const part of parts) {
|
|
13074
|
+
if (!part) continue;
|
|
13075
|
+
const found = value.slice(position).indexOf(part);
|
|
13076
|
+
if (found === -1) return false;
|
|
13077
|
+
position += found + part.length;
|
|
13078
|
+
}
|
|
13079
|
+
return true;
|
|
13080
|
+
}
|
|
13081
|
+
function filteredIssues(report, filters) {
|
|
13082
|
+
return report.issues.filter((issue) => {
|
|
13083
|
+
if (filters.category && issue.category !== filters.category) return false;
|
|
13084
|
+
if (filters.severity && issue.severity !== filters.severity) return false;
|
|
13085
|
+
if (filters.url && !matchesPattern(filters.url, issue.url)) return false;
|
|
13086
|
+
if ((filters.recommendation ?? "fix") !== recommendationForRule(issue.ruleId)) return false;
|
|
13087
|
+
return true;
|
|
13088
|
+
});
|
|
13089
|
+
}
|
|
13090
|
+
function groupIssues(issues) {
|
|
13091
|
+
const groups = /* @__PURE__ */ new Map();
|
|
13092
|
+
for (const issue of issues) {
|
|
13093
|
+
const existing = groups.get(issue.ruleId) ?? {
|
|
13094
|
+
ruleId: issue.ruleId,
|
|
13095
|
+
title: issue.title,
|
|
13096
|
+
category: issue.category,
|
|
13097
|
+
severity: issue.severity,
|
|
13098
|
+
count: 0,
|
|
13099
|
+
sampleUrls: []
|
|
13100
|
+
};
|
|
13101
|
+
existing.count += 1;
|
|
13102
|
+
if (existing.sampleUrls.length < 10) existing.sampleUrls.push(issue.url);
|
|
13103
|
+
groups.set(issue.ruleId, existing);
|
|
13104
|
+
}
|
|
13105
|
+
return [...groups.values()];
|
|
13106
|
+
}
|
|
13107
|
+
function searchValueForGroup(report, urls) {
|
|
13108
|
+
const wanted = new Set(urls);
|
|
13109
|
+
const pages = report.pages.filter((page) => wanted.has(page.url) || wanted.has(page.finalUrl));
|
|
13110
|
+
const metrics = pages.map((page) => page.searchMetrics).filter((item) => Boolean(item));
|
|
13111
|
+
const clicks = metrics.reduce((sum, item) => sum + item.clicks, 0);
|
|
13112
|
+
const impressions = metrics.reduce((sum, item) => sum + item.impressions, 0);
|
|
13113
|
+
const sessions = pages.reduce((sum, page) => sum + (page.analytics?.sessions ?? 0), 0);
|
|
13114
|
+
const observedMetrics = new Set(report.dataSources?.analytics.observedMetrics ?? [
|
|
13115
|
+
"sessions",
|
|
13116
|
+
"totalUsers",
|
|
13117
|
+
"conversions"
|
|
13118
|
+
]);
|
|
13119
|
+
const totalUsers = observedMetrics.has("totalUsers") ? pages.reduce((sum, page) => sum + (page.analytics?.totalUsers ?? 0), 0) : void 0;
|
|
13120
|
+
const conversions = observedMetrics.has("conversions") ? pages.reduce((sum, page) => sum + (page.analytics?.conversions ?? 0), 0) : void 0;
|
|
13121
|
+
const avgPosition = metrics.length ? metrics.reduce((sum, item) => sum + item.position, 0) / metrics.length : void 0;
|
|
13122
|
+
return {
|
|
13123
|
+
searchVisibleUrls: metrics.length,
|
|
13124
|
+
clicks,
|
|
13125
|
+
impressions,
|
|
13126
|
+
sessions,
|
|
13127
|
+
totalUsers,
|
|
13128
|
+
conversions,
|
|
13129
|
+
avgPosition
|
|
13130
|
+
};
|
|
13131
|
+
}
|
|
13132
|
+
function whyThisRanks(input, recommendation, searchStatus, analyticsProvider) {
|
|
13133
|
+
const visibility = input.searchVisibleUrls ? `${input.searchVisibleUrls} affected URLs have GSC visibility (${input.clicks} clicks, ${input.impressions} impressions).` : searchStatus && !["joined", "none"].includes(searchStatus) ? `No joined GSC visibility is available for these affected URLs; Search Console evidence is ${searchStatus}.` : "No affected URL has joined GSC visibility yet.";
|
|
13134
|
+
const analyticsDetails = analyticsProvider ? analyticsProviderDetails(analyticsProvider) : analyticsProviderDetails("google");
|
|
13135
|
+
const analyticsLabel = analyticsDetails.label;
|
|
13136
|
+
const sessionLabel = analyticsDetails.landingMetricLabel;
|
|
13137
|
+
return `${visibility}${input.sessions ? ` ${analyticsLabel} adds ${input.sessions} ${sessionLabel}${input.conversions === void 0 ? "" : ` and ${input.conversions} conversions`} from affected pages.` : ""} ${recommendation === "review" ? "This is a review observation. Confirm it before scheduling implementation work." : `This ranks as a fix. Severity is ${input.severity}; ${input.affectedUrls} URL${input.affectedUrls === 1 ? " is" : "s are"} affected; estimated effort is ${input.effort}.`}`;
|
|
13138
|
+
}
|
|
13139
|
+
function verificationCommand(report, severity) {
|
|
13140
|
+
return `seo crawl ${report.config.url} --severity ${severity} --max-pages ${Math.min(report.config.maxPages, 100)}`;
|
|
13141
|
+
}
|
|
13142
|
+
function topFixes(report, filters = {}) {
|
|
13143
|
+
return groupIssues(filteredIssues(report, filters)).map((group) => {
|
|
13144
|
+
const search = searchValueForGroup(report, group.sampleUrls);
|
|
13145
|
+
const effort = EFFORT_BY_CATEGORY[group.category] ?? "medium";
|
|
13146
|
+
const scoreFactors = {
|
|
13147
|
+
severity: SEVERITY_SCORE[group.severity],
|
|
13148
|
+
affectedUrls: group.count,
|
|
13149
|
+
searchVisibleUrls: search.searchVisibleUrls,
|
|
13150
|
+
clicks: search.clicks,
|
|
13151
|
+
impressions: search.impressions,
|
|
13152
|
+
sessions: search.sessions,
|
|
13153
|
+
totalUsers: search.totalUsers,
|
|
13154
|
+
conversions: search.conversions,
|
|
13155
|
+
avgPosition: search.avgPosition,
|
|
13156
|
+
effort,
|
|
13157
|
+
effortScore: EFFORT_SCORE[effort]
|
|
13158
|
+
};
|
|
13159
|
+
const valueWeight = VALUE_WEIGHT[group.severity];
|
|
13160
|
+
const score = scoreFactors.severity + scoreFactors.affectedUrls * AFFECTED_URL_WEIGHT[group.severity] + scoreFactors.searchVisibleUrls * 100 * valueWeight + scoreFactors.clicks * 20 * valueWeight + Math.min(scoreFactors.impressions, 1e4) / 25 * valueWeight + scoreFactors.sessions * 2 * valueWeight + (scoreFactors.conversions ?? 0) * 100 * valueWeight + scoreFactors.effortScore;
|
|
13161
|
+
const rule = explainRule(group.ruleId);
|
|
13162
|
+
const recommendation = recommendationForRule(group.ruleId);
|
|
13163
|
+
return {
|
|
13164
|
+
...group,
|
|
13165
|
+
recommendation,
|
|
13166
|
+
score: Math.round(score),
|
|
13167
|
+
scoreFactors,
|
|
13168
|
+
whyThisRanks: whyThisRanks(scoreFactors, recommendation, report.dataSources?.searchConsole.status, report.dataSources?.analytics.provider),
|
|
13169
|
+
howToFix: rule?.howToFix ?? "",
|
|
13170
|
+
howToVerify: rule?.howToVerify ?? "",
|
|
13171
|
+
verification: {
|
|
13172
|
+
command: verificationCommand(report, group.severity),
|
|
13173
|
+
expected: rule?.howToVerify ?? ""
|
|
13174
|
+
}
|
|
13175
|
+
};
|
|
13176
|
+
}).sort((a, b) => reachRank(a.ruleId) - reachRank(b.ruleId) || b.score - a.score || SEVERITY_SCORE[b.severity] - SEVERITY_SCORE[a.severity] || b.count - a.count || a.ruleId.localeCompare(b.ruleId)).slice(0, filters.limit ?? 10);
|
|
13177
|
+
}
|
|
13178
|
+
function reviewObservations(report, filters = {}) {
|
|
13179
|
+
return topFixes(report, {
|
|
13180
|
+
...filters,
|
|
13181
|
+
recommendation: "review"
|
|
13182
|
+
});
|
|
13183
|
+
}
|
|
13184
|
+
// packages/core/src/analyze/crawler/export.ts
|
|
13185
|
+
const issueHeaders = [
|
|
13186
|
+
"rule_id",
|
|
13187
|
+
"title",
|
|
13188
|
+
"category",
|
|
13189
|
+
"severity",
|
|
13190
|
+
"url",
|
|
13191
|
+
"detail",
|
|
13192
|
+
"clicks",
|
|
13193
|
+
"impressions",
|
|
13194
|
+
"position"
|
|
13195
|
+
];
|
|
13196
|
+
const pageHeaders = [
|
|
13197
|
+
"url",
|
|
13198
|
+
"final_url",
|
|
13199
|
+
"status",
|
|
13200
|
+
"indexable",
|
|
13201
|
+
"title",
|
|
13202
|
+
"meta_description",
|
|
13203
|
+
"canonical",
|
|
13204
|
+
"word_count",
|
|
13205
|
+
"content_extractor",
|
|
13206
|
+
"content_extractor_type",
|
|
13207
|
+
"content_extraction_fallback",
|
|
13208
|
+
"word_count_source",
|
|
13209
|
+
"internal_links",
|
|
13210
|
+
"internal_inlinks",
|
|
13211
|
+
"internal_authority_score",
|
|
13212
|
+
"external_links",
|
|
13213
|
+
"images_total",
|
|
13214
|
+
"images_missing_alt",
|
|
13215
|
+
"oversized_image_candidates",
|
|
13216
|
+
"schema_types",
|
|
13217
|
+
"clicks",
|
|
13218
|
+
"impressions",
|
|
13219
|
+
"sessions"
|
|
13220
|
+
];
|
|
13221
|
+
function renderCrawlCsv(report) {
|
|
13222
|
+
return renderCsv(report.issues.map((issue) => ({
|
|
13223
|
+
rule_id: issue.ruleId,
|
|
13224
|
+
title: issue.title,
|
|
13225
|
+
category: issue.category,
|
|
13226
|
+
severity: issue.severity,
|
|
13227
|
+
url: issue.url,
|
|
13228
|
+
detail: issue.detail,
|
|
13229
|
+
clicks: issue.searchMetrics?.clicks,
|
|
13230
|
+
impressions: issue.searchMetrics?.impressions,
|
|
13231
|
+
position: issue.searchMetrics?.position
|
|
13232
|
+
})), issueHeaders);
|
|
13233
|
+
}
|
|
13234
|
+
function renderCrawlPagesCsv(report) {
|
|
13235
|
+
return renderCsv(report.pages.map((page) => ({
|
|
13236
|
+
url: page.url,
|
|
13237
|
+
final_url: page.finalUrl,
|
|
13238
|
+
status: page.status,
|
|
13239
|
+
indexable: page.indexable,
|
|
13240
|
+
title: page.title,
|
|
13241
|
+
meta_description: page.metaDescription,
|
|
13242
|
+
canonical: page.canonical,
|
|
13243
|
+
word_count: page.wordCount,
|
|
13244
|
+
content_extractor: page.contentExtraction?.used,
|
|
13245
|
+
content_extractor_type: page.contentExtraction?.extractorType,
|
|
13246
|
+
content_extraction_fallback: page.contentExtraction?.fallback,
|
|
13247
|
+
word_count_source: page.contentExtraction?.wordCountSource,
|
|
13248
|
+
internal_links: page.outgoingInternalCount,
|
|
13249
|
+
internal_inlinks: page.internalInlinkCount,
|
|
13250
|
+
internal_authority_score: page.internalLinkAuthorityScore,
|
|
13251
|
+
external_links: page.outgoingExternalCount,
|
|
13252
|
+
images_total: page.imagesTotal,
|
|
13253
|
+
images_missing_alt: page.imagesMissingAlt,
|
|
13254
|
+
oversized_image_candidates: page.oversizedImageCandidates?.length ?? 0,
|
|
13255
|
+
schema_types: page.schemaTypes?.join("|"),
|
|
13256
|
+
clicks: page.searchMetrics?.clicks,
|
|
13257
|
+
impressions: page.searchMetrics?.impressions,
|
|
13258
|
+
sessions: page.analytics?.sessions
|
|
13259
|
+
})), pageHeaders);
|
|
13260
|
+
}
|
|
13261
|
+
/** The caveat a report must lead with when the crawl read too little. */
|
|
13262
|
+
function crawlCoverageCaveat(report) {
|
|
13263
|
+
return report.caveats.find((caveat) => caveat.startsWith(CRAWL_COVERAGE_CAVEAT_PREFIX));
|
|
13264
|
+
}
|
|
13265
|
+
function renderCrawlPretty(report, fixes = topFixes(report)) {
|
|
13266
|
+
const reviews = reviewObservations(report);
|
|
13267
|
+
const requestSummary = report.requestEvidenceStatus === "available" ? `${report.requests.length} requests` : report.requestEvidenceStatus === "partial" ? `${report.requests.length} observed requests; some started requests were still in flight when the crawl stopped` : "request evidence unavailable for this legacy report";
|
|
13268
|
+
const coverage = crawlCoverageCaveat(report);
|
|
13269
|
+
const lines = [
|
|
13270
|
+
...coverage ? [coverage, ""] : [],
|
|
13271
|
+
`Crawl report for ${report.config.url}`,
|
|
13272
|
+
"",
|
|
13273
|
+
`Status: ${report.status}`,
|
|
13274
|
+
`Strategy: ${report.config.strategy === "health" ? "sitemap health pass" : "full audit"}`,
|
|
13275
|
+
`Crawler: ${report.access.crawler.userAgent}`,
|
|
13276
|
+
`Documents: ${report.summary.totalPages} retained; ${requestSummary}, ${report.summary.discoveredUrls} URLs discovered, ${report.summary.failedUrls} failed, ${report.summary.skippedUrls} skipped`,
|
|
13277
|
+
`Issues: ${report.issues.length} (${report.summary.highIssues} high, ${report.summary.mediumIssues} medium, ${report.summary.lowIssues} low)`
|
|
13278
|
+
];
|
|
13279
|
+
if (report.externalLinkVerification) {
|
|
13280
|
+
const external = report.externalLinkVerification;
|
|
13281
|
+
lines.push(`External links: ${external.fetchedUrls} fetched, ${external.failedUrls} request failure${external.failedUrls === 1 ? "" : "s"}, ${external.deferredUrls} deferred (${external.dataStatus})`);
|
|
13282
|
+
}
|
|
13283
|
+
if (report.access.blockedRequests > 0) {
|
|
13284
|
+
lines.push("", `Access blocks: ${report.access.blockedRequests}`);
|
|
13285
|
+
for (const sample of report.access.samples.slice(0, 3)) {
|
|
13286
|
+
const requestId = sample.evidence.requestId ? `, request ID ${sample.evidence.requestId}` : "";
|
|
13287
|
+
lines.push(`- ${sample.url}: ${sample.evidence.guidance.summary} (${sample.evidence.provider}${requestId})`, ` Action: ${sample.evidence.guidance.recommendedAction}`, ` Security: ${sample.evidence.guidance.securityNote}`);
|
|
13288
|
+
if (sample.evidence.guidance.documentationUrl) lines.push(` Provider docs: ${sample.evidence.guidance.documentationUrl}`);
|
|
13289
|
+
}
|
|
13290
|
+
}
|
|
13291
|
+
if (fixes.length) {
|
|
13292
|
+
lines.push("", "Prioritised fixes");
|
|
13293
|
+
for (const fix of fixes.slice(0, 5)) {
|
|
13294
|
+
lines.push(`- ${fix.title} (${fix.severity}, ${fix.count} URL${fix.count === 1 ? "" : "s"}): ${fix.howToFix}`);
|
|
13295
|
+
if (fix.sampleUrls[0]) lines.push(` First URL: ${fix.sampleUrls[0]}`);
|
|
13296
|
+
}
|
|
13297
|
+
} else lines.push("", "No prioritised fixes found.");
|
|
13298
|
+
if (reviews.length) {
|
|
13299
|
+
lines.push("", "Review observations (check before scheduling work)");
|
|
13300
|
+
for (const observation of reviews.slice(0, 5)) {
|
|
13301
|
+
lines.push(`- ${observation.title} (${observation.severity}, ${observation.count} URL${observation.count === 1 ? "" : "s"}): ${observation.howToFix}`);
|
|
13302
|
+
if (observation.sampleUrls[0]) lines.push(` First URL: ${observation.sampleUrls[0]}`);
|
|
13303
|
+
}
|
|
13304
|
+
}
|
|
13305
|
+
lines.push("", "Next commands", report.config.strategy === "health" ? `- seo crawl ${report.config.url} --max-pages ${report.config.maxPages} --json` : `- seo crawl ${report.config.url} --json`);
|
|
13306
|
+
for (const command of [...new Set(fixes.slice(0, 3).map((fix) => fix.verification.command))]) lines.push(`- ${command}`);
|
|
13307
|
+
if (report.warnings.length) lines.push("", "Warnings", ...report.warnings.map((item) => `- ${item}`));
|
|
13308
|
+
if (report.caveats.length) lines.push("", "Caveats", ...report.caveats.map((item) => `- ${item}`));
|
|
13309
|
+
return `${lines.join("\n")}\n`;
|
|
13310
|
+
}
|
|
13311
|
+
function renderCrawlJunit(report) {
|
|
13312
|
+
const issuesByUrl = /* @__PURE__ */ new Map();
|
|
13313
|
+
for (const issue of report.issues) {
|
|
13314
|
+
const existing = issuesByUrl.get(issue.url) ?? [];
|
|
13315
|
+
existing.push(issue);
|
|
13316
|
+
issuesByUrl.set(issue.url, existing);
|
|
13317
|
+
}
|
|
13318
|
+
const sitemapCases = (report.sitemapDiscovery?.roots ?? []).flatMap((root) => {
|
|
13319
|
+
return (root.documents.length ? root.documents : [{
|
|
13320
|
+
url: root.url,
|
|
13321
|
+
dataStatus: root.dataStatus,
|
|
13322
|
+
compression: "none",
|
|
13323
|
+
warning: root.warnings.join(" ") || void 0
|
|
13324
|
+
}]).map((document) => {
|
|
13325
|
+
const failures = [
|
|
13326
|
+
...document.dataStatus !== "complete" ? [document.warning ?? `Sitemap evidence is ${document.dataStatus}.`] : [],
|
|
13327
|
+
...document.redirected ? [`Sitemap redirected to ${document.finalUrl ?? "an unknown URL"}.`] : [],
|
|
13328
|
+
...document.status !== void 0 && (document.status < 200 || document.status >= 300) ? [`Sitemap returned HTTP ${document.status}.`] : []
|
|
13329
|
+
];
|
|
13330
|
+
const output = [
|
|
13331
|
+
`dataStatus=${document.dataStatus}`,
|
|
13332
|
+
...document.status !== void 0 ? [`status=${document.status}`] : [],
|
|
13333
|
+
...document.finalUrl ? [`finalUrl=${document.finalUrl}`] : [],
|
|
13334
|
+
...document.root ? [`root=${document.root}`] : [],
|
|
13335
|
+
...document.warning ? [`warning=${document.warning}`] : []
|
|
13336
|
+
].join("\n");
|
|
13337
|
+
return {
|
|
13338
|
+
xml: [
|
|
13339
|
+
` <testcase classname="seo.sitemap-document" name="${escapeXml(document.url)}" time="0.000">`,
|
|
13340
|
+
...failures.length ? [` <failure message="${escapeXml(failures[0] ?? "Sitemap check failed")}">${escapeXml(failures.join("\n"))}</failure>`] : [],
|
|
13341
|
+
` <system-out>${escapeXml(output)}</system-out>`,
|
|
13342
|
+
" </testcase>"
|
|
13343
|
+
].join("\n"),
|
|
13344
|
+
failed: failures.length > 0
|
|
13345
|
+
};
|
|
13346
|
+
});
|
|
13347
|
+
});
|
|
13348
|
+
const requestCases = report.requests.map((request) => {
|
|
13349
|
+
const failures = [
|
|
13350
|
+
...(issuesByUrl.get(request.requestedUrl) ?? []).map((issue) => `${issue.severity}: ${issue.title}${issue.detail ? `: ${issue.detail}` : ""}`),
|
|
13351
|
+
...request.outcome === "failure" ? [`${request.failureKind}: ${request.error}`] : [],
|
|
13352
|
+
...request.outcome === "skipped" ? [request.reason === "origin-backpressure" ? request.error : `${request.reason}: ${request.robotsTxt.matchedLine ?? request.robotsTxt.availability}`] : []
|
|
13353
|
+
];
|
|
13354
|
+
const output = request.outcome === "response" ? [
|
|
13355
|
+
`status=${request.status}`,
|
|
13356
|
+
`finalUrl=${request.finalUrl}`,
|
|
13357
|
+
...request.robotsTxt ? [
|
|
13358
|
+
`robotsUrl=${request.robotsTxt.url}`,
|
|
13359
|
+
`robotsAllowed=${request.robotsTxt.allowed ?? "unknown"}`,
|
|
13360
|
+
`robotsAvailability=${request.robotsTxt.availability}`,
|
|
13361
|
+
...request.robotsTxt.status !== void 0 ? [`robotsStatus=${request.robotsTxt.status}`] : [],
|
|
13362
|
+
...request.robotsTxt.matchedLine ? [`robotsMatchedLine=${request.robotsTxt.matchedLine}`] : [],
|
|
13363
|
+
...request.robotsTxt.error ? [`robotsError=${request.robotsTxt.error}`] : []
|
|
13364
|
+
] : [],
|
|
13365
|
+
...request.accessBlock ? [
|
|
13366
|
+
`accessProvider=${request.accessBlock.provider}`,
|
|
13367
|
+
`crawler=${request.accessBlock.crawler.userAgent}`,
|
|
13368
|
+
...request.accessBlock.requestId ? [`requestId=${request.accessBlock.requestId}`] : []
|
|
13369
|
+
] : []
|
|
13370
|
+
].join("\n") : request.outcome === "failure" ? request.error : request.reason === "origin-backpressure" ? `reason=${request.reason}\nerror=${request.error}` : [
|
|
13371
|
+
`reason=${request.reason}`,
|
|
13372
|
+
`robotsUrl=${request.robotsTxt.url}`,
|
|
13373
|
+
`robotsAllowed=${request.robotsTxt.allowed ?? "unknown"}`,
|
|
13374
|
+
`robotsAvailability=${request.robotsTxt.availability}`,
|
|
13375
|
+
...request.robotsTxt.status !== void 0 ? [`robotsStatus=${request.robotsTxt.status}`] : [],
|
|
13376
|
+
...request.robotsTxt.matchedLine ? [`robotsMatchedLine=${request.robotsTxt.matchedLine}`] : [],
|
|
13377
|
+
...request.robotsTxt.error ? [`robotsError=${request.robotsTxt.error}`] : []
|
|
13378
|
+
].join("\n");
|
|
13379
|
+
const duration = ((request.durationMs ?? 0) / 1e3).toFixed(3);
|
|
13380
|
+
return {
|
|
13381
|
+
xml: [
|
|
13382
|
+
` <testcase classname="seo.sitemap-health" name="${escapeXml(request.requestedUrl)}" time="${duration}">`,
|
|
13383
|
+
...failures.length ? [` <failure message="${escapeXml(failures[0] ?? "Health check failed")}">${escapeXml(failures.join("\n"))}</failure>`] : [],
|
|
13384
|
+
` <system-out>${escapeXml(output)}</system-out>`,
|
|
13385
|
+
" </testcase>"
|
|
13386
|
+
].join("\n"),
|
|
13387
|
+
failed: failures.length > 0
|
|
13388
|
+
};
|
|
13389
|
+
});
|
|
13390
|
+
const cases = [...sitemapCases, ...requestCases];
|
|
13391
|
+
const failures = cases.filter((item) => item.failed).length;
|
|
13392
|
+
const elapsed = report.requests.reduce((sum, request) => sum + (request.durationMs ?? 0), 0);
|
|
13393
|
+
return [
|
|
13394
|
+
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>",
|
|
13395
|
+
`<testsuite name="seo sitemap health" tests="${cases.length}" failures="${failures}" errors="0" skipped="0" time="${(elapsed / 1e3).toFixed(3)}">`,
|
|
13396
|
+
" <properties>",
|
|
13397
|
+
` <property name="report.id" value="${escapeXml(report.id)}"/>`,
|
|
13398
|
+
` <property name="report.status" value="${escapeXml(report.status)}"/>`,
|
|
13399
|
+
` <property name="crawl.strategy" value="${escapeXml(report.config.strategy)}"/>`,
|
|
13400
|
+
` <property name="crawler.userAgent" value="${escapeXml(report.access.crawler.userAgent)}"/>`,
|
|
13401
|
+
" </properties>",
|
|
13402
|
+
...cases.map((item) => item.xml),
|
|
13403
|
+
"</testsuite>",
|
|
13404
|
+
""
|
|
13405
|
+
].join("\n");
|
|
13406
|
+
}
|
|
13407
|
+
function renderCrawlHtml(report, fixes = topFixes(report)) {
|
|
13408
|
+
const reviews = reviewObservations(report);
|
|
13409
|
+
const issueRows = report.issueGroups.map((group) => {
|
|
13410
|
+
const rule = explainRule(group.ruleId);
|
|
13411
|
+
return `<tr><td>${escapeHtml(group.severity)}</td><td>${escapeHtml(group.title)}</td><td>${group.count}</td><td>${escapeHtml(rule?.howToFix ?? "")}</td><td>${escapeHtml(group.sampleUrls[0] ?? "")}</td></tr>`;
|
|
13412
|
+
}).join("\n");
|
|
13413
|
+
const fixItems = fixes.slice(0, 10).map((fix) => `<li><strong>${escapeHtml(fix.title)}</strong><br>${escapeHtml(fix.howToFix)}<br><span>Verify: ${escapeHtml(fix.howToVerify)}</span><br><code>${escapeHtml(fix.verification.command)}</code></li>`).join("\n");
|
|
13414
|
+
const reviewItems = reviews.slice(0, 10).map((observation) => `<li><strong>${escapeHtml(observation.title)}</strong><br>${escapeHtml(observation.howToFix)}<br><span>Check before scheduling work. ${escapeHtml(observation.howToVerify)}</span></li>`).join("\n");
|
|
13415
|
+
const caveatItems = report.caveats.map((caveat) => `<li>${escapeHtml(caveat)}</li>`).join("\n");
|
|
13416
|
+
const warningItems = report.warnings.map((warning) => `<li>${escapeHtml(warning)}</li>`).join("\n");
|
|
13417
|
+
const accessItems = report.access.samples.map((sample) => `<li><strong>${escapeHtml(sample.url)}</strong><br>${escapeHtml(sample.evidence.guidance.summary)}<br>${escapeHtml(sample.evidence.guidance.recommendedAction)}<br><span>${escapeHtml(sample.evidence.guidance.securityNote)}</span></li>`).join("\n");
|
|
13418
|
+
return `<!doctype html>
|
|
13419
|
+
<html lang="en">
|
|
13420
|
+
<head>
|
|
13421
|
+
<meta charset="utf-8">
|
|
13422
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
13423
|
+
<title>Crawl report for ${escapeHtml(report.config.url)}</title>
|
|
13424
|
+
<style>
|
|
13425
|
+
body { color: #17202a; font: 15px/1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; margin: 32px; max-width: 1120px; }
|
|
13426
|
+
h1, h2 { line-height: 1.2; }
|
|
13427
|
+
.summary { display: grid; gap: 12px; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); margin: 24px 0; }
|
|
13428
|
+
.metric { border: 1px solid #d8dee4; border-radius: 8px; padding: 12px; }
|
|
13429
|
+
.metric strong { display: block; font-size: 24px; }
|
|
13430
|
+
table { border-collapse: collapse; width: 100%; }
|
|
13431
|
+
th, td { border-bottom: 1px solid #d8dee4; padding: 10px; text-align: left; vertical-align: top; }
|
|
13432
|
+
th { background: #f6f8fa; }
|
|
13433
|
+
li { margin: 0 0 12px; }
|
|
13434
|
+
span { color: #57606a; }
|
|
13435
|
+
</style>
|
|
13436
|
+
</head>
|
|
13437
|
+
<body>
|
|
13438
|
+
<h1>Crawl report</h1>
|
|
13439
|
+
<p>${escapeHtml(report.config.url)}</p>
|
|
13440
|
+
<p>Strategy: ${escapeHtml(report.config.strategy === "health" ? "sitemap health pass" : "full audit")}<br>Crawler: <code>${escapeHtml(report.access.crawler.userAgent)}</code></p>
|
|
13441
|
+
<section class="summary">
|
|
13442
|
+
<div class="metric"><span>Status</span><strong>${escapeHtml(report.status)}</strong></div>
|
|
13443
|
+
<div class="metric"><span>Documents</span><strong>${report.summary.totalPages}</strong></div>
|
|
13444
|
+
<div class="metric"><span>Requests</span><strong>${report.requestEvidenceStatus === "unavailable" ? "n/a" : report.requests.length}${report.requestEvidenceStatus === "partial" ? " (partial)" : ""}</strong></div>
|
|
13445
|
+
<div class="metric"><span>Discovered</span><strong>${report.summary.discoveredUrls}</strong></div>
|
|
13446
|
+
<div class="metric"><span>Queued</span><strong>${report.summary.queuedUrls}</strong></div>
|
|
13447
|
+
<div class="metric"><span>Skipped</span><strong>${report.summary.skippedUrls}</strong></div>
|
|
13448
|
+
<div class="metric"><span>Failed fetches</span><strong>${report.summary.failedUrls}</strong></div>
|
|
13449
|
+
<div class="metric"><span>Observed internal links</span><strong>${report.summary.observedInternalLinks}</strong></div>
|
|
13450
|
+
<div class="metric"><span>Issues</span><strong>${report.issues.length}</strong></div>
|
|
13451
|
+
<div class="metric"><span>High</span><strong>${report.summary.highIssues}</strong></div>
|
|
13452
|
+
</section>
|
|
13453
|
+
${accessItems ? `<h2>Access blocks</h2><ul>${accessItems}</ul>` : ""}
|
|
13454
|
+
<h2>Prioritised fixes</h2>
|
|
13455
|
+
${fixItems ? `<ol>${fixItems}</ol>` : "<p>No prioritised fixes found.</p>"}
|
|
13456
|
+
${reviewItems ? `<h2>Review observations</h2>
|
|
13457
|
+
<p>Check these before scheduling implementation work.</p>
|
|
13458
|
+
<ul>${reviewItems}</ul>` : ""}
|
|
13459
|
+
${report.caveats.length ? `<h2>Caveats</h2>
|
|
13460
|
+
<ul>${caveatItems}</ul>` : ""}
|
|
13461
|
+
${report.warnings.length ? `<h2>Warnings</h2>
|
|
13462
|
+
<ul>${warningItems}</ul>` : ""}
|
|
13463
|
+
<h2>Issue groups</h2>
|
|
13464
|
+
<table>
|
|
13465
|
+
<thead><tr><th>Severity</th><th>Issue</th><th>URLs</th><th>Fix</th><th>Sample URL</th></tr></thead>
|
|
13466
|
+
<tbody>${issueRows}</tbody>
|
|
13467
|
+
</table>
|
|
13468
|
+
</body>
|
|
13469
|
+
</html>
|
|
13470
|
+
`;
|
|
13471
|
+
}
|
|
13472
|
+
function renderCrawlMarkdownTickets(report, fixes = topFixes(report)) {
|
|
13473
|
+
const reviews = reviewObservations(report);
|
|
13474
|
+
const lines = [
|
|
13475
|
+
`# Crawl Implementation Tickets`,
|
|
13476
|
+
"",
|
|
13477
|
+
`Source: ${report.config.url}`,
|
|
13478
|
+
`Report: ${report.id}`,
|
|
13479
|
+
`Status: ${report.status}`,
|
|
13480
|
+
`Strategy: ${report.config.strategy}`,
|
|
13481
|
+
`Crawler: ${report.access.crawler.userAgent}`,
|
|
13482
|
+
""
|
|
13483
|
+
];
|
|
13484
|
+
if (report.caveats.length) lines.push("## Caveats", "", ...report.caveats.map((item) => `- ${item}`), "");
|
|
13485
|
+
if (report.warnings.length) lines.push("## Warnings", "", ...report.warnings.map((item) => `- ${item}`), "");
|
|
13486
|
+
if (report.access.blockedRequests > 0) lines.push("## Access blocks", "", ...report.access.samples.flatMap((sample) => [
|
|
13487
|
+
`- ${sample.url}: ${sample.evidence.guidance.summary}`,
|
|
13488
|
+
` - Action: ${sample.evidence.guidance.recommendedAction}`,
|
|
13489
|
+
` - Security: ${sample.evidence.guidance.securityNote}`
|
|
13490
|
+
]), "");
|
|
13491
|
+
for (const [index, fix] of fixes.entries()) lines.push(`## ${index + 1}. ${fix.title}`, "", `- [ ] Fix ${fix.count} affected URL${fix.count === 1 ? "" : "s"}`, `- Severity: ${fix.severity}`, `- Rule: ${fix.ruleId}`, `- Why this ranks: ${fix.whyThisRanks}`, `- Plain-English fix: ${fix.howToFix}`, `- Verify: ${fix.howToVerify}`, `- Command: ${fix.verification.command}`, "", `Affected URLs:`, ...fix.sampleUrls.slice(0, 10).map((url) => `- ${url}`), "");
|
|
13492
|
+
if (!fixes.length) lines.push("No implementation tickets were generated.");
|
|
13493
|
+
if (reviews.length) lines.push("## Review observations", "", "These are not implementation tickets. Check the evidence before deciding whether to work on them.", "", ...reviews.slice(0, 10).flatMap((observation) => [`- ${observation.title} (${observation.severity}, ${observation.count} URL${observation.count === 1 ? "" : "s"}): ${observation.howToFix}`, ` - First URL: ${observation.sampleUrls[0] ?? "No sample URL recorded"}`]), "");
|
|
13494
|
+
return `${lines.join("\n")}\n`;
|
|
13495
|
+
}
|
|
13496
|
+
function escapeHtml(value) {
|
|
13497
|
+
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll("\"", """).replaceAll("'", "'");
|
|
13498
|
+
}
|
|
13499
|
+
function escapeXml(value) {
|
|
13500
|
+
return escapeHtml(value);
|
|
13501
|
+
}
|
|
13220
13502
|
// packages/core/src/analyze/crawler/site-checks-schema.ts
|
|
13221
13503
|
const crawlSiteChecksSchema = z.object({ soft404: z.object({
|
|
13222
13504
|
status: z.enum([
|
|
@@ -13643,6 +13925,7 @@ const crawlPageSnapshotSchema = z.object({
|
|
|
13643
13925
|
source: z.enum([
|
|
13644
13926
|
"environment",
|
|
13645
13927
|
"playwright-cache",
|
|
13928
|
+
"agent-browser-cache",
|
|
13646
13929
|
"system"
|
|
13647
13930
|
]),
|
|
13648
13931
|
product: z.string(),
|
|
@@ -14346,14 +14629,6 @@ function agentReadyIssues(input) {
|
|
|
14346
14629
|
}
|
|
14347
14630
|
return issues;
|
|
14348
14631
|
}
|
|
14349
|
-
// packages/core/src/analyze/crawler/crawl-caveats.ts
|
|
14350
|
-
function crawlCaveats(input) {
|
|
14351
|
-
const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
|
|
14352
|
-
if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
|
|
14353
|
-
if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
|
|
14354
|
-
if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
|
|
14355
|
-
return caveats;
|
|
14356
|
-
}
|
|
14357
14632
|
// packages/core/src/analyze/crawler/crawl-control.ts
|
|
14358
14633
|
const CRAWL_CANCELLED = Symbol("crawl-cancelled");
|
|
14359
14634
|
function abortController(input) {
|
|
@@ -14395,83 +14670,94 @@ function crawlRssLimitBytes(totalMemoryBytes) {
|
|
|
14395
14670
|
function crawlMemoryPressure(input) {
|
|
14396
14671
|
return input.memoryUsage.heapUsed + input.memoryUsage.external >= crawlMemoryLimitBytes(input.totalMemoryBytes) || input.memoryUsage.rss >= crawlRssLimitBytes(input.totalMemoryBytes);
|
|
14397
14672
|
}
|
|
14398
|
-
// packages/core/src/analyze/crawler/crawl-
|
|
14399
|
-
|
|
14400
|
-
|
|
14673
|
+
// packages/core/src/analyze/crawler/crawl-rate-limit.ts
|
|
14674
|
+
const RETRIES_PER_URL = 3;
|
|
14675
|
+
const BASE_BACKOFF_MS = 2e3;
|
|
14676
|
+
const MAX_BACKOFF_MS = 3e4;
|
|
14677
|
+
const COOLDOWN_BUDGET_MS = 18e4;
|
|
14678
|
+
const LIMITED_INTERVAL_MS = 1e3;
|
|
14679
|
+
const LIMITED_MAX_INTERVAL_MS = 4e3;
|
|
14680
|
+
function rateLimitEvidence(result, status, nowMs) {
|
|
14681
|
+
if (status !== 429 && status !== 503) return void 0;
|
|
14682
|
+
const header = retryAfterMs(result.page?.responseHeaders?.["retry-after"]);
|
|
14683
|
+
const cooldownUntil = Date.parse(result.page?.fetchDiagnostics?.backpressure?.cooldownUntil ?? "");
|
|
14684
|
+
const fromCooldown = Number.isFinite(cooldownUntil) ? Math.max(0, cooldownUntil - nowMs) : void 0;
|
|
14685
|
+
const wait = header ?? fromCooldown;
|
|
14686
|
+
if (status === 503 && wait === void 0) return void 0;
|
|
14687
|
+
return wait === void 0 ? {} : { retryAfterMs: Math.min(wait, MAX_BACKOFF_MS) };
|
|
14401
14688
|
}
|
|
14402
|
-
|
|
14403
|
-
|
|
14404
|
-
|
|
14405
|
-
|
|
14406
|
-
|
|
14407
|
-
|
|
14408
|
-
|
|
14409
|
-
|
|
14410
|
-
|
|
14411
|
-
|
|
14412
|
-
|
|
14413
|
-
|
|
14414
|
-
|
|
14415
|
-
|
|
14416
|
-
this
|
|
14417
|
-
}
|
|
14418
|
-
|
|
14419
|
-
|
|
14420
|
-
|
|
14421
|
-
|
|
14422
|
-
|
|
14423
|
-
|
|
14424
|
-
|
|
14425
|
-
|
|
14426
|
-
|
|
14427
|
-
|
|
14428
|
-
|
|
14429
|
-
|
|
14430
|
-
|
|
14431
|
-
|
|
14432
|
-
|
|
14433
|
-
|
|
14434
|
-
|
|
14435
|
-
|
|
14436
|
-
|
|
14437
|
-
|
|
14438
|
-
|
|
14439
|
-
|
|
14440
|
-
|
|
14441
|
-
|
|
14442
|
-
|
|
14443
|
-
|
|
14444
|
-
|
|
14445
|
-
|
|
14446
|
-
this
|
|
14447
|
-
|
|
14448
|
-
|
|
14449
|
-
|
|
14450
|
-
|
|
14451
|
-
|
|
14452
|
-
|
|
14453
|
-
|
|
14454
|
-
|
|
14455
|
-
|
|
14456
|
-
|
|
14457
|
-
|
|
14689
|
+
/**
|
|
14690
|
+
* Rate-limit state for one crawl. After the first 429 the crawl drops to one
|
|
14691
|
+
* request at a time with a per-host interval and never speeds back up.
|
|
14692
|
+
*/
|
|
14693
|
+
var CrawlRateLimit = class {
|
|
14694
|
+
rate;
|
|
14695
|
+
responses = 0;
|
|
14696
|
+
skippedUrls = 0;
|
|
14697
|
+
pendingMs = 0;
|
|
14698
|
+
spentMs = 0;
|
|
14699
|
+
#level = 0;
|
|
14700
|
+
#budgetSpent = false;
|
|
14701
|
+
#retries = /* @__PURE__ */ new Map();
|
|
14702
|
+
constructor(rate) {
|
|
14703
|
+
this.rate = rate;
|
|
14704
|
+
}
|
|
14705
|
+
get limited() {
|
|
14706
|
+
return this.responses > 0;
|
|
14707
|
+
}
|
|
14708
|
+
get intervalMs() {
|
|
14709
|
+
return this.rate.intervalMs ?? LIMITED_INTERVAL_MS;
|
|
14710
|
+
}
|
|
14711
|
+
/** The cool-down before probes that run after the crawl. */
|
|
14712
|
+
get settleMs() {
|
|
14713
|
+
return Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** this.#level);
|
|
14714
|
+
}
|
|
14715
|
+
/**
|
|
14716
|
+
* `retry` requeues the URL after `pendingMs`. `skip` gives it up, and
|
|
14717
|
+
* `stop` gives up the rest of the queue too once the budget is spent.
|
|
14718
|
+
*/
|
|
14719
|
+
observe(url, result, status, nowMs) {
|
|
14720
|
+
const evidence = rateLimitEvidence(result, status, nowMs);
|
|
14721
|
+
if (!evidence) return void 0;
|
|
14722
|
+
this.responses += 1;
|
|
14723
|
+
if (this.pendingMs === 0) this.#level += 1;
|
|
14724
|
+
const step = 2 ** Math.max(0, this.#level - 1);
|
|
14725
|
+
const waitMs = evidence.retryAfterMs ?? Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * step);
|
|
14726
|
+
this.rate = {
|
|
14727
|
+
...this.rate,
|
|
14728
|
+
concurrency: 1,
|
|
14729
|
+
intervalCap: 1,
|
|
14730
|
+
intervalMs: Math.min(LIMITED_MAX_INTERVAL_MS, Math.max(this.rate.intervalMs ?? 0, LIMITED_INTERVAL_MS * step))
|
|
14731
|
+
};
|
|
14732
|
+
const nextMs = Math.max(this.pendingMs, waitMs);
|
|
14733
|
+
if (this.spentMs + nextMs > COOLDOWN_BUDGET_MS) this.#budgetSpent = true;
|
|
14734
|
+
const retries = this.#retries.get(url) ?? 0;
|
|
14735
|
+
if (!this.#budgetSpent && retries < RETRIES_PER_URL) {
|
|
14736
|
+
this.#retries.set(url, retries + 1);
|
|
14737
|
+
this.pendingMs = nextMs;
|
|
14738
|
+
return "retry";
|
|
14739
|
+
}
|
|
14740
|
+
this.skippedUrls += 1;
|
|
14741
|
+
return this.#budgetSpent ? "stop" : "skip";
|
|
14742
|
+
}
|
|
14743
|
+
async settle(sleep, signal) {
|
|
14744
|
+
if (this.pendingMs <= 0) return;
|
|
14745
|
+
await sleep(this.pendingMs, signal);
|
|
14746
|
+
this.spentMs += this.pendingMs;
|
|
14747
|
+
this.pendingMs = 0;
|
|
14748
|
+
}
|
|
14749
|
+
caveatInput(config, readPages) {
|
|
14750
|
+
if (this.skippedUrls === 0) return void 0;
|
|
14751
|
+
return {
|
|
14752
|
+
readPages,
|
|
14753
|
+
skippedUrls: this.skippedUrls,
|
|
14754
|
+
rerunCommand: rateLimitRerunCommand(config, this.intervalMs)
|
|
14755
|
+
};
|
|
14458
14756
|
}
|
|
14459
|
-
|
|
14460
|
-
|
|
14461
|
-
|
|
14462
|
-
|
|
14463
|
-
const right = left + 1;
|
|
14464
|
-
let next = index;
|
|
14465
|
-
const item = this.#items[next];
|
|
14466
|
-
const leftItem = this.#items[left];
|
|
14467
|
-
const rightItem = this.#items[right];
|
|
14468
|
-
if (leftItem && item && before(leftItem, item)) next = left;
|
|
14469
|
-
const nextItem = this.#items[next];
|
|
14470
|
-
if (rightItem && nextItem && before(rightItem, nextItem)) next = right;
|
|
14471
|
-
if (next === index) return;
|
|
14472
|
-
this.#swap(index, next);
|
|
14473
|
-
index = next;
|
|
14474
|
-
}
|
|
14757
|
+
warning() {
|
|
14758
|
+
if (!this.limited) return void 0;
|
|
14759
|
+
const plural = (count, noun) => `${count} ${noun}${count === 1 ? "" : "s"}`;
|
|
14760
|
+
return `The origin rate limited ${plural(this.responses, "request")}. The crawl slowed to one request at a time, at least ${Math.round(this.intervalMs / 1e3)}s apart, and waited ${Math.round(this.spentMs / 1e3)}s in cool-downs${this.skippedUrls > 0 ? `, then left ${plural(this.skippedUrls, "URL")} unread` : " before reading every queued URL"}.`;
|
|
14475
14761
|
}
|
|
14476
14762
|
};
|
|
14477
14763
|
// packages/core/src/analyze/crawler/external-link-checks.ts
|
|
@@ -14706,11 +14992,15 @@ function observationFromPage(requestedUrl, page) {
|
|
|
14706
14992
|
};
|
|
14707
14993
|
}
|
|
14708
14994
|
// packages/core/src/analyze/crawler/site-checks.ts
|
|
14709
|
-
const PROBE_COUNT =
|
|
14995
|
+
const PROBE_COUNT = 3;
|
|
14710
14996
|
function probeUrls(startUrl) {
|
|
14711
14997
|
const origin = new URL(startUrl).origin;
|
|
14712
14998
|
const seed = createHash("sha256").update(origin).digest("hex").slice(0, 12);
|
|
14713
|
-
return
|
|
14999
|
+
return [
|
|
15000
|
+
`/${seed}-not-found`,
|
|
15001
|
+
`/${seed}-not-found/${seed}-missing`,
|
|
15002
|
+
`/.well-known/seo-audit/not-found-${seed}-1`
|
|
15003
|
+
].map((path) => new URL(path, origin).toString());
|
|
14714
15004
|
}
|
|
14715
15005
|
function probeObservation(url, result) {
|
|
14716
15006
|
const request = result.request;
|
|
@@ -14730,29 +15020,40 @@ function probeObservation(url, result) {
|
|
|
14730
15020
|
error: request.outcome === "failure" ? request.error : `Probe skipped: ${request.reason}`
|
|
14731
15021
|
};
|
|
14732
15022
|
}
|
|
15023
|
+
function isConclusive(probe) {
|
|
15024
|
+
return probe.status !== void 0 && !probe.accessBlocked && probe.status !== 429 && probe.status !== 503;
|
|
15025
|
+
}
|
|
14733
15026
|
async function collectCrawlSiteChecks(input) {
|
|
15027
|
+
const probe = async (url) => probeObservation(url, await input.fetchStatusPage(url, {
|
|
15028
|
+
js: "off",
|
|
15029
|
+
refresh: true,
|
|
15030
|
+
timeoutMs: input.timeoutMs,
|
|
15031
|
+
signal: input.signal,
|
|
15032
|
+
writeCache: false,
|
|
15033
|
+
respectRobots: input.respectRobots,
|
|
15034
|
+
...input.rate ? { rate: input.rate } : {}
|
|
15035
|
+
}));
|
|
14734
15036
|
const probes = [];
|
|
14735
15037
|
for (const url of probeUrls(input.startUrl)) {
|
|
14736
15038
|
if (input.signal?.aborted) break;
|
|
14737
|
-
|
|
14738
|
-
|
|
14739
|
-
|
|
14740
|
-
|
|
14741
|
-
|
|
14742
|
-
|
|
14743
|
-
|
|
14744
|
-
}
|
|
14745
|
-
probes.push(probeObservation(url, result));
|
|
15039
|
+
probes.push(await probe(url));
|
|
15040
|
+
}
|
|
15041
|
+
if (input.sleep && !input.signal?.aborted && probes.some((item) => !isConclusive(item))) {
|
|
15042
|
+
await input.sleep(input.retryDelayMs ?? 2e3, input.signal);
|
|
15043
|
+
for (const [index, item] of probes.entries()) {
|
|
15044
|
+
if (input.signal?.aborted) break;
|
|
15045
|
+
if (!isConclusive(item)) probes[index] = await probe(item.url);
|
|
15046
|
+
}
|
|
14746
15047
|
}
|
|
14747
|
-
const conclusive = probes.filter(
|
|
15048
|
+
const conclusive = probes.filter(isConclusive);
|
|
14748
15049
|
const successful = conclusive.filter((probe) => (probe.status ?? 0) >= 200 && (probe.status ?? 0) < 400);
|
|
14749
15050
|
const correctMissing = conclusive.filter((probe) => probe.status === 404 || probe.status === 410);
|
|
14750
|
-
const status =
|
|
15051
|
+
const status = successful.length > 0 ? "warning" : correctMissing.length === PROBE_COUNT ? "pass" : "unknown";
|
|
14751
15052
|
const checks = { soft404: {
|
|
14752
15053
|
status,
|
|
14753
15054
|
probes,
|
|
14754
15055
|
probeLimit: PROBE_COUNT,
|
|
14755
|
-
complete:
|
|
15056
|
+
complete: conclusive.length === PROBE_COUNT
|
|
14756
15057
|
} };
|
|
14757
15058
|
if (status !== "warning") return {
|
|
14758
15059
|
checks,
|
|
@@ -14768,7 +15069,7 @@ async function collectCrawlSiteChecks(input) {
|
|
|
14768
15069
|
category: rule.category,
|
|
14769
15070
|
severity: rule.defaultSeverity,
|
|
14770
15071
|
url: new URL("/", input.startUrl).toString(),
|
|
14771
|
-
detail: `${successful.length}
|
|
15072
|
+
detail: `${successful.length} of ${probes.length} made-up paths returned a successful response or redirect: ${successful.map((probe) => `${new URL(probe.url).pathname} (${probe.status})`).join(", ")}.`,
|
|
14772
15073
|
evidence: checks.soft404
|
|
14773
15074
|
}]
|
|
14774
15075
|
};
|
|
@@ -17308,6 +17609,18 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
17308
17609
|
}[connection.provider](connection);
|
|
17309
17610
|
}
|
|
17310
17611
|
// packages/core/src/analyze/crawler/site-crawl-dependencies.ts
|
|
17612
|
+
function sleep$2(ms, signal) {
|
|
17613
|
+
return new Promise((resolve) => {
|
|
17614
|
+
if (signal?.aborted) return resolve();
|
|
17615
|
+
const done = () => {
|
|
17616
|
+
clearTimeout(timer);
|
|
17617
|
+
signal?.removeEventListener("abort", done);
|
|
17618
|
+
resolve();
|
|
17619
|
+
};
|
|
17620
|
+
const timer = setTimeout(done, ms);
|
|
17621
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
17622
|
+
});
|
|
17623
|
+
}
|
|
17311
17624
|
function resolveCrawlSiteDependencies(dependencies = {}) {
|
|
17312
17625
|
const hasInjectedPerPageSearchProvider = Boolean(dependencies.queryPageMetrics || dependencies.queryPageTopQuery);
|
|
17313
17626
|
const hasInjectedLegacyBulkSearchProvider = Boolean(dependencies.queryPagesMetrics || dependencies.queryPagesTopQueries);
|
|
@@ -17328,6 +17641,7 @@ function resolveCrawlSiteDependencies(dependencies = {}) {
|
|
|
17328
17641
|
fetchLandingPageValues: dependencies.fetchLandingPageValues ?? fetchLandingPageValues,
|
|
17329
17642
|
landingValueForUrl: dependencies.landingValueForUrl ?? landingValueForUrl,
|
|
17330
17643
|
now: dependencies.now ?? (() => /* @__PURE__ */ new Date()),
|
|
17644
|
+
sleep: dependencies.sleep ?? sleep$2,
|
|
17331
17645
|
memoryUsage: dependencies.memoryUsage ?? (() => process.memoryUsage()),
|
|
17332
17646
|
totalMemory: dependencies.totalMemory ?? totalmem
|
|
17333
17647
|
};
|
|
@@ -17543,6 +17857,10 @@ async function crawlSite(input, dependencies) {
|
|
|
17543
17857
|
const queueSafetyDeferredUrls = /* @__PURE__ */ new Set();
|
|
17544
17858
|
const maxTrackedQueueSafetyUrls = Math.min(5e4, Math.max(1e3, config.maxPages * 5));
|
|
17545
17859
|
let queueSafetyInventoryCapped = false;
|
|
17860
|
+
const queryVariantsByPath = /* @__PURE__ */ new Map();
|
|
17861
|
+
const queryVariantSkipped = /* @__PURE__ */ new Set();
|
|
17862
|
+
let renderingFallbackUrls = 0;
|
|
17863
|
+
const rateLimit = new CrawlRateLimit(config.fetchRate);
|
|
17546
17864
|
let failedUrls = 0;
|
|
17547
17865
|
let observedInternalLinks = 0;
|
|
17548
17866
|
let memoryLimited = false;
|
|
@@ -17627,6 +17945,15 @@ async function crawlSite(input, dependencies) {
|
|
|
17627
17945
|
});
|
|
17628
17946
|
return;
|
|
17629
17947
|
}
|
|
17948
|
+
if (isCdnUtilityUrl(normalized)) {
|
|
17949
|
+
recordSkip("cdn-utility-url");
|
|
17950
|
+
emitStatus("url_skipped", {
|
|
17951
|
+
url: normalized,
|
|
17952
|
+
depth,
|
|
17953
|
+
reason: "cdn_utility_url"
|
|
17954
|
+
});
|
|
17955
|
+
return;
|
|
17956
|
+
}
|
|
17630
17957
|
if (isLikelyAsset(normalized)) {
|
|
17631
17958
|
recordSkip("asset-url");
|
|
17632
17959
|
emitStatus("url_skipped", {
|
|
@@ -17645,6 +17972,20 @@ async function crawlSite(input, dependencies) {
|
|
|
17645
17972
|
});
|
|
17646
17973
|
return;
|
|
17647
17974
|
}
|
|
17975
|
+
const parsed = new URL(normalized);
|
|
17976
|
+
const pathKey = parsed.search ? `${parsed.origin}${parsed.pathname}` : "";
|
|
17977
|
+
if (pathKey && (queryVariantsByPath.get(pathKey) ?? 0) >= 5) {
|
|
17978
|
+
if (!queryVariantSkipped.has(normalized) && queryVariantSkipped.size < maxTrackedQueueSafetyUrls) {
|
|
17979
|
+
queryVariantSkipped.add(normalized);
|
|
17980
|
+
recordSkip("query-variant-limit");
|
|
17981
|
+
emitStatus("url_skipped", {
|
|
17982
|
+
url: normalized,
|
|
17983
|
+
depth,
|
|
17984
|
+
reason: "query_variant_limit"
|
|
17985
|
+
});
|
|
17986
|
+
}
|
|
17987
|
+
return;
|
|
17988
|
+
}
|
|
17648
17989
|
if (queue.size + visited.size + inFlight.size >= config.maxPages * 5) {
|
|
17649
17990
|
if (!queueSafetyDeferredUrls.has(normalized)) {
|
|
17650
17991
|
if (queueSafetyDeferredUrls.size < maxTrackedQueueSafetyUrls) {
|
|
@@ -17660,6 +18001,7 @@ async function crawlSite(input, dependencies) {
|
|
|
17660
18001
|
}
|
|
17661
18002
|
return;
|
|
17662
18003
|
}
|
|
18004
|
+
if (pathKey) queryVariantsByPath.set(pathKey, (queryVariantsByPath.get(pathKey) ?? 0) + 1);
|
|
17663
18005
|
discovered.add(normalized);
|
|
17664
18006
|
queuedUrls += 1;
|
|
17665
18007
|
queue.push({
|
|
@@ -17716,7 +18058,7 @@ async function crawlSite(input, dependencies) {
|
|
|
17716
18058
|
js: config.js,
|
|
17717
18059
|
refresh: config.refresh,
|
|
17718
18060
|
timeoutMs: config.timeoutMs,
|
|
17719
|
-
rate:
|
|
18061
|
+
rate: rateLimit.rate,
|
|
17720
18062
|
signal,
|
|
17721
18063
|
writeCache: false,
|
|
17722
18064
|
respectRobots: config.respectRobots,
|
|
@@ -17747,7 +18089,11 @@ async function crawlSite(input, dependencies) {
|
|
|
17747
18089
|
});
|
|
17748
18090
|
};
|
|
17749
18091
|
while (!isCancelled() && (queue.size || inFlight.size) && (!memoryLimited || inFlight.size > 0) && pages.length < config.maxPages) {
|
|
17750
|
-
|
|
18092
|
+
if (rateLimit.pendingMs > 0 && inFlight.size === 0) {
|
|
18093
|
+
await rateLimit.settle(deps.sleep, signal);
|
|
18094
|
+
if (isCancelled()) break;
|
|
18095
|
+
}
|
|
18096
|
+
while (!isCancelled() && !memoryLimited && rateLimit.pendingMs === 0 && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
|
|
17751
18097
|
const item = nextQueueItem();
|
|
17752
18098
|
if (!item) break;
|
|
17753
18099
|
startTask(item);
|
|
@@ -17767,11 +18113,46 @@ async function crawlSite(input, dependencies) {
|
|
|
17767
18113
|
error: result.warning ?? "Crawler returned no page snapshot.",
|
|
17768
18114
|
extraction: "not-applicable"
|
|
17769
18115
|
});
|
|
18116
|
+
const limited = rateLimit.observe(task.url, result, request.outcome === "response" ? request.status : void 0, deps.now().getTime());
|
|
18117
|
+
if (limited) {
|
|
18118
|
+
activeConcurrency = 1;
|
|
18119
|
+
healthyProbeStreak = 0;
|
|
18120
|
+
}
|
|
18121
|
+
if (limited === "retry") {
|
|
18122
|
+
visited.delete(task.url);
|
|
18123
|
+
if (!queue.has(task.url)) queue.push({
|
|
18124
|
+
url: task.url,
|
|
18125
|
+
depth: task.depth
|
|
18126
|
+
});
|
|
18127
|
+
emitStatus("url_queued", {
|
|
18128
|
+
url: task.url,
|
|
18129
|
+
depth: task.depth,
|
|
18130
|
+
reason: "rate_limited_retry",
|
|
18131
|
+
message: `The origin rate limited ${task.url}. Waiting ${Math.ceil(rateLimit.pendingMs / 1e3)}s, then retrying one request at a time.`
|
|
18132
|
+
});
|
|
18133
|
+
continue;
|
|
18134
|
+
}
|
|
18135
|
+
if (limited) {
|
|
18136
|
+
requests.push(request);
|
|
18137
|
+
recordSkip("origin-backpressure");
|
|
18138
|
+
emitStatus("page_skipped", {
|
|
18139
|
+
url: task.url,
|
|
18140
|
+
depth: task.depth,
|
|
18141
|
+
reason: "origin-backpressure"
|
|
18142
|
+
});
|
|
18143
|
+
if (limited === "stop") {
|
|
18144
|
+
let left = 0;
|
|
18145
|
+
while (queue.take()) left += 1;
|
|
18146
|
+
recordSkips("origin-backpressure", left);
|
|
18147
|
+
rateLimit.skippedUrls += left;
|
|
18148
|
+
}
|
|
18149
|
+
continue;
|
|
18150
|
+
}
|
|
17770
18151
|
requests.push(request);
|
|
17771
18152
|
if (healthStrategy) {
|
|
17772
18153
|
if (request.outcome === "response" && request.status < 400 && !request.accessBlock) {
|
|
17773
18154
|
healthyProbeStreak += 1;
|
|
17774
|
-
activeConcurrency = Math.min(config.concurrency, 1 + Math.floor(healthyProbeStreak / 20));
|
|
18155
|
+
activeConcurrency = Math.min(rateLimit.limited ? 1 : config.concurrency, 1 + Math.floor(healthyProbeStreak / 20));
|
|
17775
18156
|
} else {
|
|
17776
18157
|
healthyProbeStreak = 0;
|
|
17777
18158
|
activeConcurrency = 1;
|
|
@@ -17812,6 +18193,9 @@ async function crawlSite(input, dependencies) {
|
|
|
17812
18193
|
continue;
|
|
17813
18194
|
}
|
|
17814
18195
|
const page = result.page;
|
|
18196
|
+
const rendering = page.fetchDiagnostics?.rendering;
|
|
18197
|
+
if (rendering?.status === "unavailable" && config.js === "on") throw new SeoError("INVALID_INPUT", `--rendering on needs a local Chrome or Chromium browser, and none could start. ${rendering.error ?? ""} Install Chromium, or set SEO_BROWSER_EXECUTABLE_PATH to a Chromium executable, then rerun. Use --rendering auto to accept raw HTML.`.replace(/\s+/g, " ").trim());
|
|
18198
|
+
if (rendering?.status === "unavailable" || rendering?.status === "failed") renderingFallbackUrls += 1;
|
|
17815
18199
|
const finalUrl = normalizeUrl$1(page.finalUrl, config.url);
|
|
17816
18200
|
if (!finalUrl) {
|
|
17817
18201
|
failedUrls += 1;
|
|
@@ -17841,7 +18225,7 @@ async function crawlSite(input, dependencies) {
|
|
|
17841
18225
|
const storeDocument = (urls) => {
|
|
17842
18226
|
const sourceRequest = normalizeUrl$1(task.url, config.url) ?? task.url;
|
|
17843
18227
|
const directlyRequested = sourceRequest === finalUrl;
|
|
17844
|
-
const normalizedLinks = urls.map((url) => normalizeUrl$1(url, finalUrl)).filter((url) => Boolean(url));
|
|
18228
|
+
const normalizedLinks = urls.map((url) => normalizeUrl$1(url, finalUrl)).filter((url) => Boolean(url) && !isCdnUtilityUrl(url ?? ""));
|
|
17845
18229
|
const syncLinkEvidence = (target, links) => {
|
|
17846
18230
|
const sortedLinks = [...links].sort((left, right) => left < right ? -1 : left > right ? 1 : 0);
|
|
17847
18231
|
target.outgoingInternalCount = sortedLinks.length;
|
|
@@ -17920,7 +18304,7 @@ async function crawlSite(input, dependencies) {
|
|
|
17920
18304
|
if (followLinks && task.depth < config.maxDepth) {
|
|
17921
18305
|
if (memoryLimited) for (const value of result.urls) {
|
|
17922
18306
|
const normalized = normalizeUrl$1(value, finalUrl);
|
|
17923
|
-
if (normalized && sameOrigin(normalized, origin) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
|
|
18307
|
+
if (normalized && sameOrigin(normalized, origin) && !isCdnUtilityUrl(normalized) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
|
|
17924
18308
|
discovered.add(normalized);
|
|
17925
18309
|
memoryDeferredUrls.add(normalized);
|
|
17926
18310
|
}
|
|
@@ -17945,6 +18329,8 @@ async function crawlSite(input, dependencies) {
|
|
|
17945
18329
|
cancelled = isCancelled();
|
|
17946
18330
|
const robotsDisallowed = skipReasonCounts["robots-disallowed"] ?? 0;
|
|
17947
18331
|
if (robotsDisallowed > 0) warnings.push(`${robotsDisallowed} URL${robotsDisallowed === 1 ? "" : "s"} disallowed ${SEO_CRAWLER_USER_AGENT} in robots.txt. If the audit should reach them, allow the SEO-Skill token on only those public paths.`);
|
|
18332
|
+
const rateLimitWarning = rateLimit.warning();
|
|
18333
|
+
if (rateLimitWarning) warnings.push(rateLimitWarning);
|
|
17948
18334
|
if (cancelled) {
|
|
17949
18335
|
warnings.push("Crawl cancelled; returning a partial report.");
|
|
17950
18336
|
emitStatus("cancelled", {
|
|
@@ -17956,7 +18342,7 @@ async function crawlSite(input, dependencies) {
|
|
|
17956
18342
|
const sitemapEvidencePartial = sitemapDiscovery?.dataStatus === "partial";
|
|
17957
18343
|
const accessEvidencePartial = requests.some((request) => request.outcome === "response" && request.accessBlock);
|
|
17958
18344
|
const requiredSitemapUnavailable = config.mode === "sitemap" && sitemapDiscovery?.dataStatus === "unavailable";
|
|
17959
|
-
const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
|
|
18345
|
+
const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || rateLimit.skippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
|
|
17960
18346
|
const failed = !cancelled && pages.length === 0 && (requiredSitemapUnavailable || requests.length > 0 && requests.every((request) => request.outcome === "failure"));
|
|
17961
18347
|
const requestEvidenceStatus = cancelled && inFlight.size > 0 ? "partial" : "available";
|
|
17962
18348
|
const dataSources = await crawlDataSources({
|
|
@@ -18003,7 +18389,10 @@ async function crawlSite(input, dependencies) {
|
|
|
18003
18389
|
timeoutMs: config.timeoutMs,
|
|
18004
18390
|
fetchStatusPage: deps.fetchStatusPage,
|
|
18005
18391
|
respectRobots: config.respectRobots,
|
|
18006
|
-
signal
|
|
18392
|
+
signal,
|
|
18393
|
+
rate: rateLimit.rate,
|
|
18394
|
+
sleep: deps.sleep,
|
|
18395
|
+
retryDelayMs: rateLimit.settleMs
|
|
18007
18396
|
}) : void 0;
|
|
18008
18397
|
for (const page of pages) {
|
|
18009
18398
|
if (!page.geo || !llmsTxt) continue;
|
|
@@ -18034,6 +18423,12 @@ async function crawlSite(input, dependencies) {
|
|
|
18034
18423
|
...startPage.responseHeaders?.["content-signal"] ? { contentSignal: startPage.responseHeaders["content-signal"] } : {}
|
|
18035
18424
|
} } : {}
|
|
18036
18425
|
} : aiSignals?.signals;
|
|
18426
|
+
const reachIssues = followLinks && config.maxDepth > 0 && config.maxPages > 1 && !cancelled ? crawlReachIssues({
|
|
18427
|
+
startUrl: config.url,
|
|
18428
|
+
pages: pages.filter((page) => page.status >= 200 && page.status < 300),
|
|
18429
|
+
crawledPages: pages.length,
|
|
18430
|
+
sitemapUrlsReturned: sitemapDiscovery?.urlsReturned ?? 0
|
|
18431
|
+
}) : [];
|
|
18037
18432
|
const report = createCrawlReport({
|
|
18038
18433
|
config,
|
|
18039
18434
|
site,
|
|
@@ -18048,6 +18443,7 @@ async function crawlSite(input, dependencies) {
|
|
|
18048
18443
|
...externalLinkVerification ? { externalLinkVerification } : {},
|
|
18049
18444
|
...siteCheckResult ? { siteChecks: siteCheckResult.checks } : {},
|
|
18050
18445
|
additionalIssues: [
|
|
18446
|
+
...reachIssues,
|
|
18051
18447
|
...siteCheckResult?.issues ?? [],
|
|
18052
18448
|
...sitemapDiscovery ? sitemapHostIssues(sitemapDiscovery, config.url) : [],
|
|
18053
18449
|
...ai ? agentReadyIssues({
|
|
@@ -18065,7 +18461,12 @@ async function crawlSite(input, dependencies) {
|
|
|
18065
18461
|
queueSafetySkippedUrls,
|
|
18066
18462
|
queueSafetyInventoryCapped,
|
|
18067
18463
|
originBackpressureSkippedUrls,
|
|
18068
|
-
|
|
18464
|
+
rateLimit: rateLimit.caveatInput(config, pages.length),
|
|
18465
|
+
sitemapLimit: sitemapLimitCaveatInput(config, sitemapDiscovery, input.maxPages === void 0),
|
|
18466
|
+
memoryPressureSkippedUrls,
|
|
18467
|
+
queryVariantSkippedUrls: queryVariantSkipped.size,
|
|
18468
|
+
renderingFallbackUrls,
|
|
18469
|
+
reachIssues
|
|
18069
18470
|
}),
|
|
18070
18471
|
stats: {
|
|
18071
18472
|
discoveredUrls: discovered.size,
|
|
@@ -18620,7 +19021,7 @@ function analyzeCtrUnderperformersFromRows(input) {
|
|
|
18620
19021
|
// packages/core/src/analyze/ctr-underperformers-report.ts
|
|
18621
19022
|
const MAX_GSC_ROWS$3 = 1e5;
|
|
18622
19023
|
const DEFAULT_DAYS$3 = 28;
|
|
18623
|
-
const MAX_DAYS$
|
|
19024
|
+
const MAX_DAYS$3 = 548;
|
|
18624
19025
|
function plural$8(count, singular, pluralLabel = `${singular}s`) {
|
|
18625
19026
|
return count === 1 ? singular : pluralLabel;
|
|
18626
19027
|
}
|
|
@@ -18639,12 +19040,12 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
18639
19040
|
maximum: 100,
|
|
18640
19041
|
label: "limit"
|
|
18641
19042
|
});
|
|
18642
|
-
const explicitRange = explicitDateRange(input, MAX_DAYS$
|
|
19043
|
+
const explicitRange = explicitDateRange(input, MAX_DAYS$3);
|
|
18643
19044
|
const days = explicitRange?.days ?? integerOption$1({
|
|
18644
19045
|
value: input.days,
|
|
18645
19046
|
fallback: DEFAULT_DAYS$3,
|
|
18646
19047
|
minimum: 1,
|
|
18647
|
-
maximum: MAX_DAYS$
|
|
19048
|
+
maximum: MAX_DAYS$3,
|
|
18648
19049
|
label: "Days"
|
|
18649
19050
|
});
|
|
18650
19051
|
const now = dependencies.now();
|
|
@@ -21885,7 +22286,7 @@ function quickWinTemplateSummaries(items) {
|
|
|
21885
22286
|
}
|
|
21886
22287
|
// packages/core/src/analyze/site-diagnostics/quick-wins-report.ts
|
|
21887
22288
|
const DEFAULT_DAYS$2 = 28;
|
|
21888
|
-
const MAX_DAYS$
|
|
22289
|
+
const MAX_DAYS$2 = 548;
|
|
21889
22290
|
const DEFAULT_VERIFY_LIMIT$1 = 5;
|
|
21890
22291
|
const MAX_VERIFY_LIMIT = 100;
|
|
21891
22292
|
const MAX_SOURCE_ROWS$2 = 1e5;
|
|
@@ -21903,12 +22304,12 @@ function applyVerification$1(item, coverage) {
|
|
|
21903
22304
|
};
|
|
21904
22305
|
}
|
|
21905
22306
|
async function quickWinsReportWith(input, dependencies) {
|
|
21906
|
-
const explicitRange = explicitDateRange(input, MAX_DAYS$
|
|
22307
|
+
const explicitRange = explicitDateRange(input, MAX_DAYS$2);
|
|
21907
22308
|
const days = explicitRange?.days ?? integerOption$1({
|
|
21908
22309
|
value: input.days,
|
|
21909
22310
|
fallback: DEFAULT_DAYS$2,
|
|
21910
22311
|
minimum: 1,
|
|
21911
|
-
maximum: MAX_DAYS$
|
|
22312
|
+
maximum: MAX_DAYS$2,
|
|
21912
22313
|
label: "Days"
|
|
21913
22314
|
});
|
|
21914
22315
|
const verifyLimit = integerOption$1({
|
|
@@ -22327,7 +22728,7 @@ function analyzeStrikingDistanceRows(input) {
|
|
|
22327
22728
|
}
|
|
22328
22729
|
// packages/core/src/analyze/striking-distance-report.ts
|
|
22329
22730
|
const DEFAULT_DAYS$1 = 28;
|
|
22330
|
-
const MAX_DAYS = 548;
|
|
22731
|
+
const MAX_DAYS$1 = 548;
|
|
22331
22732
|
const DEFAULT_VERIFY_LIMIT = 5;
|
|
22332
22733
|
const MAX_SOURCE_ROWS$1 = 1e5;
|
|
22333
22734
|
function boundedInteger$2(value, fallback, minimum, maximum) {
|
|
@@ -22336,7 +22737,7 @@ function boundedInteger$2(value, fallback, minimum, maximum) {
|
|
|
22336
22737
|
}
|
|
22337
22738
|
function reportDays(value) {
|
|
22338
22739
|
if (value === void 0) return DEFAULT_DAYS$1;
|
|
22339
|
-
if (!Number.isInteger(value) || value < 1 || value > MAX_DAYS) throw new SeoError("INVALID_INPUT", `Days must be a whole number between 1 and ${MAX_DAYS}.`);
|
|
22740
|
+
if (!Number.isInteger(value) || value < 1 || value > MAX_DAYS$1) throw new SeoError("INVALID_INPUT", `Days must be a whole number between 1 and ${MAX_DAYS$1}.`);
|
|
22340
22741
|
return value;
|
|
22341
22742
|
}
|
|
22342
22743
|
function dateRange(days, now) {
|
|
@@ -22376,7 +22777,7 @@ function verdict$1(analysis) {
|
|
|
22376
22777
|
return `${countLabel(analysis.summary.eligibleRows, "eligible query/page row")} found; ${countLabel(analysis.summary.returnedRows, "row")} returned for review.`;
|
|
22377
22778
|
}
|
|
22378
22779
|
async function strikingDistanceWith(input, dependencies) {
|
|
22379
|
-
const explicitRange = explicitDateRange(input, MAX_DAYS);
|
|
22780
|
+
const explicitRange = explicitDateRange(input, MAX_DAYS$1);
|
|
22380
22781
|
const days = explicitRange?.days ?? reportDays(input.days);
|
|
22381
22782
|
const generatedAt = dependencies.now().toISOString();
|
|
22382
22783
|
const range = explicitRange?.range ?? dateRange(days, new Date(generatedAt));
|
|
@@ -31789,6 +32190,7 @@ const wait = (ms) => new Promise((done) => setTimeout(done, ms));
|
|
|
31789
32190
|
function requireInput(input) {
|
|
31790
32191
|
const needsKeywords = input.action === "keyword_metrics" || input.action === "serp";
|
|
31791
32192
|
if (needsKeywords && input.keywords.length === 0) throw new CommandError("INVALID_INPUT", `${input.action} needs keywords.`);
|
|
32193
|
+
if (input.action === "link_gap" && !input.target) throw new CommandError("INVALID_INPUT", "link_gap needs a target domain and competitors.");
|
|
31792
32194
|
if (!needsKeywords && !input.target) throw new CommandError("INVALID_INPUT", `${input.action} needs a target domain or URL.`);
|
|
31793
32195
|
}
|
|
31794
32196
|
async function eachKeyword(keywords, missed, rowsFor) {
|
|
@@ -31806,14 +32208,17 @@ async function eachKeyword(keywords, missed, rowsFor) {
|
|
|
31806
32208
|
}
|
|
31807
32209
|
async function research(provider, headers, input) {
|
|
31808
32210
|
requireInput(input);
|
|
32211
|
+
if (input.action === "link_gap") return linkGap(provider, headers, input);
|
|
31809
32212
|
const keys = need(headers, provider);
|
|
31810
32213
|
if (provider === "dataforseo") return dataForSeo(keys, input);
|
|
31811
32214
|
const missed = [];
|
|
32215
|
+
const rows = await ahrefs(keys[0] ?? "", input, missed);
|
|
31812
32216
|
return {
|
|
31813
|
-
rows
|
|
32217
|
+
rows,
|
|
31814
32218
|
costUsd: null,
|
|
31815
32219
|
partial: false,
|
|
31816
|
-
missed
|
|
32220
|
+
missed,
|
|
32221
|
+
noData: withoutData(input, rows, missed, 100)
|
|
31817
32222
|
};
|
|
31818
32223
|
}
|
|
31819
32224
|
function countryName(code) {
|
|
@@ -31865,7 +32270,8 @@ async function dataForSeoTask([login, password], path, task) {
|
|
|
31865
32270
|
if (result?.status_code === NO_RESULTS) return {
|
|
31866
32271
|
cost: result.cost ?? 0,
|
|
31867
32272
|
result: null,
|
|
31868
|
-
partial: false
|
|
32273
|
+
partial: false,
|
|
32274
|
+
noResults: true
|
|
31869
32275
|
};
|
|
31870
32276
|
if (!result || result.status_code >= 4e4 && result.status_code !== PARTIAL) throw Object.assign(new CommandError("PROVIDER_ERROR", `dataforseo: ${result?.status_message ?? "no task result"}`), { cost: result?.cost ?? 0 });
|
|
31871
32277
|
return {
|
|
@@ -31906,18 +32312,29 @@ async function dataForSeo(keys, input) {
|
|
|
31906
32312
|
partial: false,
|
|
31907
32313
|
missed: []
|
|
31908
32314
|
};
|
|
32315
|
+
const rows = await dataForSeoRows(keys, input, spent);
|
|
31909
32316
|
return {
|
|
31910
|
-
rows
|
|
32317
|
+
rows,
|
|
31911
32318
|
costUsd: Math.round(spent.cost * 1e4) / 1e4,
|
|
31912
32319
|
partial: spent.partial,
|
|
31913
|
-
missed: spent.missed
|
|
32320
|
+
missed: spent.missed,
|
|
32321
|
+
noData: withoutData(input, rows, spent.missed, 700)
|
|
31914
32322
|
};
|
|
31915
32323
|
}
|
|
32324
|
+
function withoutData(input, rows, missed, keywordLimit) {
|
|
32325
|
+
if (input.action === "keyword_metrics" || input.action === "serp") {
|
|
32326
|
+
const max = input.action === "serp" ? 5 : keywordLimit;
|
|
32327
|
+
const answered = new Set(rows.map((row) => String(row.keyword ?? "").toLowerCase()));
|
|
32328
|
+
const failed = new Set(missed);
|
|
32329
|
+
return input.keywords.slice(0, max).filter((keyword) => !failed.has(keyword) && !answered.has(keyword.toLowerCase()));
|
|
32330
|
+
}
|
|
32331
|
+
return [];
|
|
32332
|
+
}
|
|
31916
32333
|
async function dataForSeoRows(keys, input, spent) {
|
|
31917
32334
|
const language = dataForSeoLanguage(input.country);
|
|
31918
32335
|
if (!language && input.action !== "serp" && input.action !== "backlinks") throw new CommandError("INVALID_INPUT", `DataForSEO has no ${input.action} data for country "${input.country}".`);
|
|
31919
32336
|
const market = dataForSeoMarket(input.country, language);
|
|
31920
|
-
const post = async (path, task) => {
|
|
32337
|
+
const post = async (path, task, attempts = 1) => {
|
|
31921
32338
|
const done = await dataForSeoTask(keys, path, {
|
|
31922
32339
|
...market,
|
|
31923
32340
|
...task
|
|
@@ -31927,6 +32344,7 @@ async function dataForSeoRows(keys, input, spent) {
|
|
|
31927
32344
|
});
|
|
31928
32345
|
spent.cost += done.cost;
|
|
31929
32346
|
spent.partial ||= done.partial;
|
|
32347
|
+
if (done.noResults && attempts > 1) return post(path, task, attempts - 1);
|
|
31930
32348
|
return done.result?.items ?? [];
|
|
31931
32349
|
};
|
|
31932
32350
|
const get = (value, path) => path.split(".").reduce((current, key) => current && typeof current === "object" ? current[key] : void 0, value);
|
|
@@ -31956,7 +32374,7 @@ async function dataForSeoRows(keys, input, spent) {
|
|
|
31956
32374
|
return (await post("v3/serp/google/organic/live/advanced", {
|
|
31957
32375
|
keyword,
|
|
31958
32376
|
depth: Math.min(input.limit, 100)
|
|
31959
|
-
})).filter((item) => item.type === "organic").map((item) => {
|
|
32377
|
+
}, 2)).filter((item) => item.type === "organic").map((item) => {
|
|
31960
32378
|
const row = pick(item, {
|
|
31961
32379
|
position: "rank_absolute",
|
|
31962
32380
|
url: "url",
|
|
@@ -31964,9 +32382,11 @@ async function dataForSeoRows(keys, input, spent) {
|
|
|
31964
32382
|
title: "title"
|
|
31965
32383
|
});
|
|
31966
32384
|
if (/^https?:\/\/(www\.)?google\.[a-z.]+\/goto\b/.test(String(row.url))) {
|
|
31967
|
-
const
|
|
32385
|
+
const crumb = String(item.breadcrumb ?? "");
|
|
32386
|
+
const site = /^https?:\/\/([^/\s›]+)/.exec(crumb);
|
|
31968
32387
|
row.url = null;
|
|
31969
32388
|
row.domain = site?.[1] ?? null;
|
|
32389
|
+
row.url_hint = site ? crumb.replace(/^https?:\/\//, "") : null;
|
|
31970
32390
|
}
|
|
31971
32391
|
return {
|
|
31972
32392
|
keyword,
|
|
@@ -31996,6 +32416,93 @@ async function dataForSeoRows(keys, input, spent) {
|
|
|
31996
32416
|
organic_etv: "full_domain_metrics.organic.etv"
|
|
31997
32417
|
}));
|
|
31998
32418
|
}
|
|
32419
|
+
const GAP_DOMAINS = 1e3;
|
|
32420
|
+
const GAP_COMPETITORS = 5;
|
|
32421
|
+
const SHORTENERS = [
|
|
32422
|
+
"bit.ly",
|
|
32423
|
+
"tinyurl.com",
|
|
32424
|
+
"t.co",
|
|
32425
|
+
"ow.ly",
|
|
32426
|
+
"buff.ly",
|
|
32427
|
+
"is.gd",
|
|
32428
|
+
"cutt.ly",
|
|
32429
|
+
"rebrand.ly",
|
|
32430
|
+
"shorturl.at",
|
|
32431
|
+
"rb.gy",
|
|
32432
|
+
"lnkd.in"
|
|
32433
|
+
];
|
|
32434
|
+
const FREE_APP_HOSTS = [
|
|
32435
|
+
"web.app",
|
|
32436
|
+
"firebaseapp.com",
|
|
32437
|
+
"pages.dev"
|
|
32438
|
+
];
|
|
32439
|
+
const LINK_LIST_NAME = /(subdomain|whois|login|signin|coupon|promo|voucher|discount|sitelike|similarsite|sitesimilar|siteprice|sitevalue|siteworth|websiteworth|hypestat|statshow|websiteoutlook|siteinfo|domaininfo|domainstat|domaintools|ipaddress|dnslookup|linkdirectory|webdirectory|alternativeto-)/;
|
|
32440
|
+
function isLinkList(domain) {
|
|
32441
|
+
const within = (name) => domain === name || domain.endsWith(`.${name}`);
|
|
32442
|
+
if (SHORTENERS.some(within)) return true;
|
|
32443
|
+
if (FREE_APP_HOSTS.some((name) => domain.endsWith(`.${name}`))) return true;
|
|
32444
|
+
return LINK_LIST_NAME.test(domain.replace(/[-.]/g, ""));
|
|
32445
|
+
}
|
|
32446
|
+
async function linkGap(provider, headers, input) {
|
|
32447
|
+
const target = bareTarget(input.target).replace(/^www\./, "");
|
|
32448
|
+
const competitors = [...new Set((input.competitors ?? []).map((site) => bareTarget(site).replace(/^www\./, "")))].filter((site) => site && site !== target);
|
|
32449
|
+
if (competitors.length < 2 || competitors.length > GAP_COMPETITORS) throw new CommandError("INVALID_INPUT", `link_gap needs a target and 2 to ${GAP_COMPETITORS} competitors.`);
|
|
32450
|
+
const score = provider === "ahrefs" ? "domain_rating" : "rank";
|
|
32451
|
+
const sites = [target, ...competitors];
|
|
32452
|
+
const lists = [];
|
|
32453
|
+
let cost = 0;
|
|
32454
|
+
let partial = false;
|
|
32455
|
+
for (const site of sites) {
|
|
32456
|
+
const got = await research(provider, headers, {
|
|
32457
|
+
...input,
|
|
32458
|
+
action: "backlinks",
|
|
32459
|
+
target: site,
|
|
32460
|
+
limit: GAP_DOMAINS
|
|
32461
|
+
});
|
|
32462
|
+
cost += got.costUsd ?? 0;
|
|
32463
|
+
partial ||= got.partial;
|
|
32464
|
+
lists.push(got.rows);
|
|
32465
|
+
}
|
|
32466
|
+
const domainOf = (row) => String(row.domain ?? "").toLowerCase().replace(/^www\./, "");
|
|
32467
|
+
const authority = (row) => Number(row[score] ?? 0);
|
|
32468
|
+
const floors = lists.filter((rows) => rows.length >= GAP_DOMAINS).map((rows) => Math.min(...rows.map(authority)));
|
|
32469
|
+
const floor = floors.length ? Math.max(...floors) : null;
|
|
32470
|
+
const own = new Set((lists[0] ?? []).map(domainOf));
|
|
32471
|
+
const found = /* @__PURE__ */ new Map();
|
|
32472
|
+
competitors.forEach((site, index) => {
|
|
32473
|
+
for (const row of lists[index + 1] ?? []) {
|
|
32474
|
+
const domain = domainOf(row);
|
|
32475
|
+
if (!domain || own.has(domain) || domain === target) continue;
|
|
32476
|
+
if (domain.endsWith(`.${target}`)) continue;
|
|
32477
|
+
const entry = found.get(domain) ?? {
|
|
32478
|
+
value: 0,
|
|
32479
|
+
sites: []
|
|
32480
|
+
};
|
|
32481
|
+
entry.value = Math.max(entry.value, authority(row));
|
|
32482
|
+
if (!entry.sites.includes(site)) entry.sites.push(site);
|
|
32483
|
+
found.set(domain, entry);
|
|
32484
|
+
}
|
|
32485
|
+
});
|
|
32486
|
+
const gap = [...found.entries()].filter(([, entry]) => entry.sites.length >= 2 && (floor === null || entry.value >= floor));
|
|
32487
|
+
const linkLists = gap.map(([domain]) => domain).filter(isLinkList).sort((a, b) => a.localeCompare(b));
|
|
32488
|
+
return {
|
|
32489
|
+
rows: gap.filter(([domain]) => !isLinkList(domain)).sort(([a, left], [b, right]) => right.value - left.value || right.sites.length - left.sites.length || a.localeCompare(b)).map(([domain, entry]) => ({
|
|
32490
|
+
domain,
|
|
32491
|
+
[score]: entry.value,
|
|
32492
|
+
competitors_linking: entry.sites.length,
|
|
32493
|
+
links_to: entry.sites.join(" ")
|
|
32494
|
+
})),
|
|
32495
|
+
costUsd: provider === "dataforseo" ? Math.round(cost * 1e4) / 1e4 : null,
|
|
32496
|
+
partial,
|
|
32497
|
+
missed: [],
|
|
32498
|
+
noData: [],
|
|
32499
|
+
...floor !== null ? { floor: {
|
|
32500
|
+
field: score,
|
|
32501
|
+
value: floor
|
|
32502
|
+
} } : {},
|
|
32503
|
+
...linkLists.length ? { leftOut: linkLists } : {}
|
|
32504
|
+
};
|
|
32505
|
+
}
|
|
31999
32506
|
function isPage(target) {
|
|
32000
32507
|
return /\/./.test(bareTarget(target));
|
|
32001
32508
|
}
|
|
@@ -32086,7 +32593,7 @@ const DIMENSIONS = [
|
|
|
32086
32593
|
"device",
|
|
32087
32594
|
"searchAppearance"
|
|
32088
32595
|
];
|
|
32089
|
-
const ROWS = 25e3;
|
|
32596
|
+
const ROWS$1 = 25e3;
|
|
32090
32597
|
async function runRows(input, deps) {
|
|
32091
32598
|
const dimensions = input.dimensions?.length ? input.dimensions : ["query"];
|
|
32092
32599
|
const filters = input.filters ?? [];
|
|
@@ -32095,7 +32602,7 @@ async function runRows(input, deps) {
|
|
|
32095
32602
|
const query = {
|
|
32096
32603
|
dimensions,
|
|
32097
32604
|
type: "web",
|
|
32098
|
-
maxRows: ROWS,
|
|
32605
|
+
maxRows: ROWS$1,
|
|
32099
32606
|
...filters.length ? { dimensionFilterGroups: [{
|
|
32100
32607
|
groupType: "and",
|
|
32101
32608
|
filters
|
|
@@ -32152,9 +32659,9 @@ async function runRows(input, deps) {
|
|
|
32152
32659
|
const sum = (key) => rows.reduce((total, row) => total + Number(row[key] ?? 0), 0);
|
|
32153
32660
|
notes.push(`Totals: clicks ${sum("clicks")}, impressions ${sum("impressions")}; previous period clicks ${sum("previous_clicks")}.`);
|
|
32154
32661
|
}
|
|
32155
|
-
notes.push(`Compared with ${previous.startDate} to ${previous.endDate}${before.rowsFetched >= ROWS ? " (earlier period hit the row limit)" : ""}. 0 clicks with no position means the row had no impressions in the current period.`);
|
|
32662
|
+
notes.push(`Compared with ${previous.startDate} to ${previous.endDate}${before.rowsFetched >= ROWS$1 ? " (earlier period hit the row limit)" : ""}. 0 clicks with no position means the row had no impressions in the current period.`);
|
|
32156
32663
|
if (pageAt >= 0) notes.push("Page URLs with #fragments are merged into their page.");
|
|
32157
|
-
if (current.rowsFetched >= ROWS) notes.push(`Search Console has more than ${ROWS} rows; add a filter to narrow it.`);
|
|
32664
|
+
if (current.rowsFetched >= ROWS$1) notes.push(`Search Console has more than ${ROWS$1} rows; add a filter to narrow it.`);
|
|
32158
32665
|
notes.push("Search Console hides some rare queries for privacy, so query rows do not add up to the site totals.");
|
|
32159
32666
|
const detail = input.detail ?? "summary";
|
|
32160
32667
|
return {
|
|
@@ -32263,6 +32770,22 @@ const hostOf$2 = (url) => {
|
|
|
32263
32770
|
return null;
|
|
32264
32771
|
}
|
|
32265
32772
|
};
|
|
32773
|
+
const NOT_SOURCES = [
|
|
32774
|
+
"dataforseo.com",
|
|
32775
|
+
"images.openai.com",
|
|
32776
|
+
"gstatic.com",
|
|
32777
|
+
"googleusercontent.com",
|
|
32778
|
+
"ggpht.com",
|
|
32779
|
+
"example.com",
|
|
32780
|
+
"example.org",
|
|
32781
|
+
"example.net"
|
|
32782
|
+
];
|
|
32783
|
+
const IMAGE = /\.(png|jpe?g|gif|webp|avif|svg)(\?|#|$)/i;
|
|
32784
|
+
const isSource = (url) => {
|
|
32785
|
+
const host = hostOf$2(url);
|
|
32786
|
+
if (!host || IMAGE.test(url)) return false;
|
|
32787
|
+
return !NOT_SOURCES.some((name) => host === name || host.endsWith(`.${name}`));
|
|
32788
|
+
};
|
|
32266
32789
|
const empty = {
|
|
32267
32790
|
cited_url: null,
|
|
32268
32791
|
sources: [],
|
|
@@ -32291,7 +32814,7 @@ function judge(engine, task, site) {
|
|
|
32291
32814
|
const cited = all.find((url) => onSite(url, site.domain)) ?? null;
|
|
32292
32815
|
const base = {
|
|
32293
32816
|
cited_url: cited,
|
|
32294
|
-
sources: [...new Set(all.filter((url) => !onSite(url, site.domain)).map(hostOf$2).filter((host) => Boolean(host)))].slice(0, MAX_SOURCES),
|
|
32817
|
+
sources: [...new Set(all.filter((url) => !onSite(url, site.domain) && isSource(url)).map(hostOf$2).filter((host) => Boolean(host)))].slice(0, MAX_SOURCES),
|
|
32295
32818
|
model: shown.model,
|
|
32296
32819
|
answered_at
|
|
32297
32820
|
};
|
|
@@ -32344,6 +32867,7 @@ async function runAiAnswers(input, headers, deps, more) {
|
|
|
32344
32867
|
question,
|
|
32345
32868
|
impressions,
|
|
32346
32869
|
task: null,
|
|
32870
|
+
ref: null,
|
|
32347
32871
|
cost_usd: 0,
|
|
32348
32872
|
...empty
|
|
32349
32873
|
});
|
|
@@ -32355,7 +32879,8 @@ async function runAiAnswers(input, headers, deps, more) {
|
|
|
32355
32879
|
...blank("", null),
|
|
32356
32880
|
engine: engine.name,
|
|
32357
32881
|
result: "pending",
|
|
32358
|
-
task: value
|
|
32882
|
+
task: value,
|
|
32883
|
+
ref: value
|
|
32359
32884
|
};
|
|
32360
32885
|
rows.push(row);
|
|
32361
32886
|
waiting.push({
|
|
@@ -32408,6 +32933,7 @@ async function runAiAnswers(input, headers, deps, more) {
|
|
|
32408
32933
|
}
|
|
32409
32934
|
row.result = "pending";
|
|
32410
32935
|
row.task = `${engine.name}:${task.id}`;
|
|
32936
|
+
row.ref = row.task;
|
|
32411
32937
|
waiting.push({
|
|
32412
32938
|
row,
|
|
32413
32939
|
engine,
|
|
@@ -32448,6 +32974,7 @@ async function runAiAnswers(input, headers, deps, more) {
|
|
|
32448
32974
|
const cost = usd(rows.reduce((sum, row) => sum + row.cost_usd, 0));
|
|
32449
32975
|
const count = (result) => rows.filter((row) => row.result === result).length;
|
|
32450
32976
|
const left = rows.filter((row) => row.task).map((row) => row.task);
|
|
32977
|
+
const refs = rows.flatMap((row) => row.ref ? [row.ref] : []);
|
|
32451
32978
|
const cites = /* @__PURE__ */ new Map();
|
|
32452
32979
|
for (const row of rows) for (const host of row.sources) cites.set(host, (cites.get(host) ?? 0) + 1);
|
|
32453
32980
|
const topCited = [...cites].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10);
|
|
@@ -32481,7 +33008,7 @@ async function runAiAnswers(input, headers, deps, more) {
|
|
|
32481
33008
|
limit: 50,
|
|
32482
33009
|
detail: input.detail,
|
|
32483
33010
|
notes,
|
|
32484
|
-
more
|
|
33011
|
+
more: (offset) => more(offset, refs)
|
|
32485
33012
|
}),
|
|
32486
33013
|
data: {
|
|
32487
33014
|
provider: "dataforseo",
|
|
@@ -32495,6 +33022,7 @@ async function runAiAnswers(input, headers, deps, more) {
|
|
|
32495
33022
|
questions: [...new Set(rows.map((row) => row.question))],
|
|
32496
33023
|
costUsd: cost,
|
|
32497
33024
|
pendingTasks: left,
|
|
33025
|
+
tasks: refs,
|
|
32498
33026
|
citedSites: topCited.map(([host, answers]) => ({
|
|
32499
33027
|
host,
|
|
32500
33028
|
answers
|
|
@@ -32614,7 +33142,7 @@ function ga4MatchReason(match, site) {
|
|
|
32614
33142
|
}
|
|
32615
33143
|
// packages/core/src/commands/analytics-site.ts
|
|
32616
33144
|
const STREAM_CHECKS = 30;
|
|
32617
|
-
const AT_ONCE = 5;
|
|
33145
|
+
const AT_ONCE$1 = 5;
|
|
32618
33146
|
const TIED = 3;
|
|
32619
33147
|
const hostOf$1 = (site) => {
|
|
32620
33148
|
const value = site.replace(/^sc-domain:/, "");
|
|
@@ -32679,8 +33207,8 @@ async function lookProperty(access, site) {
|
|
|
32679
33207
|
}
|
|
32680
33208
|
const order = ranked(site, candidates).slice(0, STREAM_CHECKS);
|
|
32681
33209
|
let failures = 0;
|
|
32682
|
-
for (let start = 0; start < order.length; start += AT_ONCE) {
|
|
32683
|
-
const batch = order.slice(start, start + AT_ONCE);
|
|
33210
|
+
for (let start = 0; start < order.length; start += AT_ONCE$1) {
|
|
33211
|
+
const batch = order.slice(start, start + AT_ONCE$1);
|
|
32684
33212
|
const streams = await Promise.all(batch.map((candidate) => listAnalyticsDataStreams(candidate.google, candidate.property, { signIn: access.signIn }).catch(() => {
|
|
32685
33213
|
failures += 1;
|
|
32686
33214
|
return [];
|
|
@@ -32881,7 +33409,7 @@ async function totals$1(found, access, range, before, pages) {
|
|
|
32881
33409
|
notes
|
|
32882
33410
|
};
|
|
32883
33411
|
}
|
|
32884
|
-
const change$
|
|
33412
|
+
const change$2 = (value) => value === null ? "n/a" : `${value > 0 ? "+" : ""}${value}%`;
|
|
32885
33413
|
function engineText(rows) {
|
|
32886
33414
|
const shown = rows.filter((row) => row.sessions || row.previousSessions);
|
|
32887
33415
|
if (shown.length < 2) return "";
|
|
@@ -32891,7 +33419,7 @@ function aiText(ai) {
|
|
|
32891
33419
|
const list = (rows, key) => rows.slice(0, AI_ROWS).map((row) => `${row[key]} ${row.sessions} (was ${row.previousSessions})`).join(", ");
|
|
32892
33420
|
if (!ai.sessions && !ai.previousSessions) return "Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): none in either period.";
|
|
32893
33421
|
return [
|
|
32894
|
-
`Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): ${ai.sessions} sessions (was ${ai.previousSessions}, ${change$
|
|
33422
|
+
`Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): ${ai.sessions} sessions (was ${ai.previousSessions}, ${change$2(ai.sessionsChangePct)}). By source: ${list(ai.bySource, "source")}.`,
|
|
32895
33423
|
ai.topLandingPages.length ? `Top landing pages from AI assistants: ${list(ai.topLandingPages, "page")}.` : "",
|
|
32896
33424
|
"Only visits that pass a referrer or utm_source count; AI apps that send none show as Direct."
|
|
32897
33425
|
].filter(Boolean).join(" ");
|
|
@@ -33008,7 +33536,7 @@ async function analyticsBeside(access, site, range, before, pages) {
|
|
|
33008
33536
|
const keyEvents = organic.keyEvents || organic.previousKeyEvents ? `, Organic Search key events ${organic.keyEvents} (was ${organic.previousKeyEvents})` : "";
|
|
33009
33537
|
return {
|
|
33010
33538
|
text: [
|
|
33011
|
-
`Google Analytics, property ${data.propertyName} (${data.property}), matched by its web stream ${data.webStream}${pages ? `, landing pages whose path contains "${pages}"` : ""}: Organic Search sessions from Google ${google.sessions} (was ${google.previousSessions}, ${change$
|
|
33539
|
+
`Google Analytics, property ${data.propertyName} (${data.property}), matched by its web stream ${data.webStream}${pages ? `, landing pages whose path contains "${pages}"` : ""}: Organic Search sessions from Google ${google.sessions} (was ${google.previousSessions}, ${change$2(google.sessionsChangePct)}), from every search engine ${organic.sessions} (was ${organic.previousSessions}, ${change$2(organic.sessionsChangePct)})${keyEvents}, sessions from all channels ${all.sessions} (was ${all.previousSessions}, ${change$2(all.sessionsChangePct)}).`,
|
|
33012
33540
|
engineText(data.organicSearchByEngine),
|
|
33013
33541
|
"Sessions are Google Analytics’ own count, not Search Console clicks.",
|
|
33014
33542
|
...data.notes,
|
|
@@ -33235,7 +33763,7 @@ function auditPreview(snapshot) {
|
|
|
33235
33763
|
}
|
|
33236
33764
|
const WEEK_MS = 6048e5;
|
|
33237
33765
|
const day$1 = (ms) => ms === null ? "" : new Date(ms).toISOString().slice(0, 10);
|
|
33238
|
-
const count$
|
|
33766
|
+
const count$3 = (value) => value.toLocaleString("en-US");
|
|
33239
33767
|
function relative$2(value, origin) {
|
|
33240
33768
|
if (!value) return "";
|
|
33241
33769
|
return origin && value.startsWith(origin) ? value.slice(origin.length) || "/" : value;
|
|
@@ -33256,7 +33784,7 @@ function rowsTable(rows, checks) {
|
|
|
33256
33784
|
changed(row) ? row.prev_coverage ?? "" : "",
|
|
33257
33785
|
changed(row) ? day$1(row.changed_at) : "",
|
|
33258
33786
|
day$1(row.checked_at),
|
|
33259
|
-
`${count$
|
|
33787
|
+
`${count$3(row.clicks)} (${count$3(row.previous_clicks)})${row.lost ? " lost" : ""}`,
|
|
33260
33788
|
row.in_sitemap ? "yes" : "no",
|
|
33261
33789
|
row.google_canonical && row.google_canonical !== row.url ? relative$2(row.google_canonical, checks.origin) : "",
|
|
33262
33790
|
row.error ?? ""
|
|
@@ -33264,12 +33792,12 @@ function rowsTable(rows, checks) {
|
|
|
33264
33792
|
}
|
|
33265
33793
|
function head(site, checks) {
|
|
33266
33794
|
const { queued } = checks;
|
|
33267
|
-
const quota = checks.properties.map((item) => `${item.property} (${count$
|
|
33795
|
+
const quota = checks.properties.map((item) => `${item.property} (${count$3(item.urls)} URLs: ${count$3(item.top)} top pages checked daily, the rest every ${item.cycle_days === 1 ? "day" : `${item.cycle_days} days`}; ${count$3(item.used_today)} of ${count$3(checks.per_day)} checks used today)`).join(", ");
|
|
33268
33796
|
return [
|
|
33269
33797
|
`Google indexing of ${site}, from the URL Inspection results this server keeps for it (as of ${new Date(checks.as_of).toISOString().slice(0, 16).replace("T", " ")} UTC).`,
|
|
33270
|
-
checks.dropped_this_week || checks.returned_this_week ? `In the last 7 days ${count$
|
|
33271
|
-
`Covered: ${count$
|
|
33272
|
-
queued.total ? `Queued: ${count$
|
|
33798
|
+
checks.dropped_this_week || checks.returned_this_week ? `In the last 7 days ${count$3(checks.dropped_this_week)} ${checks.dropped_this_week === 1 ? "URL" : "URLs"} dropped out of Google's index and ${count$3(checks.returned_this_week)} came back; they are listed first. A top page's drop counts only after a second check an hour later.` : null,
|
|
33799
|
+
`Covered: ${count$3(checks.covered)} of ${count$3(checks.known)} known URLs have a current result. Known URLs are the sitemap's, the crawl's and the Search Console pages with clicks or impressions. Each property spends its full ${count$3(checks.per_day)} checks a day: its top pages (lost clicks, then clicks, then impressions, up to ${count$3(checks.top_per_day)}) daily, the rest as often as the quota allows, and a page the weekly crawl saw change at once.`,
|
|
33800
|
+
queued.total ? `Queued: ${count$3(queued.total)} (${count$3(queued.lost)} pages that lost their clicks, ${count$3(queued.search)} with clicks or impressions, ${count$3(queued.sitemap)} sitemap URLs never checked, ${count$3(queued.stale)} older or changed), checked in that order. Time left: ${etaText(checks.eta_hours)}, at up to ${count$3(checks.per_day)} checks a day per property, spread over the day.` : "Queued: nothing; every known URL has a current result.",
|
|
33273
33801
|
quota ? `Checked through ${quota}.` : null,
|
|
33274
33802
|
checks.stopped ? `Checks stopped early: ${checks.stopped}` : null,
|
|
33275
33803
|
"A URL without a result has none here, so this is not an all-clear for it."
|
|
@@ -33278,9 +33806,9 @@ function head(site, checks) {
|
|
|
33278
33806
|
function suggestionsText(checks) {
|
|
33279
33807
|
if (!checks.suggestions.length) return null;
|
|
33280
33808
|
return [
|
|
33281
|
-
`The sitemap has more URLs than these properties can check every 7 days. Each Search Console property gets its own ${count$
|
|
33809
|
+
`The sitemap has more URLs than these properties can check every 7 days. Each Search Console property gets its own ${count$3(checks.per_day)} checks a day, so adding these folders as URL-prefix properties (on the same Google account) would check more:`,
|
|
33282
33810
|
"property to add | sitemap URLs in it | more URLs checked a week",
|
|
33283
|
-
...checks.suggestions.map((item) => `${item.property} | ${count$
|
|
33811
|
+
...checks.suggestions.map((item) => `${item.property} | ${count$3(item.urls)} | ${count$3(item.gain)}`),
|
|
33284
33812
|
"The next day the checks find a new property in Search Console and use its quota for the URLs in that folder."
|
|
33285
33813
|
].join("\n");
|
|
33286
33814
|
}
|
|
@@ -33295,7 +33823,7 @@ function runAuditChecks(input, checks, call) {
|
|
|
33295
33823
|
data: checks
|
|
33296
33824
|
};
|
|
33297
33825
|
if (!checks.started) return {
|
|
33298
|
-
text: `${site} is in the weekly site audit, and its Google checks are starting now: pages that lost their clicks first, then pages by clicks and impressions, then sitemap URLs. ${checks.known ? `${count$
|
|
33826
|
+
text: `${site} is in the weekly site audit, and its Google checks are starting now: pages that lost their clicks first, then pages by clicks and impressions, then sitemap URLs. ${checks.known ? `${count$3(checks.known)} URLs are known so far. ` : ""}Run ${call("index", { site })} again in a few minutes for the coverage and the time left.`,
|
|
33299
33827
|
data: checks
|
|
33300
33828
|
};
|
|
33301
33829
|
const more = (offset) => input.paged ? `More: ${call("index", {
|
|
@@ -33319,7 +33847,7 @@ function runAuditChecks(input, checks, call) {
|
|
|
33319
33847
|
const reasons = checks.reasons.length ? [
|
|
33320
33848
|
"Google's reason at each URL's latest check:",
|
|
33321
33849
|
"reason | verdict | urls | changed to it this week",
|
|
33322
|
-
...checks.reasons.map((item) => `${item.coverage ?? ""} | ${item.verdict ?? ""} | ${count$
|
|
33850
|
+
...checks.reasons.map((item) => `${item.coverage ?? ""} | ${item.verdict ?? ""} | ${count$3(item.urls)} | ${count$3(item.changed_this_week)}`)
|
|
33323
33851
|
].join("\n") : "No URL has a result yet.";
|
|
33324
33852
|
const shown = checks.rows.slice(0, 20);
|
|
33325
33853
|
return {
|
|
@@ -33376,7 +33904,7 @@ async function bingFor(deps, property) {
|
|
|
33376
33904
|
missing: site ? `${site.url} is in the connected Bing account but is not verified, so Bing gives no data for it. Verify it at ${BING_HOME}.` : `The connected Bing account does not have ${property.replace(/^sc-domain:/, "")}. Add and verify it at ${BING_HOME} (it can import sites from Search Console). New sites take a few days to show data.`
|
|
33377
33905
|
};
|
|
33378
33906
|
}
|
|
33379
|
-
function days(range) {
|
|
33907
|
+
function days$1(range) {
|
|
33380
33908
|
return Math.round((Date.parse(range.endDate) - Date.parse(range.startDate)) / 864e5) + 1;
|
|
33381
33909
|
}
|
|
33382
33910
|
async function bingTotals(source, range, before) {
|
|
@@ -33389,7 +33917,7 @@ async function bingTotals(source, range, before) {
|
|
|
33389
33917
|
clicks: rows.reduce((total, row) => total + row.clicks, 0),
|
|
33390
33918
|
impressions: rows.reduce((total, row) => total + row.impressions, 0),
|
|
33391
33919
|
days: new Set(rows.map((row) => row.date)).size,
|
|
33392
|
-
expectedDays: days(window)
|
|
33920
|
+
expectedDays: days$1(window)
|
|
33393
33921
|
};
|
|
33394
33922
|
};
|
|
33395
33923
|
return {
|
|
@@ -33399,14 +33927,14 @@ async function bingTotals(source, range, before) {
|
|
|
33399
33927
|
invalidRows: traffic.invalidRows
|
|
33400
33928
|
};
|
|
33401
33929
|
}
|
|
33402
|
-
const change = (now, before) => {
|
|
33930
|
+
const change$1 = (now, before) => {
|
|
33403
33931
|
const value = pct(now, before);
|
|
33404
33932
|
return value === null ? "n/a" : `${value > 0 ? "+" : ""}${value}%`;
|
|
33405
33933
|
};
|
|
33406
33934
|
function enginesText(google, bing) {
|
|
33407
33935
|
if ("missing" in bing) return `Bing: ${bing.missing}`;
|
|
33408
33936
|
const { current: now, previous: then } = bing;
|
|
33409
|
-
const lines = [`Search engines, same dates. Google (Search Console): clicks ${google.clicks} (was ${google.previousClicks}, ${change(google.clicks, google.previousClicks)}), impressions ${google.impressions}. Bing (${bing.site}): clicks ${now.clicks} (was ${then.clicks}, ${change(now.clicks, then.clicks)}), impressions ${now.impressions} (was ${then.impressions}).`];
|
|
33937
|
+
const lines = [`Search engines, same dates. Google (Search Console): clicks ${google.clicks} (was ${google.previousClicks}, ${change$1(google.clicks, google.previousClicks)}), impressions ${google.impressions}. Bing (${bing.site}): clicks ${now.clicks} (was ${then.clicks}, ${change$1(now.clicks, then.clicks)}), impressions ${now.impressions} (was ${then.impressions}).`];
|
|
33410
33938
|
const both = google.clicks + now.clicks;
|
|
33411
33939
|
if (both > 0) lines.push(`Share of these clicks: Google ${Math.round(google.clicks / both * 100)}%, Bing ${Math.round(now.clicks / both * 100)}%.`);
|
|
33412
33940
|
if (now.days < now.expectedDays || then.days < then.expectedDays) lines.push(`Bing has data for ${now.days} of ${now.expectedDays} days (earlier period ${then.days} of ${then.expectedDays}), so its totals are partial.`);
|
|
@@ -33560,7 +34088,7 @@ async function runBingRows(input, deps) {
|
|
|
33560
34088
|
}
|
|
33561
34089
|
};
|
|
33562
34090
|
}
|
|
33563
|
-
function moved(rows, range, before, name, limit) {
|
|
34091
|
+
function moved$1(rows, range, before, name, limit) {
|
|
33564
34092
|
const now = sumWeeks(rows, range);
|
|
33565
34093
|
const then = sumWeeks(rows, before);
|
|
33566
34094
|
const old = new Map(then.rows.map((row) => [row.key, row]));
|
|
@@ -33598,8 +34126,8 @@ async function bingOverview(source, range, before) {
|
|
|
33598
34126
|
]);
|
|
33599
34127
|
if ("missing" in totals) throw new CommandError("NOT_CONNECTED", totals.missing);
|
|
33600
34128
|
const { current: now, previous: then } = totals;
|
|
33601
|
-
const q = moved(queries.rows, range, before, "query", 15);
|
|
33602
|
-
const p = moved(pages.rows, range, before, "page", 15);
|
|
34129
|
+
const q = moved$1(queries.rows, range, before, "query", 15);
|
|
34130
|
+
const p = moved$1(pages.rows, range, before, "page", 15);
|
|
33603
34131
|
return {
|
|
33604
34132
|
bingSite: source.site,
|
|
33605
34133
|
range,
|
|
@@ -33650,7 +34178,7 @@ function bingOverviewText(report, heading, origin, rowsPerList) {
|
|
|
33650
34178
|
};
|
|
33651
34179
|
return [
|
|
33652
34180
|
`${heading}, ${range.startDate} to ${range.endDate} against ${before.startDate} to ${before.endDate}.`,
|
|
33653
|
-
t.days || t.previousDays ? `Bing totals: clicks ${t.clicks} (was ${t.previousClicks}, ${change(t.clicks, t.previousClicks)}), impressions ${t.impressions} (was ${t.previousImpressions}, ${change(t.impressions, t.previousImpressions)}).` : "Bing totals: unknown. Bing has no daily data for these dates.",
|
|
34181
|
+
t.days || t.previousDays ? `Bing totals: clicks ${t.clicks} (was ${t.previousClicks}, ${change$1(t.clicks, t.previousClicks)}), impressions ${t.impressions} (was ${t.previousImpressions}, ${change$1(t.impressions, t.previousImpressions)}).` : "Bing totals: unknown. Bing has no daily data for these dates.",
|
|
33654
34182
|
group("pages", "page"),
|
|
33655
34183
|
group("queries", "query"),
|
|
33656
34184
|
`Limits:\n${report.caveats.map((line) => `- ${line}`).join("\n")}`
|
|
@@ -33727,13 +34255,13 @@ function clickyRange(input) {
|
|
|
33727
34255
|
endDate: input.endDate
|
|
33728
34256
|
};
|
|
33729
34257
|
}
|
|
33730
|
-
const count$
|
|
34258
|
+
const count$2 = (item) => {
|
|
33731
34259
|
const value = Number(item.value);
|
|
33732
34260
|
return Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
33733
34261
|
};
|
|
33734
34262
|
const totals = (items) => items.map((item) => ({
|
|
33735
34263
|
name: item.title ?? "",
|
|
33736
|
-
visits: count$
|
|
34264
|
+
visits: count$2(item)
|
|
33737
34265
|
})).filter((item) => Boolean(item.name) && item.visits !== null);
|
|
33738
34266
|
const line = (items) => {
|
|
33739
34267
|
const top = items.slice(0, 10).map((item) => `${item.name} ${item.visits}`);
|
|
@@ -33764,7 +34292,7 @@ async function runClickyRows(input, keys, options) {
|
|
|
33764
34292
|
const entrances = stats["pages-entrance"] ?? [];
|
|
33765
34293
|
const visitsByPage = /* @__PURE__ */ new Map();
|
|
33766
34294
|
for (const item of entrances) {
|
|
33767
|
-
const visits = count$
|
|
34295
|
+
const visits = count$2(item);
|
|
33768
34296
|
if (visits === null || !item.url) continue;
|
|
33769
34297
|
let page;
|
|
33770
34298
|
try {
|
|
@@ -33813,7 +34341,8 @@ async function runClickyRows(input, keys, options) {
|
|
|
33813
34341
|
// packages/core/src/commands/domain-rating.ts
|
|
33814
34342
|
const DR_ATTRIBUTION = "domain_rating: Domain Rating by Ahrefs (https://ahrefs.com/).";
|
|
33815
34343
|
const ENDPOINT = "https://api.ahrefs.com/v3/public/domain-rating-free";
|
|
33816
|
-
const MAX_DOMAINS =
|
|
34344
|
+
const MAX_DOMAINS = 50;
|
|
34345
|
+
const AT_ONCE = 25;
|
|
33817
34346
|
const TTL_SECONDS = 86400;
|
|
33818
34347
|
const DOMAIN_FIELDS = [
|
|
33819
34348
|
"domain",
|
|
@@ -33844,23 +34373,26 @@ async function lookup(domain, key) {
|
|
|
33844
34373
|
}
|
|
33845
34374
|
return body.domain_rating?.domain_rating ?? null;
|
|
33846
34375
|
}
|
|
33847
|
-
async function domainRatings(domains, key) {
|
|
34376
|
+
async function domainRatings(domains, key, max = MAX_DOMAINS) {
|
|
33848
34377
|
const found = /* @__PURE__ */ new Map();
|
|
33849
34378
|
if (!key) return found;
|
|
33850
|
-
const unique = [...new Set(domains.map(bareDomain).filter(Boolean))].slice(0,
|
|
33851
|
-
|
|
33852
|
-
|
|
33853
|
-
|
|
33854
|
-
|
|
34379
|
+
const unique = [...new Set(domains.map(bareDomain).filter(Boolean))].slice(0, max);
|
|
34380
|
+
for (let start = 0; start < unique.length; start += AT_ONCE) {
|
|
34381
|
+
const batch = unique.slice(start, start + AT_ONCE);
|
|
34382
|
+
const values = await Promise.all(batch.map((domain) => lookup(domain, key).catch(() => null)));
|
|
34383
|
+
batch.forEach((domain, index) => {
|
|
34384
|
+
found.set(domain, values[index] ?? null);
|
|
34385
|
+
});
|
|
34386
|
+
}
|
|
33855
34387
|
return found;
|
|
33856
34388
|
}
|
|
33857
|
-
async function withDomainRating(rows, key) {
|
|
34389
|
+
async function withDomainRating(rows, key, start = 0) {
|
|
33858
34390
|
const field = DOMAIN_FIELDS.find((name) => rows.some((row) => typeof row[name] === "string"));
|
|
33859
34391
|
if (!key || !field || rows.some((row) => "domain_rating" in row)) return {
|
|
33860
34392
|
rows,
|
|
33861
34393
|
added: false
|
|
33862
34394
|
};
|
|
33863
|
-
const ratings = await domainRatings(rows.map((row) => String(row[field] ?? "")), key);
|
|
34395
|
+
const ratings = await domainRatings(rows.slice(start).map((row) => String(row[field] ?? "")), key);
|
|
33864
34396
|
return {
|
|
33865
34397
|
rows: rows.map((row) => {
|
|
33866
34398
|
const next = {};
|
|
@@ -35035,6 +35567,265 @@ function temporaryRedirectNote(pages) {
|
|
|
35035
35567
|
if (!temporary.length) return null;
|
|
35036
35568
|
return `Note: 302 and 307 are temporary redirects; a permanent move usually uses 301 or 308. Temporary redirects here: ${temporary.map((page) => `${page.url} (${page.status} to ${page.redirects_to})`).join("; ")}.`;
|
|
35037
35569
|
}
|
|
35570
|
+
// packages/core/src/commands/updates.ts
|
|
35571
|
+
const UPDATES_SECTION = "google-updates";
|
|
35572
|
+
const UPDATES_SECTION_TEXT = "One Google ranking update (core, spam and similar, from Google’s Search Status Dashboard) laid over the site’s Search Console data: clicks and impressions from the day it started against the same number of days just before (whole weeks, up to 28 days), whether that change is outside the site’s own movement in the 8 comparisons before it (a heuristic), and the folders, pages and queries that gained or lost most. The latest update by default; with end_date, the latest update that started on or before it. Timing only, not cause.";
|
|
35573
|
+
const MAX_DAYS = 28;
|
|
35574
|
+
const BASELINE = 8;
|
|
35575
|
+
const ROWS = 25e3;
|
|
35576
|
+
const LISTED = 10;
|
|
35577
|
+
const SHOWN = 5;
|
|
35578
|
+
const MIN_CLICKS = 50;
|
|
35579
|
+
const days = (from, to) => Math.round((Date.parse(to) - Date.parse(from)) / 864e5) + 1;
|
|
35580
|
+
const asImpactUpdate = (update) => ({
|
|
35581
|
+
id: update.id,
|
|
35582
|
+
name: update.name,
|
|
35583
|
+
start: update.start,
|
|
35584
|
+
end: update.end ?? null,
|
|
35585
|
+
url: update.sourceUrl
|
|
35586
|
+
});
|
|
35587
|
+
function sum(rows, from, to) {
|
|
35588
|
+
let clicks = 0;
|
|
35589
|
+
let impressions = 0;
|
|
35590
|
+
for (const row of rows) {
|
|
35591
|
+
const day = row.keys[0] ?? "";
|
|
35592
|
+
if (day < from || day > to) continue;
|
|
35593
|
+
clicks += row.clicks;
|
|
35594
|
+
impressions += row.impressions;
|
|
35595
|
+
}
|
|
35596
|
+
return {
|
|
35597
|
+
clicks,
|
|
35598
|
+
impressions
|
|
35599
|
+
};
|
|
35600
|
+
}
|
|
35601
|
+
const change = (now, before) => before > 0 ? (now - before) / before * 100 : null;
|
|
35602
|
+
function joined(before, after) {
|
|
35603
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
35604
|
+
const at = (key) => {
|
|
35605
|
+
let row = byKey.get(key);
|
|
35606
|
+
if (!row) {
|
|
35607
|
+
row = {
|
|
35608
|
+
key,
|
|
35609
|
+
clicks: 0,
|
|
35610
|
+
previous: 0,
|
|
35611
|
+
impressions: 0,
|
|
35612
|
+
previous_impressions: 0
|
|
35613
|
+
};
|
|
35614
|
+
byKey.set(key, row);
|
|
35615
|
+
}
|
|
35616
|
+
return row;
|
|
35617
|
+
};
|
|
35618
|
+
for (const row of before) {
|
|
35619
|
+
const item = at(row.keys[0] ?? "");
|
|
35620
|
+
item.previous += row.clicks;
|
|
35621
|
+
item.previous_impressions += row.impressions;
|
|
35622
|
+
}
|
|
35623
|
+
for (const row of after) {
|
|
35624
|
+
const item = at(row.keys[0] ?? "");
|
|
35625
|
+
item.clicks += row.clicks;
|
|
35626
|
+
item.impressions += row.impressions;
|
|
35627
|
+
}
|
|
35628
|
+
return [...byKey.values()];
|
|
35629
|
+
}
|
|
35630
|
+
const moved = (row) => row.clicks - row.previous;
|
|
35631
|
+
const byKey = (a, b) => a.key < b.key ? -1 : a.key > b.key ? 1 : 0;
|
|
35632
|
+
function extremes(rows) {
|
|
35633
|
+
return {
|
|
35634
|
+
lost: rows.filter((row) => moved(row) < 0).sort((a, b) => moved(a) - moved(b) || byKey(a, b)).slice(0, LISTED),
|
|
35635
|
+
gained: rows.filter((row) => moved(row) > 0).sort((a, b) => moved(b) - moved(a) || byKey(a, b)).slice(0, LISTED)
|
|
35636
|
+
};
|
|
35637
|
+
}
|
|
35638
|
+
function folderOf(url) {
|
|
35639
|
+
try {
|
|
35640
|
+
const parts = new URL(url).pathname.split("/").filter(Boolean);
|
|
35641
|
+
return parts.length > 1 ? `/${parts[0]}/` : "/";
|
|
35642
|
+
} catch {
|
|
35643
|
+
return "/";
|
|
35644
|
+
}
|
|
35645
|
+
}
|
|
35646
|
+
function folders(pages) {
|
|
35647
|
+
const byFolder = /* @__PURE__ */ new Map();
|
|
35648
|
+
for (const page of pages) {
|
|
35649
|
+
const key = folderOf(page.key);
|
|
35650
|
+
const row = byFolder.get(key) ?? {
|
|
35651
|
+
key,
|
|
35652
|
+
clicks: 0,
|
|
35653
|
+
previous: 0,
|
|
35654
|
+
impressions: 0,
|
|
35655
|
+
previous_impressions: 0
|
|
35656
|
+
};
|
|
35657
|
+
row.clicks += page.clicks;
|
|
35658
|
+
row.previous += page.previous;
|
|
35659
|
+
row.impressions += page.impressions;
|
|
35660
|
+
row.previous_impressions += page.previous_impressions;
|
|
35661
|
+
byFolder.set(key, row);
|
|
35662
|
+
}
|
|
35663
|
+
return [...byFolder.values()].filter((row) => moved(row) !== 0).sort((a, b) => Math.abs(moved(b)) - Math.abs(moved(a)) || byKey(a, b)).slice(0, LISTED);
|
|
35664
|
+
}
|
|
35665
|
+
function windowDays(update, latest) {
|
|
35666
|
+
const start = update.start.slice(0, 10);
|
|
35667
|
+
const end = update.end?.slice(0, 10) ?? null;
|
|
35668
|
+
const wanted = Math.min(MAX_DAYS, Math.max(7, Math.ceil(days(start, end ?? latest) / 7) * 7));
|
|
35669
|
+
const covered = Math.floor(Math.max(0, days(start, latest)) / 7) * 7;
|
|
35670
|
+
return {
|
|
35671
|
+
start,
|
|
35672
|
+
end,
|
|
35673
|
+
wanted,
|
|
35674
|
+
n: Math.min(wanted, covered)
|
|
35675
|
+
};
|
|
35676
|
+
}
|
|
35677
|
+
async function updateImpact(site, update, read, now) {
|
|
35678
|
+
const { start, end, wanted, n } = windowDays(update, latestFinalGscDate(now));
|
|
35679
|
+
const empty = {
|
|
35680
|
+
update,
|
|
35681
|
+
site,
|
|
35682
|
+
before: null,
|
|
35683
|
+
after: null,
|
|
35684
|
+
normal_range: null,
|
|
35685
|
+
verdict: "not-enough-data",
|
|
35686
|
+
capped: {
|
|
35687
|
+
pages: false,
|
|
35688
|
+
queries: false
|
|
35689
|
+
},
|
|
35690
|
+
folders: [],
|
|
35691
|
+
pages: {
|
|
35692
|
+
lost: [],
|
|
35693
|
+
gained: []
|
|
35694
|
+
},
|
|
35695
|
+
queries: {
|
|
35696
|
+
lost: [],
|
|
35697
|
+
gained: []
|
|
35698
|
+
}
|
|
35699
|
+
};
|
|
35700
|
+
if (n < 7) return {
|
|
35701
|
+
...empty,
|
|
35702
|
+
window: "waiting"
|
|
35703
|
+
};
|
|
35704
|
+
const after = {
|
|
35705
|
+
startDate: start,
|
|
35706
|
+
endDate: shiftIsoDate(start, n - 1)
|
|
35707
|
+
};
|
|
35708
|
+
const before = {
|
|
35709
|
+
startDate: shiftIsoDate(start, -n),
|
|
35710
|
+
endDate: shiftIsoDate(start, -1)
|
|
35711
|
+
};
|
|
35712
|
+
const first = Math.ceil(n / 7);
|
|
35713
|
+
const pairs = Array.from({ length: BASELINE }, (_, i) => {
|
|
35714
|
+
const back = 7 * (first + i);
|
|
35715
|
+
return {
|
|
35716
|
+
after: [shiftIsoDate(start, -back), shiftIsoDate(start, n - 1 - back)],
|
|
35717
|
+
before: [shiftIsoDate(start, -back - n), shiftIsoDate(start, -back - 1)]
|
|
35718
|
+
};
|
|
35719
|
+
});
|
|
35720
|
+
const shared = sharedReader(read);
|
|
35721
|
+
const call = (range, dimension) => shared(site, {
|
|
35722
|
+
...range,
|
|
35723
|
+
dimensions: [dimension],
|
|
35724
|
+
type: "web",
|
|
35725
|
+
dataState: "final",
|
|
35726
|
+
...dimension === "date" ? {} : { maxRows: ROWS }
|
|
35727
|
+
});
|
|
35728
|
+
const [daily, pagesBefore, pagesAfter, queriesBefore, queriesAfter] = await Promise.all([
|
|
35729
|
+
call({
|
|
35730
|
+
startDate: pairs.at(-1)?.before[0],
|
|
35731
|
+
endDate: after.endDate
|
|
35732
|
+
}, "date"),
|
|
35733
|
+
call(before, "page"),
|
|
35734
|
+
call(after, "page"),
|
|
35735
|
+
call(before, "query"),
|
|
35736
|
+
call(after, "query")
|
|
35737
|
+
]);
|
|
35738
|
+
const totals = (range) => ({
|
|
35739
|
+
...range,
|
|
35740
|
+
...sum(daily.rows, range.startDate, range.endDate)
|
|
35741
|
+
});
|
|
35742
|
+
const was = totals(before);
|
|
35743
|
+
const is = totals(after);
|
|
35744
|
+
const seen = pairs.map((pair) => change(sum(daily.rows, pair.after[0], pair.after[1]).clicks, sum(daily.rows, pair.before[0], pair.before[1]).clicks)).filter((value) => value !== null);
|
|
35745
|
+
const normal = seen.length ? {
|
|
35746
|
+
comparisons: seen.length,
|
|
35747
|
+
clicks_pct: Math.round(Math.max(...seen.map((v) => Math.abs(v))) * 10) / 10
|
|
35748
|
+
} : null;
|
|
35749
|
+
const moves = change(is.clicks, was.clicks);
|
|
35750
|
+
const verdict = !normal || normal.comparisons < BASELINE / 2 || was.clicks < MIN_CLICKS || moves === null ? "not-enough-data" : Math.abs(moves) > normal.clicks_pct ? "outside-normal" : "within-normal";
|
|
35751
|
+
const pageRows = joined(pagesBefore.rows, pagesAfter.rows);
|
|
35752
|
+
return {
|
|
35753
|
+
...empty,
|
|
35754
|
+
window: end && n === wanted ? "full" : "partial",
|
|
35755
|
+
before: was,
|
|
35756
|
+
after: is,
|
|
35757
|
+
normal_range: normal,
|
|
35758
|
+
verdict,
|
|
35759
|
+
capped: {
|
|
35760
|
+
pages: pagesBefore.rows.length >= ROWS || pagesAfter.rows.length >= ROWS,
|
|
35761
|
+
queries: queriesBefore.rows.length >= ROWS || queriesAfter.rows.length >= ROWS
|
|
35762
|
+
},
|
|
35763
|
+
folders: folders(pageRows),
|
|
35764
|
+
pages: extremes(pageRows),
|
|
35765
|
+
queries: extremes(joined(queriesBefore.rows, queriesAfter.rows))
|
|
35766
|
+
};
|
|
35767
|
+
}
|
|
35768
|
+
const count$1 = (n) => n.toLocaleString("en-US");
|
|
35769
|
+
const dayLabel$1 = (date) => (/* @__PURE__ */ new Date(`${date}T00:00:00Z`)).toLocaleDateString("en-GB", {
|
|
35770
|
+
day: "numeric",
|
|
35771
|
+
month: "short",
|
|
35772
|
+
year: "numeric",
|
|
35773
|
+
timeZone: "UTC"
|
|
35774
|
+
});
|
|
35775
|
+
function trend(now, before) {
|
|
35776
|
+
const pct = change(now, before);
|
|
35777
|
+
if (pct === null) return `from ${count$1(before)}`;
|
|
35778
|
+
const rounded = Math.round(pct);
|
|
35779
|
+
return rounded === 0 ? `level with ${count$1(before)}` : `${rounded > 0 ? "up" : "down"} ${Math.abs(rounded)}% on ${count$1(before)}`;
|
|
35780
|
+
}
|
|
35781
|
+
function rowText(row, origin) {
|
|
35782
|
+
const name = row.key.startsWith(origin) ? row.key.slice(origin.length) || "/" : row.key;
|
|
35783
|
+
const diff = moved(row);
|
|
35784
|
+
return `${name} ${count$1(row.clicks)} (${diff > 0 ? "+" : ""}${count$1(diff)})`;
|
|
35785
|
+
}
|
|
35786
|
+
function impactLines(impact) {
|
|
35787
|
+
const { update, before, after } = impact;
|
|
35788
|
+
const origin = originOf(impact.site);
|
|
35789
|
+
const dates = `${dayLabel$1(update.start.slice(0, 10))} to ${update.end ? dayLabel$1(update.end.slice(0, 10)) : "now, still rolling out"}`;
|
|
35790
|
+
const lines = [`${update.name} (${dates}). [Google’s notice](${update.url})`];
|
|
35791
|
+
if (!before || !after) return [...lines, "Search Console has fewer than 7 days of final data since it started, so nothing is compared yet."];
|
|
35792
|
+
const n = days(after.startDate, after.endDate);
|
|
35793
|
+
lines.push(`Compared: ${dayLabel$1(after.startDate)} to ${dayLabel$1(after.endDate)} against the ${n} days before (same weekdays)${update.end && update.end.slice(0, 10) > after.endDate && impact.window === "full" ? `, the first ${n} days of the update` : ""}.${impact.window === "partial" ? " Final data does not cover the whole update yet, so this is part of it." : ""}`, `Clicks ${count$1(after.clicks)}, ${trend(after.clicks, before.clicks)}. Impressions ${count$1(after.impressions)}, ${trend(after.impressions, before.impressions)}.`);
|
|
35794
|
+
const normal = impact.normal_range;
|
|
35795
|
+
lines.push(impact.verdict === "not-enough-data" ? `Not enough data to say whether this is unusual for the site${before.clicks < MIN_CLICKS ? ` (${count$1(before.clicks)} clicks before the update)` : ""}.` : `${impact.verdict === "outside-normal" ? "Outside" : "Within"} the site’s normal movement: in the ${normal?.comparisons} earlier ${n}-day comparisons, clicks moved by at most ${normal?.clicks_pct}% (a heuristic).`);
|
|
35796
|
+
const list = (title, rows) => rows.length ? [`${title}: ${rows.slice(0, SHOWN).map((row) => rowText(row, origin)).join("; ")}.`] : [];
|
|
35797
|
+
lines.push(...list("Folders that moved most, by clicks", impact.folders), ...list("Pages that lost most", impact.pages.lost), ...list("Pages that gained most", impact.pages.gained), ...list("Queries that lost most", impact.queries.lost), ...list("Queries that gained most", impact.queries.gained));
|
|
35798
|
+
const capped = [impact.capped.pages ? "pages" : null, impact.capped.queries ? "queries" : null].filter(Boolean);
|
|
35799
|
+
if (capped.length) lines.push(`Search Console returned the row limit for ${capped.join(" and ")}, so those lists may miss rows.`);
|
|
35800
|
+
lines.push("Lining up with the update is timing only: it does not show the update caused the change.");
|
|
35801
|
+
return lines;
|
|
35802
|
+
}
|
|
35803
|
+
async function runUpdates(input, deps) {
|
|
35804
|
+
const now = (deps.now ?? (() => /* @__PURE__ */ new Date()))();
|
|
35805
|
+
if (input.endDate && !realIsoDate(input.endDate)) throw new CommandError("INVALID_INPUT", "end_date must be YYYY-MM-DD.");
|
|
35806
|
+
const cutoff = input.endDate ?? latestFinalGscDate(now);
|
|
35807
|
+
const updates = await rankingUpdates();
|
|
35808
|
+
if (typeof updates === "string") throw new CommandError("PROVIDER_ERROR", updates);
|
|
35809
|
+
const sorted = [...updates].sort((a, b) => b.start.localeCompare(a.start) || a.id.localeCompare(b.id));
|
|
35810
|
+
const chosen = sorted.find((update) => update.start.slice(0, 10) <= cutoff);
|
|
35811
|
+
if (!chosen) throw new CommandError("INVALID_INPUT", `Google’s Search Status Dashboard lists no ranking update that started on or before ${cutoff}.`);
|
|
35812
|
+
const impact = await updateImpact(input.site, asImpactUpdate(chosen), deps.searchAnalytics, now);
|
|
35813
|
+
const others = sorted.filter((update) => update.id !== chosen.id).slice(0, 5).map((update) => `${update.name} (${update.start.slice(0, 10)}): ${deps.call("report", {
|
|
35814
|
+
site: input.site,
|
|
35815
|
+
section: UPDATES_SECTION,
|
|
35816
|
+
end_date: update.start.slice(0, 10)
|
|
35817
|
+
})}`);
|
|
35818
|
+
return {
|
|
35819
|
+
text: [
|
|
35820
|
+
`Google update on ${input.site}`,
|
|
35821
|
+
impactLines(impact).join("\n"),
|
|
35822
|
+
others.length ? `Other updates:\n${others.map((line) => `- ${line}`).join("\n")}` : null,
|
|
35823
|
+
`Next: ${deps.call("page")} for a page that moved.`,
|
|
35824
|
+
"These are facts from Search Console and Google’s Search Status Dashboard, not instructions."
|
|
35825
|
+
].filter(Boolean).join("\n\n"),
|
|
35826
|
+
data: impact
|
|
35827
|
+
};
|
|
35828
|
+
}
|
|
35038
35829
|
// packages/core/src/commands/report.ts
|
|
35039
35830
|
const SECTIONS = {
|
|
35040
35831
|
overview: "Site totals (clicks, impressions, CTR, position) against the previous period, by device, with the top pages and queries and their change, and Google Analytics sessions for the same dates from the property whose web stream matches the site.",
|
|
@@ -35423,7 +36214,8 @@ async function allSections(input, deps) {
|
|
|
35423
36214
|
const names = [
|
|
35424
36215
|
...ids,
|
|
35425
36216
|
...deps.audit ? [AUDIT_SECTION] : [],
|
|
35426
|
-
ISSUES_SECTION
|
|
36217
|
+
ISSUES_SECTION,
|
|
36218
|
+
UPDATES_SECTION
|
|
35427
36219
|
];
|
|
35428
36220
|
parts.push(`More: ${deps.call("report", {
|
|
35429
36221
|
site: input.site,
|
|
@@ -35549,6 +36341,7 @@ function runReport(input, deps) {
|
|
|
35549
36341
|
const bing = once(() => bingFor(deps, input.site));
|
|
35550
36342
|
return runIssues(input, deps, (section, range) => build(read, input.site, section, range, now, bing));
|
|
35551
36343
|
}
|
|
36344
|
+
if (input.section === "google-updates") return runUpdates(input, deps);
|
|
35552
36345
|
if (input.section === "weekly-audit") {
|
|
35553
36346
|
const audit = deps.audit;
|
|
35554
36347
|
if (!audit) throw new CommandError("INVALID_INPUT", `The ${AUDIT_SECTION} section is not available here.`);
|
|
@@ -35587,10 +36380,11 @@ const RESEARCH_ACTIONS = [
|
|
|
35587
36380
|
"serp",
|
|
35588
36381
|
"competitors",
|
|
35589
36382
|
"backlinks",
|
|
36383
|
+
"link_gap",
|
|
35590
36384
|
"ai_answers"
|
|
35591
36385
|
];
|
|
35592
36386
|
const RESEARCH_PROVIDERS = ["dataforseo", "ahrefs"];
|
|
35593
|
-
const RESEARCH_HELP = `keyword_metrics (keywords: volume, difficulty, CPC), ranked_keywords (target: keywords a domain or URL ranks for), serp (keywords, up to 5: the ranking pages), competitors (target: domains that share keywords), backlinks (target: referring domains), ai_answers (DataForSEO only; target: the site. Without keywords it takes the site’s top 10 chat-style and long questions from Search Console’s AI-shaped queries itself. With up to 10 questions in keywords it asks those instead, and the target can be any domain, such as a competitor's. Each answer row also lists the other sites it cites, and the notes rank them. Answers come from DataForSEO's queue: answers not ready after about 40 seconds are returned as tasks, already paid for; call again with the same target and those tasks to read them at no cost. It asks ChatGPT search, Google AI Mode and Google AI Overviews each one and says whether the answer cites or mentions the site. DataForSEO’s queue prices are about $0.0072 a question across the three engines, so about $0.07 a run: tell the person before running it). Rows with a domain, and the target, get a free Ahrefs domain_rating column. Numbers are provider estimates, not Search Console data.`;
|
|
36387
|
+
const RESEARCH_HELP = `keyword_metrics (keywords: volume, difficulty, CPC), ranked_keywords (target: keywords a domain or URL ranks for), serp (keywords, up to 5: the ranking pages), competitors (target: domains that share keywords), backlinks (target: referring domains), link_gap (target and 2 to 5 competitors: referring domains that link to two or more competitors and not the target, highest authority first, from each site's top 1,000 referring domains; costs one backlinks call per site), ai_answers (DataForSEO only; target: the site. Without keywords it takes the site’s top 10 chat-style and long questions from Search Console’s AI-shaped queries itself. With up to 10 questions in keywords it asks those instead, and the target can be any domain, such as a competitor's. Each answer row also lists the other sites it cites, and the notes rank them. Answers come from DataForSEO's queue: answers not ready after about 40 seconds are returned as tasks, already paid for; call again with the same target and those tasks to read them at no cost. It asks ChatGPT search, Google AI Mode and Google AI Overviews each one and says whether the answer cites or mentions the site. DataForSEO’s queue prices are about $0.0072 a question across the three engines, so about $0.07 a run: tell the person before running it). Rows with a domain, and the target, get a free Ahrefs domain_rating column. Numbers are provider estimates, not Search Console data.`;
|
|
35594
36388
|
const GENERIC_PLATFORMS = [
|
|
35595
36389
|
"youtube.com",
|
|
35596
36390
|
"facebook.com",
|
|
@@ -35634,10 +36428,11 @@ async function runResearch(input, keys, options) {
|
|
|
35634
36428
|
const offset = input.offset ?? 0;
|
|
35635
36429
|
const language = input.provider === "dataforseo" ? dataForSeoLanguage(country) : null;
|
|
35636
36430
|
if (input.action === "ai_answers") throw new CommandError("INVALID_INPUT", "ai_answers runs on its own.");
|
|
35637
|
-
const { rows: found, costUsd, partial, missed } = await research(input.provider, keys, {
|
|
36431
|
+
const { rows: found, costUsd, partial, missed, noData, floor, leftOut } = await research(input.provider, keys, {
|
|
35638
36432
|
action: input.action,
|
|
35639
36433
|
keywords: input.keywords ?? [],
|
|
35640
36434
|
target,
|
|
36435
|
+
competitors: input.competitors ?? [],
|
|
35641
36436
|
country,
|
|
35642
36437
|
limit: Math.min(1e3, Math.max(100, offset + 50))
|
|
35643
36438
|
});
|
|
@@ -35645,8 +36440,10 @@ async function runResearch(input, keys, options) {
|
|
|
35645
36440
|
rows: found,
|
|
35646
36441
|
note: null
|
|
35647
36442
|
};
|
|
36443
|
+
const unresolved = rows.filter((row) => row.url === null && row.url_hint !== void 0).length;
|
|
35648
36444
|
const drKey = keys.get("x-ahrefs-key") ?? options.drKey;
|
|
35649
|
-
const
|
|
36445
|
+
const byDr = input.action === "link_gap" && input.provider === "dataforseo" && drKey;
|
|
36446
|
+
const rated = byDr ? await sortedByDomainRating(rows, drKey) : await withDomainRating(rows, drKey, offset);
|
|
35650
36447
|
const targetDr = target ? (await domainRatings([target], drKey)).get(bareDomain(target)) : void 0;
|
|
35651
36448
|
const hasTargetDr = targetDr !== void 0 && targetDr !== null;
|
|
35652
36449
|
return {
|
|
@@ -35661,6 +36458,11 @@ async function runResearch(input, keys, options) {
|
|
|
35661
36458
|
...costUsd !== null ? [`DataForSEO cost: $${costUsd}.`] : [],
|
|
35662
36459
|
...partial ? ["Partial: DataForSEO could not fetch some pages of results, so rows may be missing."] : [],
|
|
35663
36460
|
...missed.length ? [`Partial: ${input.provider} failed for ${missed.join(", ")}, so those keywords have no rows. Ask for them again.`] : [],
|
|
36461
|
+
...noData.length ? [noDataNote(input, noData)] : [],
|
|
36462
|
+
...byDr ? [rows.length > GAP_RATED ? `Sorted by Domain Rating for the top ${GAP_RATED} rows by DataForSEO rank; the ${rows.length - GAP_RATED} rows after them follow by rank, with no Domain Rating.` : "Sorted by Domain Rating, then DataForSEO rank."] : [],
|
|
36463
|
+
...leftOut?.length ? [`Left out ${leftOut.length} link lists, URL shorteners and directories that link to most sites (a guess from the host name): ${leftOut.join(", ")}.`] : [],
|
|
36464
|
+
...floor ? [`Cut at ${floor.field} ${floor.value}: at least one site has more than 1,000 referring domains, and only its top 1,000 were read, so domains below ${floor.field} ${floor.value} are left out rather than guessed.`] : [],
|
|
36465
|
+
...unresolved ? [unresolvedNote(input, unresolved, rows.length)] : [],
|
|
35664
36466
|
...note ? [note] : [],
|
|
35665
36467
|
...hasTargetDr ? [`${bareDomain(target)} domain_rating: ${targetDr}.`] : [],
|
|
35666
36468
|
...rated.added || hasTargetDr ? [DR_ATTRIBUTION] : []
|
|
@@ -35679,12 +36481,44 @@ async function runResearch(input, keys, options) {
|
|
|
35679
36481
|
costUsd,
|
|
35680
36482
|
partial,
|
|
35681
36483
|
missedKeywords: missed,
|
|
36484
|
+
noDataKeywords: noData,
|
|
36485
|
+
...input.action === "link_gap" ? {
|
|
36486
|
+
competitors: input.competitors ?? [],
|
|
36487
|
+
cutAt: floor ?? null,
|
|
36488
|
+
leftOut: leftOut ?? []
|
|
36489
|
+
} : {},
|
|
35682
36490
|
total: rated.rows.length,
|
|
35683
36491
|
offset,
|
|
35684
36492
|
rows: rated.rows.slice(offset, offset + 50)
|
|
35685
36493
|
}
|
|
35686
36494
|
};
|
|
35687
36495
|
}
|
|
36496
|
+
function noDataNote(input, keywords) {
|
|
36497
|
+
const list = keywords.join(", ");
|
|
36498
|
+
if (input.action === "serp") return `No results: ${input.provider} got an empty results page for ${list}${input.provider === "dataforseo" ? ", twice" : ""}. Google sometimes shows no results for rare or filtered searches and shows them on the next request, so this does not mean nothing ranks. Ask again later${input.provider === "dataforseo" ? ", or use provider ahrefs" : ""}.`;
|
|
36499
|
+
return `No data: ${input.provider} has no metrics for ${keywords.length} of the keywords asked for (${list}). Their volume is unknown, not zero.`;
|
|
36500
|
+
}
|
|
36501
|
+
function unresolvedNote(input, count, total) {
|
|
36502
|
+
return `${count} of ${total} results have no exact page: ${input.provider} returned Google redirect links for them. Their domain and url_hint come from Google's breadcrumb, which is shortened.${input.provider === "dataforseo" ? " For exact pages, use provider ahrefs." : ""}`;
|
|
36503
|
+
}
|
|
36504
|
+
const GAP_RATED = 200;
|
|
36505
|
+
async function sortedByDomainRating(rows, key) {
|
|
36506
|
+
const top = rows.slice(0, GAP_RATED);
|
|
36507
|
+
const ratings = await domainRatings(top.map((row) => String(row.domain ?? "")), key, GAP_RATED);
|
|
36508
|
+
const withDr = (row, rated) => {
|
|
36509
|
+
const { domain = null, ...rest } = row;
|
|
36510
|
+
return {
|
|
36511
|
+
domain,
|
|
36512
|
+
domain_rating: rated ? ratings.get(bareDomain(String(domain ?? ""))) ?? null : null,
|
|
36513
|
+
...rest
|
|
36514
|
+
};
|
|
36515
|
+
};
|
|
36516
|
+
const dr = (row) => Number(row.domain_rating ?? -1);
|
|
36517
|
+
return {
|
|
36518
|
+
rows: [...top.map((row) => withDr(row, true)).sort((a, b) => dr(b) - dr(a) || Number(b.rank ?? 0) - Number(a.rank ?? 0) || String(a.domain).localeCompare(String(b.domain))), ...rows.slice(GAP_RATED).map((row) => withDr(row, false))],
|
|
36519
|
+
added: true
|
|
36520
|
+
};
|
|
36521
|
+
}
|
|
35688
36522
|
// packages/core/src/commands/sitemap-urls.ts
|
|
35689
36523
|
const MAX_FILES = 20;
|
|
35690
36524
|
const MAX_URLS = 5e4;
|
|
@@ -36234,6 +37068,23 @@ function siteLines(site, startMs, endMs) {
|
|
|
36234
37068
|
} else lines.push("Search clicks: not available this week.");
|
|
36235
37069
|
return lines;
|
|
36236
37070
|
}
|
|
37071
|
+
function updateLines(site) {
|
|
37072
|
+
return (site.google_updates ?? []).map((update) => {
|
|
37073
|
+
const when = update.end ? "ran" : "has been rolling out";
|
|
37074
|
+
const result = update.result;
|
|
37075
|
+
if (!result && update.ready) return `- Google update: the ${update.name} ran ${update.start.slice(0, 10)} to ${update.end?.slice(0, 10) ?? "now"}. Ask for the report's google-updates section on this site for what moved.`;
|
|
37076
|
+
if (!result) return `- Google update: the ${update.name} ${when}${update.end ? ` (${update.start.slice(0, 10)} to ${update.end.slice(0, 10)})` : ` since ${update.start.slice(0, 10)}`}. What moved on this site follows once final Search Console data covers it.`;
|
|
37077
|
+
const diff = result.clicks - result.previous;
|
|
37078
|
+
const pct = result.previous > 0 ? Math.round(diff / result.previous * 100) : null;
|
|
37079
|
+
const moved = pct === null ? `${count(result.clicks)} clicks from ${count(result.previous)}` : pct === 0 ? `clicks level at ${count(result.clicks)}` : `clicks ${pct > 0 ? "up" : "down"} ${Math.abs(pct)}% (${count(result.clicks)} from ${count(result.previous)})`;
|
|
37080
|
+
const verdict = {
|
|
37081
|
+
"outside-normal": "more than this site usually moves",
|
|
37082
|
+
"within-normal": "within this site’s normal movement",
|
|
37083
|
+
"not-enough-data": "too little data to say if that is unusual"
|
|
37084
|
+
}[result.verdict];
|
|
37085
|
+
return `- Google update: the ${update.name}, ${moved} over the ${result.days} days from its start against the ${result.days} before, ${verdict} (timing only, not cause).`;
|
|
37086
|
+
});
|
|
37087
|
+
}
|
|
36237
37088
|
function opportunityLines(site) {
|
|
36238
37089
|
const found = site.opportunities;
|
|
36239
37090
|
if (!found) return [];
|
|
@@ -36286,7 +37137,7 @@ function weekMarkdown(data) {
|
|
|
36286
37137
|
"",
|
|
36287
37138
|
attention.length ? total === 1 ? "Your site had changes worth a look this week." : `${attention.length} of your ${total} sites had changes worth a look this week.` : quiet.length ? total === 1 ? "Your site had a quiet week." : quiet.length === total ? `All ${total} of your sites had a quiet week.` : `${quiet.length} of your ${total} sites had a quiet week.` : "Your sites are still on their first checks."
|
|
36288
37139
|
];
|
|
36289
|
-
for (const { site, found } of rows) out.push("", `## ${siteName(site.property)}${found.length ? `: ${list(found.map((item) => item.heading))}` : ""}`, "", ...siteLines(site, startMs, endMs).map((line) => `- ${line}`), ...opportunityLines(site), ...measuredLines(site), ...site.link ? [`- [View site](${site.link})`] : []);
|
|
37140
|
+
for (const { site, found } of rows) out.push("", `## ${siteName(site.property)}${found.length ? `: ${list(found.map((item) => item.heading))}` : ""}`, "", ...siteLines(site, startMs, endMs).map((line) => `- ${line}`), ...opportunityLines(site), ...measuredLines(site), ...updateLines(site), ...site.link ? [`- [View site](${site.link})`] : []);
|
|
36290
37141
|
return out.join("\n");
|
|
36291
37142
|
}
|
|
36292
37143
|
async function runAllSites(deps, sites) {
|
|
@@ -36389,10 +37240,14 @@ const SHARED_TOOLS = [
|
|
|
36389
37240
|
name: "seo_report",
|
|
36390
37241
|
command: "report",
|
|
36391
37242
|
title: "Search Console report",
|
|
36392
|
-
description: `Start here. Without site: a short summary of every site in the person's weekly site audit (beta) for last week, with each site's Google index, weekly crawl and search clicks, sites with changes first, ending with the site to ask about next; an account outside that beta gets its site list to pick from. With site: the main report for that Search Console property, the overview plus each section's row count and top rows. With site and section: that section's rows; page sections also check their top 20 URLs live. Sections:\n${Object.entries(SECTIONS).map(([id, text]) => `- ${id}: ${text}`).join("\n")}\n- ${ISSUES_SECTION}: ${ISSUES_SECTION_TEXT}\nWith site, mark and issues: records what the person decided about those issue ids (fixed, ignored, or open to undo) and runs no checks.`,
|
|
37243
|
+
description: `Start here. Without site: a short summary of every site in the person's weekly site audit (beta) for last week, with each site's Google index, weekly crawl and search clicks, sites with changes first, ending with the site to ask about next; an account outside that beta gets its site list to pick from. With site: the main report for that Search Console property, the overview plus each section's row count and top rows. With site and section: that section's rows; page sections also check their top 20 URLs live. Sections:\n${Object.entries(SECTIONS).map(([id, text]) => `- ${id}: ${text}`).join("\n")}\n- ${ISSUES_SECTION}: ${ISSUES_SECTION_TEXT}\n- ${UPDATES_SECTION}: ${UPDATES_SECTION_TEXT}\nWith site, mark and issues: records what the person decided about those issue ids (fixed, ignored, or open to undo) and runs no checks.`,
|
|
36393
37244
|
input: z.object({
|
|
36394
37245
|
site: site.optional().describe("Leave it out for the summary of every audited site (weekly site audit beta). Otherwise a Search Console property or site, such as sc-domain:example.com, https://example.com/ or example.com."),
|
|
36395
|
-
section: z.enum([
|
|
37246
|
+
section: z.enum([
|
|
37247
|
+
...SECTION_IDS,
|
|
37248
|
+
ISSUES_SECTION,
|
|
37249
|
+
UPDATES_SECTION
|
|
37250
|
+
]).optional().describe("Optional, needs site. Leave it out first; then ask for one section."),
|
|
36396
37251
|
...dates,
|
|
36397
37252
|
...detail,
|
|
36398
37253
|
...marks
|
|
@@ -36509,6 +37364,7 @@ const SHARED_TOOLS = [
|
|
|
36509
37364
|
action: z.enum(RESEARCH_ACTIONS),
|
|
36510
37365
|
keywords: z.array(z.string().min(1)).max(100).default([]),
|
|
36511
37366
|
target: z.string().default("").describe("Domain or URL."),
|
|
37367
|
+
competitors: z.array(z.string().min(1)).max(5).default([]).describe("link_gap only: 2 to 5 competitor domains."),
|
|
36512
37368
|
tasks: z.array(z.string().regex(AI_ANSWER_TASK_PATTERN)).max(30).default([]).describe("ai_answers only: the tasks an earlier run returned as not ready, to read back at no cost."),
|
|
36513
37369
|
country: z.string().regex(/^[a-z]{2}$/).default("us").describe("Two-letter country code."),
|
|
36514
37370
|
...detail
|
|
@@ -36524,7 +37380,7 @@ const SHARED_TOOLS = [
|
|
|
36524
37380
|
offset: input.offset,
|
|
36525
37381
|
questions: input.keywords,
|
|
36526
37382
|
tasks: input.tasks
|
|
36527
|
-
}, await surface.keys(), surface.deps,
|
|
37383
|
+
}, await surface.keys(), surface.deps, (offset, tasks) => `seo_research with the same provider, action and target, tasks ${JSON.stringify(tasks)}, detail "rows", offset ${offset}. Passing the tasks reads these answers back at no cost; without them every question is asked and paid for again`);
|
|
36528
37384
|
}
|
|
36529
37385
|
return runResearch(input, await surface.keys(), {
|
|
36530
37386
|
drKey: surface.drKey,
|
|
@@ -36655,6 +37511,7 @@ SHARED_TOOLS.map((shared) => shared.name === "seo_index" ? indexFromStore(shared
|
|
|
36655
37511
|
input: shared.input.extend({ section: z.enum([
|
|
36656
37512
|
...SECTION_IDS,
|
|
36657
37513
|
ISSUES_SECTION,
|
|
37514
|
+
UPDATES_SECTION,
|
|
36658
37515
|
AUDIT_SECTION
|
|
36659
37516
|
]).optional().describe("Optional. Leave it out first; then ask for one section.") })
|
|
36660
37517
|
} : shared);
|
|
@@ -39367,6 +40224,6 @@ function isoDate(daysAgo = 0) {
|
|
|
39367
40224
|
date.setUTCDate(date.getUTCDate() - daysAgo);
|
|
39368
40225
|
return date.toISOString().slice(0, 10);
|
|
39369
40226
|
}
|
|
39370
|
-
export { AHREFS_API_KEY_ENV, AHREFS_API_KEY_SECRET, AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, ANALYTICS_PROVIDER_IDS, AUDIT_PAGE_RULE_IDS, AhrefsClient, BING_API_KEY_ENV, BING_API_KEY_SECRET, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_IMPRESSIONS, CANNIBAL_MINIMUM_PAGE_SHARE, CLICKY_SITEKEY_ENV, CLICKY_SITE_KEYS_SECRET, CLOUD_SIGN_IN, CLOUD_URL, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DATAFORSEO_CREDENTIAL_SECRET, DATAFORSEO_LOGIN_ENV, DATAFORSEO_PASSWORD_ENV, DEFAULT_MAX_PAGES, DEFAULT_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, DEFAULT_UNREACHED_PAGE_LIMIT, DataForSeoClient, DecisionModelError, DecisionRunner, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, GoogleApiError, INCIDENTS_URL, JsonFileStorageAdapter, LINK_ROLES, LINT_MAX_FILES, LOCAL_BING_CONNECT, LOCAL_SIGN_IN, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MANAGED_PROVIDER_SECRET_NAMES, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, MAX_MAX_PAGES, MAX_SEARCH_CONSOLE_EXPORT_FILES, MAX_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, PAGE_TYPES, PROVIDER_ERROR_CODES, PROVIDER_SECRET_NAMES, ProviderError, ProviderExtensionRegistry, RESEARCH_IMPORT_COLUMN_KEYS, ResponseSizeLimitError, SEARCH_INTENTS, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_PROVIDER_API_VERSION, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SITEMAP_FILE_URL_LIMIT, SITE_CHECK_MAX_ISSUES, SITE_CHECK_MAX_PAGES, STEP_CHANGE_RULES, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_FAILURE_CONTEXTS, TELEMETRY_FAILURE_REASONS, TELEMETRY_NOTICE, TELEMETRY_OPERATIONS, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, TYPESAFE_API_KEY_ENV, agentActionsView, aiReferralSourceForValue, aiReferralsReport, analyticsConnectionLabel, analyticsConnectionProviderId, analyticsConnectionSchema, analyticsProviderDetails, analyticsProviderSchema, analyzeAiReferralRows, analyzeCannibalImportRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, auditCrawlPages, auditCrawlRequests, auditPage, authStatus, bearerFetch, boundedSitemapInventory, buildAgentFindings, buildQueryClusterReportFromRows, buildTelemetryPayload, builtPagePath, cacheDb, cannibalDataStatus, cannibalImportReport, cannibalReport, cannibalReportWith, changeWindow, checkBuiltSite, checkDatabaseReadiness, classifyTelemetryFailure, clearCache, cloudAccountFor, cloudBing, cloudCredentials, cloudGoogleEmails, cloudGoogleToken, cloudKeys, cloudLinked, cloudLogout, cloudWeek, clusterQueryRows, combineRobotsValues, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, ctrUnderperformersReportWith, decayClusterDrafts, decayComparisonRange, decayingReport, decayingReportWith, deleteAhrefsApiKey, deleteBingApiKey, deleteClickySiteKey, deleteDataForSeoCredentials, deleteManagedProviderSecrets, deleteProviderExtensionAccount, deleteProviderExtensionCredentials, deleteProviderSecret, deleteTokens, deriveBrandTerms, describeInstalledProviderExtension, detectAgentName, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensurePrivateDatabaseFiles, ensureSeoCliDirs, estimateSerpTitleWidth, expectedCtrForPosition, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fileSize, finalGscDateRange, findOverlappingSearchUpdates, finishCloudLogin, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, getAuthModeStatus, getCacheStats, getClientConfig, getDb, getPrivacySnapshot, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleJson, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoverageSignals, inferTemplateMovement, initializeTelemetry, inspectProviderPackage, inspectSearchConsoleUrl, inspectUrl, installProviderPackage, installedProviderPackageSchema, installedProviderPackagesSchema, internalLinksReport, internalLinksReportWith, interpretUpdateCorrelation, isBrandQuery, isNewerVersion, isSkippableReportError, isTelemetryCi, isTelemetryOperation, isTelemetryReport, isoDate, isoWeek, latestFinalGscDate, linkRecover, linkRecoverRecommendation, lintPaths, listAnalyticsAccounts, listAnalyticsDataStreams, listGa4AccountSummaries, listGa4DataStreams, listGoogleAccounts, listInstalledProviderExtensions, listRules, listSearchConsoleSites, listSearchUpdates, listSites, loadInstalledProviderExtensions, loadProviderExtensions, loadProviderPackage, loadSearchConsoleExport, localAccounts, localBing, localCredentialsProvider, localError, localGoogleFetch, localInternalLinksDependencies, localRefreshClicky, localResearchKeys, loginWithLoopback, makeRunId, matchGa4WebStreams, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, nodeCommandDeps, normalizeBrandText, normalizeClickySiteId, normalizeClickySiteKey, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, normalizeProviderExtensionAccount, normalizedResearchColumnName, notSignedIn, noteCacheWrite, openCloudLink, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, pendingCloudLogin, performanceAudit, performanceReportIsCacheable, plural, providerCapabilitySchema, providerConnectionFieldSchema, providerExactVersionSchema, providerExtensionIdSchema, providerIdSchema, providerIntegritySchema, providerKindSchema, providerPackageManifestSchema, providerPackageNameSchema, providerPreferenceSchema, providerRegistrationSchema, providerRequestJson, providerRequestText, pruneLogs, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, quickWinsReportWith, readAhrefsApiKey, readAllTokens, readBingApiKey, readBoundedResponseText, readClickySiteKey, readClickySiteKeys, readConfig, readDataForSeoCredentials, readInstalledProviderPackages, readJson, readJsonOrUndefined, readLintFiles, readOauthClient, readProviderExtensionAccount, readProviderExtensionCredentials, readProviderSecret, readTokens, recommendationForRule, reconcileExportPagesWithCrawl, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, registrableDomain, removeInstalledProviderPackage, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderWorkflowMarkdown, reportNarrative, requestHeaders, requirePageVerification, researchImportColumns, resetSeoData, resolveLocalSite, resolveSegmentRanges, resolveSeoCliPaths, reviewObservations, roundedPosition, runAnalyticsProviderLandingPages, runAnalyticsReport, runDoctor, runGa4Report, runProviderExtensionAction, runtimeForProvider, safeProviderAdapterError, saveInstalledProviderPackage, searchAnalyticsRows, searchUpdatesFromIncidents, secondPage, secondPageWith, segmentComparisonRange, segmentImpact, segmentRangeDays, seoErrorEnvelope, sessionSourceInListFilter, setActiveGoogleAccount, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteStartUrl, snapshotFetchedPage, startCloudLogin, strikingDistance, strikingDistanceWith, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, telemetryErrorCategory, toSeoError, tokenSchema, tokenStoreSchema, topFixes, trackTelemetryCommandFailed, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, trafficAnomalyWith, typesafeDecisionModel, unavailableSegmentImpactReport, unicodeTokens, uninstallProviderExtension, updateCorrelation, updateNotice, updatePostmortemCsvFiles, updatePostmortemWorkflow, urlInspectionQuota, validateResearchImportColumnSources, verifyAnalyticsProvider, verifyProviderExtension, verifyQueryContent, verifyQueryContentWith, withAgentReportContract, withRankingHistory, workflowPresentation, writeAhrefsApiKey, writeBingApiKey, writeClickySiteKey, writeConfig, writeDataForSeoCredentials, writeInstalledProviderPackages, writeOauthClient, writeProviderExtensionAccount, writeProviderExtensionCredentials, writeProviderSecret, writeToCache, writeTokens, xRobotsDirectives };
|
|
40227
|
+
export { AHREFS_API_KEY_ENV, AHREFS_API_KEY_SECRET, AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, ANALYTICS_PROVIDER_IDS, AUDIT_PAGE_RULE_IDS, AhrefsClient, BING_API_KEY_ENV, BING_API_KEY_SECRET, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_IMPRESSIONS, CANNIBAL_MINIMUM_PAGE_SHARE, CLICKY_SITEKEY_ENV, CLICKY_SITE_KEYS_SECRET, CLOUD_SIGN_IN, CLOUD_URL, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DATAFORSEO_CREDENTIAL_SECRET, DATAFORSEO_LOGIN_ENV, DATAFORSEO_PASSWORD_ENV, DEFAULT_MAX_PAGES, DEFAULT_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, DEFAULT_UNREACHED_PAGE_LIMIT, DataForSeoClient, DecisionModelError, DecisionRunner, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, GoogleApiError, INCIDENTS_URL, JsonFileStorageAdapter, LINK_ROLES, LINT_MAX_FILES, LOCAL_BING_CONNECT, LOCAL_SIGN_IN, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MANAGED_PROVIDER_SECRET_NAMES, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, MAX_MAX_PAGES, MAX_SEARCH_CONSOLE_EXPORT_FILES, MAX_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, PAGE_TYPES, PROVIDER_ERROR_CODES, PROVIDER_SECRET_NAMES, ProviderError, ProviderExtensionRegistry, RESEARCH_IMPORT_COLUMN_KEYS, ResponseSizeLimitError, SEARCH_INTENTS, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_PROVIDER_API_VERSION, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SITEMAP_FILE_URL_LIMIT, SITE_CHECK_MAX_ISSUES, SITE_CHECK_MAX_PAGES, STEP_CHANGE_RULES, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_FAILURE_CONTEXTS, TELEMETRY_FAILURE_REASONS, TELEMETRY_NOTICE, TELEMETRY_OPERATIONS, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, TYPESAFE_API_KEY_ENV, agentActionsView, aiReferralSourceForValue, aiReferralsReport, analyticsConnectionLabel, analyticsConnectionProviderId, analyticsConnectionSchema, analyticsProviderDetails, analyticsProviderSchema, analyzeAiReferralRows, analyzeCannibalImportRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, auditCrawlPages, auditCrawlRequests, auditPage, authStatus, bearerFetch, boundedSitemapInventory, buildAgentFindings, buildQueryClusterReportFromRows, buildTelemetryPayload, builtPagePath, cacheDb, cannibalDataStatus, cannibalImportReport, cannibalReport, cannibalReportWith, changeWindow, checkBuiltSite, checkDatabaseReadiness, classifyTelemetryFailure, clearCache, cloudAccountFor, cloudBing, cloudCredentials, cloudGoogleEmails, cloudGoogleToken, cloudKeys, cloudLinked, cloudLogout, cloudWeek, clusterQueryRows, combineRobotsValues, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlCoverageCaveat, crawlDefinitionId, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, ctrUnderperformersReportWith, decayClusterDrafts, decayComparisonRange, decayingReport, decayingReportWith, deleteAhrefsApiKey, deleteBingApiKey, deleteClickySiteKey, deleteDataForSeoCredentials, deleteManagedProviderSecrets, deleteProviderExtensionAccount, deleteProviderExtensionCredentials, deleteProviderSecret, deleteTokens, deriveBrandTerms, describeInstalledProviderExtension, detectAgentName, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensurePrivateDatabaseFiles, ensureSeoCliDirs, estimateSerpTitleWidth, expectedCtrForPosition, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fileSize, finalGscDateRange, findOverlappingSearchUpdates, finishCloudLogin, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, getAuthModeStatus, getCacheStats, getClientConfig, getDb, getPrivacySnapshot, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleJson, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoverageSignals, inferTemplateMovement, initializeTelemetry, inspectProviderPackage, inspectSearchConsoleUrl, inspectUrl, installProviderPackage, installedProviderPackageSchema, installedProviderPackagesSchema, internalLinksReport, internalLinksReportWith, interpretUpdateCorrelation, isBrandQuery, isNewerVersion, isSkippableReportError, isTelemetryCi, isTelemetryOperation, isTelemetryReport, isoDate, isoWeek, latestFinalGscDate, linkRecover, linkRecoverRecommendation, lintPaths, listAnalyticsAccounts, listAnalyticsDataStreams, listGa4AccountSummaries, listGa4DataStreams, listGoogleAccounts, listInstalledProviderExtensions, listRules, listSearchConsoleSites, listSearchUpdates, listSites, loadInstalledProviderExtensions, loadProviderExtensions, loadProviderPackage, loadSearchConsoleExport, localAccounts, localBing, localCredentialsProvider, localError, localGoogleFetch, localInternalLinksDependencies, localRefreshClicky, localResearchKeys, loginWithLoopback, makeRunId, matchGa4WebStreams, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, nodeCommandDeps, normalizeBrandText, normalizeClickySiteId, normalizeClickySiteKey, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, normalizeProviderExtensionAccount, normalizedResearchColumnName, notSignedIn, noteCacheWrite, openCloudLink, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, pendingCloudLogin, performanceAudit, performanceReportIsCacheable, plural, providerCapabilitySchema, providerConnectionFieldSchema, providerExactVersionSchema, providerExtensionIdSchema, providerIdSchema, providerIntegritySchema, providerKindSchema, providerPackageManifestSchema, providerPackageNameSchema, providerPreferenceSchema, providerRegistrationSchema, providerRequestJson, providerRequestText, pruneLogs, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, quickWinsReportWith, readAhrefsApiKey, readAllTokens, readBingApiKey, readBoundedResponseText, readClickySiteKey, readClickySiteKeys, readConfig, readDataForSeoCredentials, readInstalledProviderPackages, readJson, readJsonOrUndefined, readLintFiles, readOauthClient, readProviderExtensionAccount, readProviderExtensionCredentials, readProviderSecret, readTokens, recommendationForRule, reconcileExportPagesWithCrawl, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, registrableDomain, removeInstalledProviderPackage, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderWorkflowMarkdown, reportNarrative, requestHeaders, requirePageVerification, researchImportColumns, resetSeoData, resolveLocalSite, resolveSegmentRanges, resolveSeoCliPaths, reviewObservations, roundedPosition, runAnalyticsProviderLandingPages, runAnalyticsReport, runDoctor, runGa4Report, runProviderExtensionAction, runtimeForProvider, safeProviderAdapterError, saveInstalledProviderPackage, searchAnalyticsRows, searchUpdatesFromIncidents, secondPage, secondPageWith, segmentComparisonRange, segmentImpact, segmentRangeDays, seoErrorEnvelope, sessionSourceInListFilter, setActiveGoogleAccount, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteStartUrl, snapshotFetchedPage, startCloudLogin, strikingDistance, strikingDistanceWith, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, telemetryErrorCategory, toSeoError, tokenSchema, tokenStoreSchema, topFixes, trackTelemetryCommandFailed, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, trafficAnomalyWith, typesafeDecisionModel, unavailableSegmentImpactReport, unicodeTokens, uninstallProviderExtension, updateCorrelation, updateNotice, updatePostmortemCsvFiles, updatePostmortemWorkflow, urlInspectionQuota, validateResearchImportColumnSources, verifyAnalyticsProvider, verifyProviderExtension, verifyQueryContent, verifyQueryContentWith, withAgentReportContract, withRankingHistory, workflowPresentation, writeAhrefsApiKey, writeBingApiKey, writeClickySiteKey, writeConfig, writeDataForSeoCredentials, writeInstalledProviderPackages, writeOauthClient, writeProviderExtensionAccount, writeProviderExtensionCredentials, writeProviderSecret, writeToCache, writeTokens, xRobotsDirectives };
|
|
39371
40228
|
|
|
39372
40229
|
//# sourceMappingURL=index.js.map
|