seo 0.15.2 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +84 -84
- package/dist/cli.js.map +1 -1
- package/dist/{dist-DOJoZZ9W.js → dist-BZ5-V3qL.js} +1 -1
- package/dist/{dist-Dpl5hSMM.js → dist-CqzyPI3J.js} +140 -136
- package/dist/dist-CqzyPI3J.js.map +1 -0
- package/dist/{esm-Dt4JJZ0P.js → esm-Cvmg0qhZ.js} +2 -2
- package/dist/{esm-Dt4JJZ0P.js.map → esm-Cvmg0qhZ.js.map} +1 -1
- package/dist/index.d.ts +108 -46
- package/dist/index.js +1664 -807
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +1223 -372
- package/dist/mcp.js.map +1 -1
- package/package.json +1 -1
- package/skills/seo/SKILL.md +10 -1
- package/dist/dist-Dpl5hSMM.js.map +0 -1
package/dist/mcp.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-
|
|
2
|
+
import { i as __toESM, n as version, r as __commonJSMin, t as load } from "./esm-Cvmg0qhZ.js";
|
|
3
3
|
import { createHash, randomUUID, sign } from "node:crypto";
|
|
4
4
|
import { ZodOptional, z } from "zod";
|
|
5
5
|
import { arch, homedir, platform, tmpdir, totalmem } from "node:os";
|
|
6
6
|
import { dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
7
7
|
import "open";
|
|
8
|
-
import { chmodSync, createReadStream, existsSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
8
|
+
import { chmodSync, createReadStream, existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
9
9
|
import lockfile from "proper-lockfile";
|
|
10
10
|
import "node:http";
|
|
11
11
|
import { Headers as Headers$1, fetch as fetch$1 } from "undici";
|
|
@@ -12461,6 +12461,24 @@ const RULE_DEFINITIONS = [
|
|
|
12461
12461
|
suggestedCommands: ["seo crawl <url> --max-pages 1 --json"]
|
|
12462
12462
|
}
|
|
12463
12463
|
},
|
|
12464
|
+
{
|
|
12465
|
+
id: "crawl_single_page_reach",
|
|
12466
|
+
title: "Crawl could reach only the start page",
|
|
12467
|
+
category: "links",
|
|
12468
|
+
defaultSeverity: "high",
|
|
12469
|
+
whyItMatters: "The start page had no crawlable internal links and no sitemap returned URLs, so the crawl checked one page. Every other finding in this report covers that page only. The page probably builds its links with JavaScript.",
|
|
12470
|
+
howToFix: "Rerun with `--rendering on` so the crawler reads the links a browser sees, or pass the pages to check with `--mode list`. For crawlers that do not run JavaScript, put plain `<a href>` links in the initial HTML and publish a sitemap.",
|
|
12471
|
+
impactIfIgnored: "Crawlers and audits that do not run JavaScript can find only this page, so the rest of the site stays unchecked and harder to discover.",
|
|
12472
|
+
howToVerify: "Rerun the crawl and confirm it retains more than one page and the start page reports internal links.",
|
|
12473
|
+
agentHints: {
|
|
12474
|
+
evidenceFields: [
|
|
12475
|
+
"page.outgoingInternalCount",
|
|
12476
|
+
"report.sitemapDiscovery.urlsReturned",
|
|
12477
|
+
"page.fetchDiagnostics.rendering"
|
|
12478
|
+
],
|
|
12479
|
+
suggestedCommands: ["seo crawl <url> --rendering on --json", "seo crawl --mode list --urls <url>,<url> --json"]
|
|
12480
|
+
}
|
|
12481
|
+
},
|
|
12464
12482
|
{
|
|
12465
12483
|
id: "soft_404",
|
|
12466
12484
|
title: "Unknown URLs return a successful response",
|
|
@@ -12469,7 +12487,7 @@ const RULE_DEFINITIONS = [
|
|
|
12469
12487
|
whyItMatters: "Known-nonexistent URLs returned a successful response or redirected to another successful page. Search engines and users cannot reliably distinguish missing content from a real page.",
|
|
12470
12488
|
howToFix: "Return 404 or 410 for missing URLs. If a replacement exists, redirect only the specific retired URL to its closest relevant replacement rather than sending every unknown path to a generic page.",
|
|
12471
12489
|
impactIfIgnored: "Missing URLs can remain in crawl queues, waste crawl attention, and create confusing search or monitoring evidence.",
|
|
12472
|
-
howToVerify: "Request several
|
|
12490
|
+
howToVerify: "Request several made-up paths, including one at the site root and one nested path, and confirm each returns 404 or 410 without redirecting to a generic successful page.",
|
|
12473
12491
|
agentHints: {
|
|
12474
12492
|
evidenceFields: ["report.siteChecks.soft404"],
|
|
12475
12493
|
suggestedCommands: ["seo crawl <url> --json"]
|
|
@@ -13918,7 +13936,7 @@ function day$3(value) {
|
|
|
13918
13936
|
const date = legacy ? new Date(Number(legacy[1])) : new Date(value);
|
|
13919
13937
|
return Number.isNaN(date.getTime()) ? null : date.toISOString().slice(0, 10);
|
|
13920
13938
|
}
|
|
13921
|
-
const count$
|
|
13939
|
+
const count$4 = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
|
|
13922
13940
|
function list$2(data, label) {
|
|
13923
13941
|
if (!Array.isArray(data)) throw new SeoError("OPTIONAL_PROVIDER_UNAVAILABLE", `Bing Webmaster returned an invalid ${label}.`);
|
|
13924
13942
|
return data.filter((item) => Boolean(item) && typeof item === "object");
|
|
@@ -13934,8 +13952,8 @@ async function bingTraffic(call, site) {
|
|
|
13934
13952
|
let invalidRows = 0;
|
|
13935
13953
|
for (const row of list$2(await call("GetRankAndTrafficStats", { siteUrl: site }), "traffic list")) {
|
|
13936
13954
|
const date = day$3(row.Date);
|
|
13937
|
-
const clicks = count$
|
|
13938
|
-
const impressions = count$
|
|
13955
|
+
const clicks = count$4(row.Clicks);
|
|
13956
|
+
const impressions = count$4(row.Impressions);
|
|
13939
13957
|
if (!date || clicks === null || impressions === null) {
|
|
13940
13958
|
invalidRows += 1;
|
|
13941
13959
|
continue;
|
|
@@ -13958,9 +13976,9 @@ async function keyed(call, method, params) {
|
|
|
13958
13976
|
for (const row of list$2(await call(method, params), "row list")) {
|
|
13959
13977
|
const date = day$3(row.Date);
|
|
13960
13978
|
const key = typeof row.Query === "string" ? row.Query.trim() : "";
|
|
13961
|
-
const clicks = count$
|
|
13962
|
-
const impressions = count$
|
|
13963
|
-
const position = count$
|
|
13979
|
+
const clicks = count$4(row.Clicks);
|
|
13980
|
+
const impressions = count$4(row.Impressions);
|
|
13981
|
+
const position = count$4(row.AvgImpressionPosition);
|
|
13964
13982
|
if (!date || !key || clicks === null || impressions === null) {
|
|
13965
13983
|
invalidRows += 1;
|
|
13966
13984
|
continue;
|
|
@@ -14491,6 +14509,14 @@ function sharedReader(read, redirects = /* @__PURE__ */ new Map()) {
|
|
|
14491
14509
|
};
|
|
14492
14510
|
}
|
|
14493
14511
|
const MAX_STATUS_BYTES = 10485760;
|
|
14512
|
+
async function rankingUpdates() {
|
|
14513
|
+
const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? readJson(response, "Google Search Status", { maxBytes: MAX_STATUS_BYTES }) : null).catch(() => null);
|
|
14514
|
+
if (!Array.isArray(incidents)) return "Google Search Status could not be read.";
|
|
14515
|
+
return searchUpdatesFromIncidents(withRankingHistory(incidents), {
|
|
14516
|
+
product: "Ranking",
|
|
14517
|
+
limit: Number.POSITIVE_INFINITY
|
|
14518
|
+
});
|
|
14519
|
+
}
|
|
14494
14520
|
async function googleUpdates(window) {
|
|
14495
14521
|
if (!window) return [];
|
|
14496
14522
|
const incidents = await fetch("https://status.search.google.com/incidents.json", { signal: AbortSignal.timeout(8e3) }).then((response) => response.ok ? readJson(response, "Google Search Status", { maxBytes: MAX_STATUS_BYTES }) : null).catch(() => null);
|
|
@@ -19132,17 +19158,15 @@ function recordHostFetch(input) {
|
|
|
19132
19158
|
const cfg = input.rate.backpressure;
|
|
19133
19159
|
health.durations.push(input.durationMs);
|
|
19134
19160
|
if (health.durations.length > 20) health.durations.shift();
|
|
19135
|
-
const
|
|
19136
|
-
|
|
19137
|
-
|
|
19138
|
-
429
|
|
19139
|
-
].includes(input.status);
|
|
19140
|
-
const errored = input.status >= 500;
|
|
19161
|
+
const rateLimited = input.status === 429 || input.status === 503 && input.retryAfterMs !== void 0;
|
|
19162
|
+
const blocked = [401, 403].includes(input.status);
|
|
19163
|
+
const errored = input.status >= 500 && !rateLimited;
|
|
19141
19164
|
const slow = input.durationMs >= cfg.slowMs;
|
|
19142
19165
|
health.consecutiveBlocked = blocked ? health.consecutiveBlocked + 1 : 0;
|
|
19143
19166
|
health.consecutiveErrors = errored ? health.consecutiveErrors + 1 : 0;
|
|
19144
19167
|
health.consecutiveSlow = slow ? health.consecutiveSlow + 1 : 0;
|
|
19145
|
-
if (!blocked && !errored && !slow) health.reason = void 0;
|
|
19168
|
+
if (!blocked && !errored && !slow && !rateLimited) health.reason = void 0;
|
|
19169
|
+
if (rateLimited) health.reason = `origin rate limited the crawler (HTTP ${input.status})`;
|
|
19146
19170
|
const retryDelay = input.retryAfterMs === void 0 ? 0 : Math.min(input.retryAfterMs, cfg.retryAfterCapMs);
|
|
19147
19171
|
if (retryDelay > 0) {
|
|
19148
19172
|
health.reason = `origin sent Retry-After (${Math.round(retryDelay / 1e3)}s)`;
|
|
@@ -19543,6 +19567,56 @@ function systemBrowserCandidates(input) {
|
|
|
19543
19567
|
product: path.includes("google-chrome") ? "Google Chrome" : "Chromium"
|
|
19544
19568
|
}));
|
|
19545
19569
|
}
|
|
19570
|
+
function listDirectorySafely(path) {
|
|
19571
|
+
try {
|
|
19572
|
+
return readdirSync(path);
|
|
19573
|
+
} catch {
|
|
19574
|
+
return [];
|
|
19575
|
+
}
|
|
19576
|
+
}
|
|
19577
|
+
function versionParts(name) {
|
|
19578
|
+
return (name.match(/\d+/g) ?? []).map(Number);
|
|
19579
|
+
}
|
|
19580
|
+
function newestFirst(left, right) {
|
|
19581
|
+
const a = versionParts(left);
|
|
19582
|
+
const b = versionParts(right);
|
|
19583
|
+
for (let index = 0; index < Math.max(a.length, b.length); index += 1) {
|
|
19584
|
+
const difference = (b[index] ?? -1) - (a[index] ?? -1);
|
|
19585
|
+
if (difference) return difference;
|
|
19586
|
+
}
|
|
19587
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
19588
|
+
}
|
|
19589
|
+
const MAX_CACHED_BROWSER_VERSIONS = 20;
|
|
19590
|
+
function versionDirectories(root, prefix, listDirectory) {
|
|
19591
|
+
return listDirectory(root).filter((name) => name.startsWith(prefix)).sort(newestFirst).slice(0, MAX_CACHED_BROWSER_VERSIONS).map((name) => join(root, name));
|
|
19592
|
+
}
|
|
19593
|
+
const CHROME_FOR_TESTING_APP = join("Google Chrome for Testing.app", "Contents", "MacOS", "Google Chrome for Testing");
|
|
19594
|
+
/**
|
|
19595
|
+
* Browsers that other local tools already downloaded: agent-browser keeps
|
|
19596
|
+
* Chrome for Testing builds and Playwright keeps one Chromium per revision.
|
|
19597
|
+
*/
|
|
19598
|
+
function cachedBrowserCandidates(input) {
|
|
19599
|
+
const agentBrowserExecutables = input.platform === "darwin" ? [CHROME_FOR_TESTING_APP] : input.platform === "win32" ? ["chrome.exe"] : ["chrome"];
|
|
19600
|
+
const agentBrowser = versionDirectories(join(input.homeDirectory, ".agent-browser", "browsers"), "chrome-", input.listDirectory).flatMap((directory) => agentBrowserExecutables.map((executable) => ({
|
|
19601
|
+
path: join(directory, executable),
|
|
19602
|
+
source: "agent-browser-cache",
|
|
19603
|
+
product: "agent-browser Chrome for Testing"
|
|
19604
|
+
})));
|
|
19605
|
+
const configuredRoot = input.environment.PLAYWRIGHT_BROWSERS_PATH?.trim();
|
|
19606
|
+
const playwrightRoot = configuredRoot && configuredRoot !== "0" ? configuredRoot : input.platform === "darwin" ? join(input.homeDirectory, "Library", "Caches", "ms-playwright") : input.platform === "win32" ? join(input.environment.LOCALAPPDATA ?? join(input.homeDirectory, "AppData", "Local"), "ms-playwright") : join(input.environment.XDG_CACHE_HOME ?? join(input.homeDirectory, ".cache"), "ms-playwright");
|
|
19607
|
+
const playwrightExecutables = input.platform === "darwin" ? [
|
|
19608
|
+
join("chrome-mac-arm64", CHROME_FOR_TESTING_APP),
|
|
19609
|
+
join("chrome-mac-x64", CHROME_FOR_TESTING_APP),
|
|
19610
|
+
join("chrome-mac", CHROME_FOR_TESTING_APP),
|
|
19611
|
+
join("chrome-mac", "Chromium.app", "Contents", "MacOS", "Chromium")
|
|
19612
|
+
] : input.platform === "win32" ? [join("chrome-win64", "chrome.exe"), join("chrome-win", "chrome.exe")] : [join("chrome-linux64", "chrome"), join("chrome-linux", "chrome")];
|
|
19613
|
+
const playwright = versionDirectories(playwrightRoot, "chromium-", input.listDirectory).flatMap((directory) => playwrightExecutables.map((executable) => ({
|
|
19614
|
+
path: join(directory, executable),
|
|
19615
|
+
source: "playwright-cache",
|
|
19616
|
+
product: "Playwright Chromium"
|
|
19617
|
+
})));
|
|
19618
|
+
return [...agentBrowser, ...playwright];
|
|
19619
|
+
}
|
|
19546
19620
|
/**
|
|
19547
19621
|
* Resolve a browser without downloading one during normal package installation.
|
|
19548
19622
|
* The explicit path wins so CI and managed workstations can select their own
|
|
@@ -19574,11 +19648,23 @@ function resolveBrowserExecutable(playwrightExecutablePath, dependencies = {}) {
|
|
|
19574
19648
|
product: "Playwright Chromium"
|
|
19575
19649
|
}
|
|
19576
19650
|
};
|
|
19577
|
-
const
|
|
19578
|
-
|
|
19579
|
-
|
|
19651
|
+
const platform = dependencies.platform ?? process.platform;
|
|
19652
|
+
const homeDirectory = dependencies.homeDirectory ?? homedir();
|
|
19653
|
+
const systemBrowser = systemBrowserCandidates({
|
|
19654
|
+
platform,
|
|
19655
|
+
homeDirectory,
|
|
19580
19656
|
environment
|
|
19581
19657
|
}).find((candidate) => fileExists(candidate.path));
|
|
19658
|
+
if (systemBrowser) return {
|
|
19659
|
+
status: "available",
|
|
19660
|
+
browser: systemBrowser
|
|
19661
|
+
};
|
|
19662
|
+
const browser = cachedBrowserCandidates({
|
|
19663
|
+
platform,
|
|
19664
|
+
homeDirectory,
|
|
19665
|
+
environment,
|
|
19666
|
+
listDirectory: dependencies.listDirectory ?? listDirectorySafely
|
|
19667
|
+
}).find((candidate) => fileExists(candidate.path));
|
|
19582
19668
|
return browser ? {
|
|
19583
19669
|
status: "available",
|
|
19584
19670
|
browser
|
|
@@ -22574,6 +22660,17 @@ function isAuditableHtmlPage(page) {
|
|
|
22574
22660
|
const contentAuditAllowed = page.contentAuditAllowed ?? (!page.blocked && page.robotsTxt?.allowed !== false);
|
|
22575
22661
|
return isSuccessfulResponse(page) && isHtmlPage(page) && !isRedirectedPage(page) && contentAuditAllowed;
|
|
22576
22662
|
}
|
|
22663
|
+
/**
|
|
22664
|
+
* Cloudflare serves its own endpoints under `/cdn-cgi/`, such as the email
|
|
22665
|
+
* obfuscation link `/cdn-cgi/l/email-protection`. They are not site pages.
|
|
22666
|
+
*/
|
|
22667
|
+
function isCdnUtilityUrl(value) {
|
|
22668
|
+
try {
|
|
22669
|
+
return new URL(value).pathname.startsWith("/cdn-cgi/");
|
|
22670
|
+
} catch {
|
|
22671
|
+
return false;
|
|
22672
|
+
}
|
|
22673
|
+
}
|
|
22577
22674
|
// packages/core/dist/analyze/crawler/audit.js
|
|
22578
22675
|
function sameUrl(a, b) {
|
|
22579
22676
|
if (!a || !b) return false;
|
|
@@ -22686,38 +22783,61 @@ function isValuablePage(page) {
|
|
|
22686
22783
|
function normalizedMetadata(value) {
|
|
22687
22784
|
return value?.trim().replace(/\s+/g, " ").toLowerCase() || void 0;
|
|
22688
22785
|
}
|
|
22689
|
-
|
|
22690
|
-
|
|
22786
|
+
/**
|
|
22787
|
+
* A page whose canonical names another crawled page in its group is already
|
|
22788
|
+
* handled. A group counts only when two or more of its pages are
|
|
22789
|
+
* self-canonical or declare no canonical, and only those pages are reported.
|
|
22790
|
+
*/
|
|
22791
|
+
function duplicateGroups(pages, read) {
|
|
22792
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
22691
22793
|
for (const page of pages) {
|
|
22692
22794
|
if (!isAuditableHtmlPage(page)) continue;
|
|
22693
|
-
const
|
|
22694
|
-
|
|
22695
|
-
|
|
22696
|
-
|
|
22697
|
-
|
|
22698
|
-
sampleUrls: [],
|
|
22699
|
-
value: raw
|
|
22795
|
+
const entry = read(page);
|
|
22796
|
+
if (!entry) continue;
|
|
22797
|
+
const group = grouped.get(entry.key) ?? {
|
|
22798
|
+
value: entry.value,
|
|
22799
|
+
pages: []
|
|
22700
22800
|
};
|
|
22701
|
-
|
|
22702
|
-
|
|
22703
|
-
|
|
22801
|
+
group.pages.push(page);
|
|
22802
|
+
grouped.set(entry.key, group);
|
|
22803
|
+
}
|
|
22804
|
+
const duplicates = /* @__PURE__ */ new Map();
|
|
22805
|
+
for (const [key, group] of grouped) {
|
|
22806
|
+
if (group.pages.length < 2) continue;
|
|
22807
|
+
const groupUrls = /* @__PURE__ */ new Set();
|
|
22808
|
+
for (const page of group.pages) for (const value of [page.url, page.finalUrl]) {
|
|
22809
|
+
const url = urlKey(value);
|
|
22810
|
+
if (url) groupUrls.add(url);
|
|
22811
|
+
}
|
|
22812
|
+
const counted = group.pages.filter((page) => {
|
|
22813
|
+
const canonical = urlKey(page.canonical);
|
|
22814
|
+
return !canonical || canonical === urlKey(page.url) || canonical === urlKey(page.finalUrl) || !groupUrls.has(canonical);
|
|
22815
|
+
});
|
|
22816
|
+
if (counted.length < 2) continue;
|
|
22817
|
+
duplicates.set(key, {
|
|
22818
|
+
count: counted.length,
|
|
22819
|
+
sampleUrls: counted.slice(0, 10).map((page) => page.url),
|
|
22820
|
+
value: group.value,
|
|
22821
|
+
urls: new Set(counted.map((page) => page.url))
|
|
22822
|
+
});
|
|
22704
22823
|
}
|
|
22705
|
-
return
|
|
22824
|
+
return duplicates;
|
|
22825
|
+
}
|
|
22826
|
+
function metadataDuplicates(pages, read) {
|
|
22827
|
+
return duplicateGroups(pages, (page) => {
|
|
22828
|
+
const value = read(page)?.trim();
|
|
22829
|
+
const key = normalizedMetadata(value);
|
|
22830
|
+
return key && value ? {
|
|
22831
|
+
key,
|
|
22832
|
+
value
|
|
22833
|
+
} : void 0;
|
|
22834
|
+
});
|
|
22706
22835
|
}
|
|
22707
22836
|
function contentDuplicates(pages) {
|
|
22708
|
-
|
|
22709
|
-
|
|
22710
|
-
|
|
22711
|
-
|
|
22712
|
-
const existing = counts.get(page.mainContentHash) ?? {
|
|
22713
|
-
count: 0,
|
|
22714
|
-
sampleUrls: []
|
|
22715
|
-
};
|
|
22716
|
-
existing.count += 1;
|
|
22717
|
-
if (existing.sampleUrls.length < 10) existing.sampleUrls.push(page.url);
|
|
22718
|
-
counts.set(page.mainContentHash, existing);
|
|
22719
|
-
}
|
|
22720
|
-
return new Map([...counts].filter(([, value]) => value.count > 1));
|
|
22837
|
+
return duplicateGroups(pages, (page) => page.wordCount >= 20 && page.mainContentHash ? {
|
|
22838
|
+
key: page.mainContentHash,
|
|
22839
|
+
value: page.mainContentHash
|
|
22840
|
+
} : void 0);
|
|
22721
22841
|
}
|
|
22722
22842
|
const QUERY_STOP_WORDS = /* @__PURE__ */ new Set([
|
|
22723
22843
|
"a",
|
|
@@ -22820,7 +22940,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
22820
22940
|
if (page.accessBlock) continue;
|
|
22821
22941
|
if (page.status >= 500) {
|
|
22822
22942
|
issues.push(issue$1("server_error", page, String(page.status), { status: page.status }));
|
|
22823
|
-
if ((page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
22943
|
+
if ((page.internalInlinkCount ?? 0) > 0 && !isCdnUtilityUrl(page.url)) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
22824
22944
|
status: page.status,
|
|
22825
22945
|
internalInlinkCount: page.internalInlinkCount
|
|
22826
22946
|
}));
|
|
@@ -22828,7 +22948,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
22828
22948
|
}
|
|
22829
22949
|
if (page.status >= 400) {
|
|
22830
22950
|
issues.push(issue$1("client_error", page, String(page.status), { status: page.status }));
|
|
22831
|
-
if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
22951
|
+
if (isBrokenLinkStatus(page.status) && (page.internalInlinkCount ?? 0) > 0 && !isCdnUtilityUrl(page.url)) issues.push(issue$1("broken_internal_link", page, String(page.status), {
|
|
22832
22952
|
status: page.status,
|
|
22833
22953
|
internalInlinkCount: page.internalInlinkCount
|
|
22834
22954
|
}));
|
|
@@ -22927,7 +23047,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
22927
23047
|
title,
|
|
22928
23048
|
...titleWidth
|
|
22929
23049
|
}));
|
|
22930
|
-
if (title && duplicateTitle) issues.push(issue$1("title_duplicate", page, duplicateTitle.value, {
|
|
23050
|
+
if (title && duplicateTitle?.urls.has(page.url)) issues.push(issue$1("title_duplicate", page, duplicateTitle.value, {
|
|
22931
23051
|
title,
|
|
22932
23052
|
duplicateCount: duplicateTitle.count,
|
|
22933
23053
|
sampleUrls: duplicateTitle.sampleUrls
|
|
@@ -22935,7 +23055,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
22935
23055
|
const metaDescription = page.metaDescription?.trim();
|
|
22936
23056
|
const duplicateDescription = duplicateDescriptions.get(normalizedMetadata(metaDescription) ?? "");
|
|
22937
23057
|
if (!metaDescription) issues.push(issue$1("missing_meta_description", page));
|
|
22938
|
-
if (metaDescription && duplicateDescription) issues.push(issue$1("meta_description_duplicate", page, duplicateDescription.value, {
|
|
23058
|
+
if (metaDescription && duplicateDescription?.urls.has(page.url)) issues.push(issue$1("meta_description_duplicate", page, duplicateDescription.value, {
|
|
22939
23059
|
metaDescription,
|
|
22940
23060
|
duplicateCount: duplicateDescription.count,
|
|
22941
23061
|
sampleUrls: duplicateDescription.sampleUrls
|
|
@@ -22983,7 +23103,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
22983
23103
|
finalUrl: page.finalUrl
|
|
22984
23104
|
}));
|
|
22985
23105
|
const contentDuplicate = page.mainContentHash ? duplicateContent.get(page.mainContentHash) : void 0;
|
|
22986
|
-
if (contentDuplicate) issues.push(issue$1("duplicate_content", page, "Duplicate main content", {
|
|
23106
|
+
if (contentDuplicate?.urls.has(page.url)) issues.push(issue$1("duplicate_content", page, "Duplicate main content", {
|
|
22987
23107
|
mainContentHash: page.mainContentHash,
|
|
22988
23108
|
duplicateCount: contentDuplicate.count,
|
|
22989
23109
|
sampleUrls: contentDuplicate.sampleUrls
|
|
@@ -23051,6 +23171,7 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
23051
23171
|
// packages/core/dist/analyze/crawler/crawl-skip-reasons.js
|
|
23052
23172
|
const CRAWL_SKIP_REASONS = [
|
|
23053
23173
|
"asset-url",
|
|
23174
|
+
"cdn-utility-url",
|
|
23054
23175
|
"configured-exclusion",
|
|
23055
23176
|
"duplicate-normalization",
|
|
23056
23177
|
"invalid-url",
|
|
@@ -23058,12 +23179,14 @@ const CRAWL_SKIP_REASONS = [
|
|
|
23058
23179
|
"off-origin",
|
|
23059
23180
|
"origin-backpressure",
|
|
23060
23181
|
"queue-safety-limit",
|
|
23182
|
+
"query-variant-limit",
|
|
23061
23183
|
"robots-disallowed",
|
|
23062
23184
|
"robots-uncertain",
|
|
23063
23185
|
"legacy-unclassified"
|
|
23064
23186
|
];
|
|
23065
23187
|
const SKIP_IMPACT = {
|
|
23066
23188
|
"asset-url": "non-impacting",
|
|
23189
|
+
"cdn-utility-url": "non-impacting",
|
|
23067
23190
|
"configured-exclusion": "non-impacting",
|
|
23068
23191
|
"duplicate-normalization": "non-impacting",
|
|
23069
23192
|
"invalid-url": "non-impacting",
|
|
@@ -23071,6 +23194,7 @@ const SKIP_IMPACT = {
|
|
|
23071
23194
|
"off-origin": "non-impacting",
|
|
23072
23195
|
"origin-backpressure": "coverage-affecting",
|
|
23073
23196
|
"queue-safety-limit": "coverage-affecting",
|
|
23197
|
+
"query-variant-limit": "coverage-affecting",
|
|
23074
23198
|
"robots-disallowed": "coverage-affecting",
|
|
23075
23199
|
"robots-uncertain": "coverage-affecting",
|
|
23076
23200
|
"legacy-unclassified": "coverage-affecting"
|
|
@@ -23123,157 +23247,85 @@ function crawlSkipReasonCountsFromRecord(input = {}) {
|
|
|
23123
23247
|
count
|
|
23124
23248
|
})) }).skipReasons;
|
|
23125
23249
|
}
|
|
23126
|
-
// packages/core/dist/analyze/crawler/
|
|
23127
|
-
|
|
23128
|
-
|
|
23129
|
-
medium: 1200,
|
|
23130
|
-
low: 100
|
|
23131
|
-
};
|
|
23132
|
-
const EFFORT_BY_CATEGORY = {
|
|
23133
|
-
canonical: "medium",
|
|
23134
|
-
content: "high",
|
|
23135
|
-
agents: "low",
|
|
23136
|
-
headings: "low",
|
|
23137
|
-
images: "medium",
|
|
23138
|
-
indexability: "medium",
|
|
23139
|
-
international: "low",
|
|
23140
|
-
metadata: "low",
|
|
23141
|
-
mobile: "low",
|
|
23142
|
-
response: "medium",
|
|
23143
|
-
security: "medium",
|
|
23144
|
-
social: "low",
|
|
23145
|
-
"structured-data": "medium"
|
|
23146
|
-
};
|
|
23147
|
-
const EFFORT_SCORE = {
|
|
23148
|
-
low: 60,
|
|
23149
|
-
medium: 30,
|
|
23150
|
-
high: 0
|
|
23151
|
-
};
|
|
23152
|
-
const AFFECTED_URL_WEIGHT = {
|
|
23153
|
-
high: 25,
|
|
23154
|
-
medium: 15,
|
|
23155
|
-
low: 2
|
|
23156
|
-
};
|
|
23157
|
-
const VALUE_WEIGHT = {
|
|
23158
|
-
high: 1,
|
|
23159
|
-
medium: .75,
|
|
23160
|
-
low: .15
|
|
23161
|
-
};
|
|
23162
|
-
function matchesPattern(pattern, value) {
|
|
23163
|
-
if (!pattern.includes("*")) return value.includes(pattern);
|
|
23164
|
-
const parts = pattern.split("*");
|
|
23165
|
-
let position = 0;
|
|
23166
|
-
for (const part of parts) {
|
|
23167
|
-
if (!part) continue;
|
|
23168
|
-
const found = value.slice(position).indexOf(part);
|
|
23169
|
-
if (found === -1) return false;
|
|
23170
|
-
position += found + part.length;
|
|
23171
|
-
}
|
|
23172
|
-
return true;
|
|
23250
|
+
// packages/core/dist/analyze/crawler/crawl-url-queue.js
|
|
23251
|
+
function before(left, right) {
|
|
23252
|
+
return left.depth < right.depth || left.depth === right.depth && left.url < right.url;
|
|
23173
23253
|
}
|
|
23174
|
-
|
|
23175
|
-
|
|
23176
|
-
|
|
23177
|
-
|
|
23178
|
-
|
|
23179
|
-
|
|
23254
|
+
var CrawlUrlQueue = class {
|
|
23255
|
+
#items = [];
|
|
23256
|
+
#positions = /* @__PURE__ */ new Map();
|
|
23257
|
+
get size() {
|
|
23258
|
+
return this.#items.length;
|
|
23259
|
+
}
|
|
23260
|
+
has(url) {
|
|
23261
|
+
return this.#positions.has(url);
|
|
23262
|
+
}
|
|
23263
|
+
push(item) {
|
|
23264
|
+
if (this.has(item.url)) throw new Error(`URL is already queued: ${item.url}`);
|
|
23265
|
+
const index = this.#items.length;
|
|
23266
|
+
this.#items.push(item);
|
|
23267
|
+
this.#positions.set(item.url, index);
|
|
23268
|
+
this.#bubbleUp(index);
|
|
23269
|
+
}
|
|
23270
|
+
decreaseDepth(url, depth) {
|
|
23271
|
+
const index = this.#positions.get(url);
|
|
23272
|
+
if (index === void 0) return false;
|
|
23273
|
+
const item = this.#items[index];
|
|
23274
|
+
if (!item || depth >= item.depth) return false;
|
|
23275
|
+
item.depth = depth;
|
|
23276
|
+
this.#bubbleUp(index);
|
|
23180
23277
|
return true;
|
|
23181
|
-
});
|
|
23182
|
-
}
|
|
23183
|
-
function groupIssues(issues) {
|
|
23184
|
-
const groups = /* @__PURE__ */ new Map();
|
|
23185
|
-
for (const issue of issues) {
|
|
23186
|
-
const existing = groups.get(issue.ruleId) ?? {
|
|
23187
|
-
ruleId: issue.ruleId,
|
|
23188
|
-
title: issue.title,
|
|
23189
|
-
category: issue.category,
|
|
23190
|
-
severity: issue.severity,
|
|
23191
|
-
count: 0,
|
|
23192
|
-
sampleUrls: []
|
|
23193
|
-
};
|
|
23194
|
-
existing.count += 1;
|
|
23195
|
-
if (existing.sampleUrls.length < 10) existing.sampleUrls.push(issue.url);
|
|
23196
|
-
groups.set(issue.ruleId, existing);
|
|
23197
23278
|
}
|
|
23198
|
-
|
|
23199
|
-
|
|
23200
|
-
|
|
23201
|
-
|
|
23202
|
-
|
|
23203
|
-
|
|
23204
|
-
|
|
23205
|
-
|
|
23206
|
-
|
|
23207
|
-
|
|
23208
|
-
|
|
23209
|
-
|
|
23210
|
-
|
|
23211
|
-
|
|
23212
|
-
|
|
23213
|
-
|
|
23214
|
-
|
|
23215
|
-
|
|
23216
|
-
|
|
23217
|
-
|
|
23218
|
-
|
|
23219
|
-
|
|
23220
|
-
|
|
23221
|
-
|
|
23222
|
-
|
|
23223
|
-
|
|
23224
|
-
|
|
23225
|
-
|
|
23226
|
-
|
|
23227
|
-
|
|
23228
|
-
|
|
23229
|
-
|
|
23230
|
-
|
|
23231
|
-
|
|
23232
|
-
|
|
23233
|
-
|
|
23234
|
-
|
|
23235
|
-
|
|
23236
|
-
|
|
23237
|
-
|
|
23238
|
-
|
|
23239
|
-
|
|
23240
|
-
|
|
23241
|
-
|
|
23242
|
-
|
|
23243
|
-
|
|
23244
|
-
|
|
23245
|
-
|
|
23246
|
-
|
|
23247
|
-
|
|
23248
|
-
avgPosition: search.avgPosition,
|
|
23249
|
-
effort,
|
|
23250
|
-
effortScore: EFFORT_SCORE[effort]
|
|
23251
|
-
};
|
|
23252
|
-
const valueWeight = VALUE_WEIGHT[group.severity];
|
|
23253
|
-
const score = scoreFactors.severity + scoreFactors.affectedUrls * AFFECTED_URL_WEIGHT[group.severity] + scoreFactors.searchVisibleUrls * 100 * valueWeight + scoreFactors.clicks * 20 * valueWeight + Math.min(scoreFactors.impressions, 1e4) / 25 * valueWeight + scoreFactors.sessions * 2 * valueWeight + (scoreFactors.conversions ?? 0) * 100 * valueWeight + scoreFactors.effortScore;
|
|
23254
|
-
const rule = explainRule(group.ruleId);
|
|
23255
|
-
const recommendation = recommendationForRule(group.ruleId);
|
|
23256
|
-
return {
|
|
23257
|
-
...group,
|
|
23258
|
-
recommendation,
|
|
23259
|
-
score: Math.round(score),
|
|
23260
|
-
scoreFactors,
|
|
23261
|
-
whyThisRanks: whyThisRanks(scoreFactors, recommendation, report.dataSources?.searchConsole.status, report.dataSources?.analytics.provider),
|
|
23262
|
-
howToFix: rule?.howToFix ?? "",
|
|
23263
|
-
howToVerify: rule?.howToVerify ?? "",
|
|
23264
|
-
verification: {
|
|
23265
|
-
command: verificationCommand(report, group.severity),
|
|
23266
|
-
expected: rule?.howToVerify ?? ""
|
|
23267
|
-
}
|
|
23268
|
-
};
|
|
23269
|
-
}).sort((a, b) => b.score - a.score || SEVERITY_SCORE[b.severity] - SEVERITY_SCORE[a.severity] || b.count - a.count || a.ruleId.localeCompare(b.ruleId)).slice(0, filters.limit ?? 10);
|
|
23270
|
-
}
|
|
23271
|
-
function reviewObservations(report, filters = {}) {
|
|
23272
|
-
return topFixes(report, {
|
|
23273
|
-
...filters,
|
|
23274
|
-
recommendation: "review"
|
|
23275
|
-
});
|
|
23276
|
-
}
|
|
23279
|
+
take() {
|
|
23280
|
+
const first = this.#items[0];
|
|
23281
|
+
if (!first) return void 0;
|
|
23282
|
+
const last = this.#items.pop();
|
|
23283
|
+
this.#positions.delete(first.url);
|
|
23284
|
+
if (last && last !== first) {
|
|
23285
|
+
this.#items[0] = last;
|
|
23286
|
+
this.#positions.set(last.url, 0);
|
|
23287
|
+
this.#bubbleDown(0);
|
|
23288
|
+
}
|
|
23289
|
+
return first;
|
|
23290
|
+
}
|
|
23291
|
+
#swap(left, right) {
|
|
23292
|
+
const leftItem = this.#items[left];
|
|
23293
|
+
const rightItem = this.#items[right];
|
|
23294
|
+
if (!leftItem || !rightItem) return;
|
|
23295
|
+
this.#items[left] = rightItem;
|
|
23296
|
+
this.#items[right] = leftItem;
|
|
23297
|
+
this.#positions.set(rightItem.url, left);
|
|
23298
|
+
this.#positions.set(leftItem.url, right);
|
|
23299
|
+
}
|
|
23300
|
+
#bubbleUp(start) {
|
|
23301
|
+
let index = start;
|
|
23302
|
+
while (index > 0) {
|
|
23303
|
+
const parent = Math.floor((index - 1) / 2);
|
|
23304
|
+
const item = this.#items[index];
|
|
23305
|
+
const parentItem = this.#items[parent];
|
|
23306
|
+
if (!item || !parentItem || !before(item, parentItem)) break;
|
|
23307
|
+
this.#swap(index, parent);
|
|
23308
|
+
index = parent;
|
|
23309
|
+
}
|
|
23310
|
+
}
|
|
23311
|
+
#bubbleDown(start) {
|
|
23312
|
+
let index = start;
|
|
23313
|
+
while (true) {
|
|
23314
|
+
const left = index * 2 + 1;
|
|
23315
|
+
const right = left + 1;
|
|
23316
|
+
let next = index;
|
|
23317
|
+
const item = this.#items[next];
|
|
23318
|
+
const leftItem = this.#items[left];
|
|
23319
|
+
const rightItem = this.#items[right];
|
|
23320
|
+
if (leftItem && item && before(leftItem, item)) next = left;
|
|
23321
|
+
const nextItem = this.#items[next];
|
|
23322
|
+
if (rightItem && nextItem && before(rightItem, nextItem)) next = right;
|
|
23323
|
+
if (next === index) return;
|
|
23324
|
+
this.#swap(index, next);
|
|
23325
|
+
index = next;
|
|
23326
|
+
}
|
|
23327
|
+
}
|
|
23328
|
+
};
|
|
23277
23329
|
// packages/core/dist/analyze/crawler/report.js
|
|
23278
23330
|
const MAX_CRAWL_PAGES = 1e4;
|
|
23279
23331
|
const MAX_CRAWL_CONCURRENCY = 16;
|
|
@@ -23443,6 +23495,10 @@ function sanitizeMessages(values = []) {
|
|
|
23443
23495
|
function compareText$15(left, right) {
|
|
23444
23496
|
return left < right ? -1 : left > right ? 1 : 0;
|
|
23445
23497
|
}
|
|
23498
|
+
function compareCaveats(left, right) {
|
|
23499
|
+
const rank = (value) => value.startsWith("Coverage incomplete:") ? 0 : 1;
|
|
23500
|
+
return rank(left) - rank(right) || compareText$15(left, right);
|
|
23501
|
+
}
|
|
23446
23502
|
function sortPages(pages) {
|
|
23447
23503
|
return [...pages].sort((left, right) => compareText$15(left.url, right.url));
|
|
23448
23504
|
}
|
|
@@ -23677,9 +23733,229 @@ function createCrawlReport(input) {
|
|
|
23677
23733
|
...input.externalLinkVerification ? { externalLinkVerification: sanitizeTenantValue(input.externalLinkVerification) } : {},
|
|
23678
23734
|
...input.siteChecks ? { siteChecks: sanitizeTenantValue(input.siteChecks) } : {},
|
|
23679
23735
|
warnings: sanitizeMessages(input.warnings).sort(compareText$15),
|
|
23680
|
-
caveats: sanitizeMessages(input.caveats).sort(
|
|
23736
|
+
caveats: sanitizeMessages(input.caveats).sort(compareCaveats)
|
|
23681
23737
|
};
|
|
23682
23738
|
}
|
|
23739
|
+
// packages/core/dist/analyze/crawler/crawl-caveats.js
|
|
23740
|
+
/** Reports rank a caveat with this prefix first and lead with it. */
|
|
23741
|
+
const CRAWL_COVERAGE_CAVEAT_PREFIX = "Coverage incomplete:";
|
|
23742
|
+
function rateLimitRerunCommand(config, intervalMs) {
|
|
23743
|
+
const scope = config.strategy === "health" ? " --health" : config.mode === "site" ? "" : ` --mode ${config.mode}`;
|
|
23744
|
+
const sitemap = config.sitemapUrl ? ` --sitemap-url ${config.sitemapUrl}` : "";
|
|
23745
|
+
const slowMs = Math.min(8e3, Math.max(2e3, intervalMs * 2));
|
|
23746
|
+
return `seo crawl ${config.url}${scope}${sitemap} --max-pages ${config.maxPages} --concurrency 1 --fetch-interval-cap 1 --fetch-interval-ms ${slowMs}`;
|
|
23747
|
+
}
|
|
23748
|
+
/** Set when the sitemaps list more URLs than the page limit let the crawl read. */
|
|
23749
|
+
function sitemapLimitCaveatInput(config, discovery, defaulted) {
|
|
23750
|
+
if (!discovery || discovery.urlsReturned < config.maxPages || !discovery.roots.some((root) => root.possiblyTruncated)) return;
|
|
23751
|
+
return {
|
|
23752
|
+
maxPages: config.maxPages,
|
|
23753
|
+
defaulted,
|
|
23754
|
+
ceiling: MAX_CRAWL_PAGES,
|
|
23755
|
+
health: config.strategy === "health",
|
|
23756
|
+
url: config.url
|
|
23757
|
+
};
|
|
23758
|
+
}
|
|
23759
|
+
function crawlCaveats(input) {
|
|
23760
|
+
const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
|
|
23761
|
+
if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
|
|
23762
|
+
if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
|
|
23763
|
+
if (input.rateLimit) {
|
|
23764
|
+
const { readPages, skippedUrls, rerunCommand } = input.rateLimit;
|
|
23765
|
+
caveats.push(`${CRAWL_COVERAGE_CAVEAT_PREFIX} read ${readPages} of ${readPages + skippedUrls} pages because the origin kept rate limiting the crawl (HTTP 429) after it backed off. Rerun slower: ${rerunCommand}`);
|
|
23766
|
+
}
|
|
23767
|
+
if (input.sitemapLimit) {
|
|
23768
|
+
const { maxPages, defaulted, ceiling, health, url } = input.sitemapLimit;
|
|
23769
|
+
caveats.push(`Checked only the first ${maxPages} sitemap URLs because the page limit is ${maxPages}${defaulted ? " by default" : ""}, and the sitemaps list more. Raise --max-pages, up to ${ceiling.toLocaleString("en-US")}, to check more: seo crawl ${url}${health ? " --health" : ""} --max-pages ${ceiling}`);
|
|
23770
|
+
}
|
|
23771
|
+
if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
|
|
23772
|
+
const renderingFallbackUrls = input.renderingFallbackUrls ?? 0;
|
|
23773
|
+
if (renderingFallbackUrls > 0) caveats.push(`JavaScript rendering could not run for ${renderingFallbackUrls} page${renderingFallbackUrls === 1 ? "" : "s"}, so ${renderingFallbackUrls === 1 ? "it was" : "they were"} audited from raw HTML. Client-rendered content, metadata and links may be missing. Install Chromium or set SEO_BROWSER_EXECUTABLE_PATH, then rerun.`);
|
|
23774
|
+
const queryVariantSkippedUrls = input.queryVariantSkippedUrls ?? 0;
|
|
23775
|
+
if (queryVariantSkippedUrls > 0) caveats.push(`Left ${queryVariantSkippedUrls} query-string variant${queryVariantSkippedUrls === 1 ? "" : "s"} unqueued after 5 variants of the same path, so tracking parameters do not fill the crawl.`);
|
|
23776
|
+
for (const issue of input.reachIssues ?? []) if (issue.detail) caveats.push(issue.detail);
|
|
23777
|
+
return caveats;
|
|
23778
|
+
}
|
|
23779
|
+
/**
|
|
23780
|
+
* A site crawl that found no internal links on the start page and no sitemap
|
|
23781
|
+
* URLs checked one page, so its other findings cannot stand for the site.
|
|
23782
|
+
*/
|
|
23783
|
+
function crawlReachIssues(input) {
|
|
23784
|
+
const [page] = input.pages;
|
|
23785
|
+
if (input.crawledPages !== 1 || input.pages.length !== 1 || !page || page.outgoingInternalCount > 0 || input.sitemapUrlsReturned > 0) return [];
|
|
23786
|
+
const rule = explainRule("crawl_single_page_reach");
|
|
23787
|
+
if (!rule) throw new Error("Missing rule guidance for crawl_single_page_reach");
|
|
23788
|
+
return [{
|
|
23789
|
+
ruleId: "crawl_single_page_reach",
|
|
23790
|
+
title: rule.title,
|
|
23791
|
+
category: rule.category,
|
|
23792
|
+
severity: rule.defaultSeverity,
|
|
23793
|
+
url: page.url,
|
|
23794
|
+
detail: page.usedJs ? "The rendered start page had 0 internal links and no sitemap returned URLs, so the crawl checked only this page. List the pages to check with --mode list." : "The start page had 0 internal links in its HTML and no sitemap returned URLs, so the crawl checked only this page. It probably builds its links with JavaScript. Rerun with --rendering on, or list the pages with --mode list.",
|
|
23795
|
+
evidence: {
|
|
23796
|
+
startUrl: input.startUrl,
|
|
23797
|
+
crawledPages: 1,
|
|
23798
|
+
outgoingInternalCount: 0,
|
|
23799
|
+
sitemapUrlsReturned: 0,
|
|
23800
|
+
rendered: Boolean(page.usedJs)
|
|
23801
|
+
}
|
|
23802
|
+
}];
|
|
23803
|
+
}
|
|
23804
|
+
// packages/core/dist/analyze/crawler/top-fixes.js
|
|
23805
|
+
const SEVERITY_SCORE = {
|
|
23806
|
+
high: 5e3,
|
|
23807
|
+
medium: 1200,
|
|
23808
|
+
low: 100
|
|
23809
|
+
};
|
|
23810
|
+
const EFFORT_BY_CATEGORY = {
|
|
23811
|
+
canonical: "medium",
|
|
23812
|
+
content: "high",
|
|
23813
|
+
agents: "low",
|
|
23814
|
+
headings: "low",
|
|
23815
|
+
images: "medium",
|
|
23816
|
+
indexability: "medium",
|
|
23817
|
+
international: "low",
|
|
23818
|
+
metadata: "low",
|
|
23819
|
+
mobile: "low",
|
|
23820
|
+
response: "medium",
|
|
23821
|
+
security: "medium",
|
|
23822
|
+
social: "low",
|
|
23823
|
+
"structured-data": "medium"
|
|
23824
|
+
};
|
|
23825
|
+
const EFFORT_SCORE = {
|
|
23826
|
+
low: 60,
|
|
23827
|
+
medium: 30,
|
|
23828
|
+
high: 0
|
|
23829
|
+
};
|
|
23830
|
+
const AFFECTED_URL_WEIGHT = {
|
|
23831
|
+
high: 25,
|
|
23832
|
+
medium: 15,
|
|
23833
|
+
low: 2
|
|
23834
|
+
};
|
|
23835
|
+
const VALUE_WEIGHT = {
|
|
23836
|
+
high: 1,
|
|
23837
|
+
medium: .75,
|
|
23838
|
+
low: .15
|
|
23839
|
+
};
|
|
23840
|
+
const CRAWL_REACH_RULES = /* @__PURE__ */ new Set(["crawl_single_page_reach"]);
|
|
23841
|
+
function reachRank(ruleId) {
|
|
23842
|
+
return CRAWL_REACH_RULES.has(ruleId) ? 0 : 1;
|
|
23843
|
+
}
|
|
23844
|
+
function matchesPattern(pattern, value) {
|
|
23845
|
+
if (!pattern.includes("*")) return value.includes(pattern);
|
|
23846
|
+
const parts = pattern.split("*");
|
|
23847
|
+
let position = 0;
|
|
23848
|
+
for (const part of parts) {
|
|
23849
|
+
if (!part) continue;
|
|
23850
|
+
const found = value.slice(position).indexOf(part);
|
|
23851
|
+
if (found === -1) return false;
|
|
23852
|
+
position += found + part.length;
|
|
23853
|
+
}
|
|
23854
|
+
return true;
|
|
23855
|
+
}
|
|
23856
|
+
function filteredIssues(report, filters) {
|
|
23857
|
+
return report.issues.filter((issue) => {
|
|
23858
|
+
if (filters.category && issue.category !== filters.category) return false;
|
|
23859
|
+
if (filters.severity && issue.severity !== filters.severity) return false;
|
|
23860
|
+
if (filters.url && !matchesPattern(filters.url, issue.url)) return false;
|
|
23861
|
+
if ((filters.recommendation ?? "fix") !== recommendationForRule(issue.ruleId)) return false;
|
|
23862
|
+
return true;
|
|
23863
|
+
});
|
|
23864
|
+
}
|
|
23865
|
+
function groupIssues(issues) {
|
|
23866
|
+
const groups = /* @__PURE__ */ new Map();
|
|
23867
|
+
for (const issue of issues) {
|
|
23868
|
+
const existing = groups.get(issue.ruleId) ?? {
|
|
23869
|
+
ruleId: issue.ruleId,
|
|
23870
|
+
title: issue.title,
|
|
23871
|
+
category: issue.category,
|
|
23872
|
+
severity: issue.severity,
|
|
23873
|
+
count: 0,
|
|
23874
|
+
sampleUrls: []
|
|
23875
|
+
};
|
|
23876
|
+
existing.count += 1;
|
|
23877
|
+
if (existing.sampleUrls.length < 10) existing.sampleUrls.push(issue.url);
|
|
23878
|
+
groups.set(issue.ruleId, existing);
|
|
23879
|
+
}
|
|
23880
|
+
return [...groups.values()];
|
|
23881
|
+
}
|
|
23882
|
+
function searchValueForGroup(report, urls) {
|
|
23883
|
+
const wanted = new Set(urls);
|
|
23884
|
+
const pages = report.pages.filter((page) => wanted.has(page.url) || wanted.has(page.finalUrl));
|
|
23885
|
+
const metrics = pages.map((page) => page.searchMetrics).filter((item) => Boolean(item));
|
|
23886
|
+
const clicks = metrics.reduce((sum, item) => sum + item.clicks, 0);
|
|
23887
|
+
const impressions = metrics.reduce((sum, item) => sum + item.impressions, 0);
|
|
23888
|
+
const sessions = pages.reduce((sum, page) => sum + (page.analytics?.sessions ?? 0), 0);
|
|
23889
|
+
const observedMetrics = new Set(report.dataSources?.analytics.observedMetrics ?? [
|
|
23890
|
+
"sessions",
|
|
23891
|
+
"totalUsers",
|
|
23892
|
+
"conversions"
|
|
23893
|
+
]);
|
|
23894
|
+
const totalUsers = observedMetrics.has("totalUsers") ? pages.reduce((sum, page) => sum + (page.analytics?.totalUsers ?? 0), 0) : void 0;
|
|
23895
|
+
const conversions = observedMetrics.has("conversions") ? pages.reduce((sum, page) => sum + (page.analytics?.conversions ?? 0), 0) : void 0;
|
|
23896
|
+
const avgPosition = metrics.length ? metrics.reduce((sum, item) => sum + item.position, 0) / metrics.length : void 0;
|
|
23897
|
+
return {
|
|
23898
|
+
searchVisibleUrls: metrics.length,
|
|
23899
|
+
clicks,
|
|
23900
|
+
impressions,
|
|
23901
|
+
sessions,
|
|
23902
|
+
totalUsers,
|
|
23903
|
+
conversions,
|
|
23904
|
+
avgPosition
|
|
23905
|
+
};
|
|
23906
|
+
}
|
|
23907
|
+
function whyThisRanks(input, recommendation, searchStatus, analyticsProvider) {
|
|
23908
|
+
const visibility = input.searchVisibleUrls ? `${input.searchVisibleUrls} affected URLs have GSC visibility (${input.clicks} clicks, ${input.impressions} impressions).` : searchStatus && !["joined", "none"].includes(searchStatus) ? `No joined GSC visibility is available for these affected URLs; Search Console evidence is ${searchStatus}.` : "No affected URL has joined GSC visibility yet.";
|
|
23909
|
+
const analyticsDetails = analyticsProvider ? analyticsProviderDetails(analyticsProvider) : analyticsProviderDetails("google");
|
|
23910
|
+
const analyticsLabel = analyticsDetails.label;
|
|
23911
|
+
const sessionLabel = analyticsDetails.landingMetricLabel;
|
|
23912
|
+
return `${visibility}${input.sessions ? ` ${analyticsLabel} adds ${input.sessions} ${sessionLabel}${input.conversions === void 0 ? "" : ` and ${input.conversions} conversions`} from affected pages.` : ""} ${recommendation === "review" ? "This is a review observation. Confirm it before scheduling implementation work." : `This ranks as a fix. Severity is ${input.severity}; ${input.affectedUrls} URL${input.affectedUrls === 1 ? " is" : "s are"} affected; estimated effort is ${input.effort}.`}`;
|
|
23913
|
+
}
|
|
23914
|
+
function verificationCommand(report, severity) {
|
|
23915
|
+
return `seo crawl ${report.config.url} --severity ${severity} --max-pages ${Math.min(report.config.maxPages, 100)}`;
|
|
23916
|
+
}
|
|
23917
|
+
function topFixes(report, filters = {}) {
|
|
23918
|
+
return groupIssues(filteredIssues(report, filters)).map((group) => {
|
|
23919
|
+
const search = searchValueForGroup(report, group.sampleUrls);
|
|
23920
|
+
const effort = EFFORT_BY_CATEGORY[group.category] ?? "medium";
|
|
23921
|
+
const scoreFactors = {
|
|
23922
|
+
severity: SEVERITY_SCORE[group.severity],
|
|
23923
|
+
affectedUrls: group.count,
|
|
23924
|
+
searchVisibleUrls: search.searchVisibleUrls,
|
|
23925
|
+
clicks: search.clicks,
|
|
23926
|
+
impressions: search.impressions,
|
|
23927
|
+
sessions: search.sessions,
|
|
23928
|
+
totalUsers: search.totalUsers,
|
|
23929
|
+
conversions: search.conversions,
|
|
23930
|
+
avgPosition: search.avgPosition,
|
|
23931
|
+
effort,
|
|
23932
|
+
effortScore: EFFORT_SCORE[effort]
|
|
23933
|
+
};
|
|
23934
|
+
const valueWeight = VALUE_WEIGHT[group.severity];
|
|
23935
|
+
const score = scoreFactors.severity + scoreFactors.affectedUrls * AFFECTED_URL_WEIGHT[group.severity] + scoreFactors.searchVisibleUrls * 100 * valueWeight + scoreFactors.clicks * 20 * valueWeight + Math.min(scoreFactors.impressions, 1e4) / 25 * valueWeight + scoreFactors.sessions * 2 * valueWeight + (scoreFactors.conversions ?? 0) * 100 * valueWeight + scoreFactors.effortScore;
|
|
23936
|
+
const rule = explainRule(group.ruleId);
|
|
23937
|
+
const recommendation = recommendationForRule(group.ruleId);
|
|
23938
|
+
return {
|
|
23939
|
+
...group,
|
|
23940
|
+
recommendation,
|
|
23941
|
+
score: Math.round(score),
|
|
23942
|
+
scoreFactors,
|
|
23943
|
+
whyThisRanks: whyThisRanks(scoreFactors, recommendation, report.dataSources?.searchConsole.status, report.dataSources?.analytics.provider),
|
|
23944
|
+
howToFix: rule?.howToFix ?? "",
|
|
23945
|
+
howToVerify: rule?.howToVerify ?? "",
|
|
23946
|
+
verification: {
|
|
23947
|
+
command: verificationCommand(report, group.severity),
|
|
23948
|
+
expected: rule?.howToVerify ?? ""
|
|
23949
|
+
}
|
|
23950
|
+
};
|
|
23951
|
+
}).sort((a, b) => reachRank(a.ruleId) - reachRank(b.ruleId) || b.score - a.score || SEVERITY_SCORE[b.severity] - SEVERITY_SCORE[a.severity] || b.count - a.count || a.ruleId.localeCompare(b.ruleId)).slice(0, filters.limit ?? 10);
|
|
23952
|
+
}
|
|
23953
|
+
function reviewObservations(report, filters = {}) {
|
|
23954
|
+
return topFixes(report, {
|
|
23955
|
+
...filters,
|
|
23956
|
+
recommendation: "review"
|
|
23957
|
+
});
|
|
23958
|
+
}
|
|
23683
23959
|
// packages/core/dist/analyze/crawler/site-checks-schema.js
|
|
23684
23960
|
const crawlSiteChecksSchema = z.object({ soft404: z.object({
|
|
23685
23961
|
status: z.enum([
|
|
@@ -24106,6 +24382,7 @@ const crawlPageSnapshotSchema = z.object({
|
|
|
24106
24382
|
source: z.enum([
|
|
24107
24383
|
"environment",
|
|
24108
24384
|
"playwright-cache",
|
|
24385
|
+
"agent-browser-cache",
|
|
24109
24386
|
"system"
|
|
24110
24387
|
]),
|
|
24111
24388
|
product: z.string(),
|
|
@@ -24794,14 +25071,6 @@ function agentReadyIssues(input) {
|
|
|
24794
25071
|
}
|
|
24795
25072
|
return issues;
|
|
24796
25073
|
}
|
|
24797
|
-
// packages/core/dist/analyze/crawler/crawl-caveats.js
|
|
24798
|
-
function crawlCaveats(input) {
|
|
24799
|
-
const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
|
|
24800
|
-
if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
|
|
24801
|
-
if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
|
|
24802
|
-
if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
|
|
24803
|
-
return caveats;
|
|
24804
|
-
}
|
|
24805
25074
|
// packages/core/dist/analyze/crawler/crawl-control.js
|
|
24806
25075
|
const CRAWL_CANCELLED = Symbol("crawl-cancelled");
|
|
24807
25076
|
function abortController(input) {
|
|
@@ -24843,83 +25112,94 @@ function crawlRssLimitBytes(totalMemoryBytes) {
|
|
|
24843
25112
|
function crawlMemoryPressure(input) {
|
|
24844
25113
|
return input.memoryUsage.heapUsed + input.memoryUsage.external >= crawlMemoryLimitBytes(input.totalMemoryBytes) || input.memoryUsage.rss >= crawlRssLimitBytes(input.totalMemoryBytes);
|
|
24845
25114
|
}
|
|
24846
|
-
// packages/core/dist/analyze/crawler/crawl-
|
|
24847
|
-
|
|
24848
|
-
|
|
25115
|
+
// packages/core/dist/analyze/crawler/crawl-rate-limit.js
|
|
25116
|
+
const RETRIES_PER_URL = 3;
|
|
25117
|
+
const BASE_BACKOFF_MS = 2e3;
|
|
25118
|
+
const MAX_BACKOFF_MS = 3e4;
|
|
25119
|
+
const COOLDOWN_BUDGET_MS = 18e4;
|
|
25120
|
+
const LIMITED_INTERVAL_MS = 1e3;
|
|
25121
|
+
const LIMITED_MAX_INTERVAL_MS = 4e3;
|
|
25122
|
+
function rateLimitEvidence(result, status, nowMs) {
|
|
25123
|
+
if (status !== 429 && status !== 503) return void 0;
|
|
25124
|
+
const header = retryAfterMs(result.page?.responseHeaders?.["retry-after"]);
|
|
25125
|
+
const cooldownUntil = Date.parse(result.page?.fetchDiagnostics?.backpressure?.cooldownUntil ?? "");
|
|
25126
|
+
const fromCooldown = Number.isFinite(cooldownUntil) ? Math.max(0, cooldownUntil - nowMs) : void 0;
|
|
25127
|
+
const wait = header ?? fromCooldown;
|
|
25128
|
+
if (status === 503 && wait === void 0) return void 0;
|
|
25129
|
+
return wait === void 0 ? {} : { retryAfterMs: Math.min(wait, MAX_BACKOFF_MS) };
|
|
24849
25130
|
}
|
|
24850
|
-
|
|
24851
|
-
|
|
24852
|
-
|
|
24853
|
-
|
|
24854
|
-
|
|
24855
|
-
|
|
24856
|
-
|
|
24857
|
-
|
|
24858
|
-
|
|
24859
|
-
|
|
24860
|
-
|
|
24861
|
-
|
|
24862
|
-
|
|
24863
|
-
|
|
24864
|
-
this
|
|
24865
|
-
}
|
|
24866
|
-
|
|
24867
|
-
|
|
24868
|
-
|
|
24869
|
-
|
|
24870
|
-
|
|
24871
|
-
|
|
24872
|
-
|
|
24873
|
-
|
|
24874
|
-
|
|
24875
|
-
take() {
|
|
24876
|
-
const first = this.#items[0];
|
|
24877
|
-
if (!first) return void 0;
|
|
24878
|
-
const last = this.#items.pop();
|
|
24879
|
-
this.#positions.delete(first.url);
|
|
24880
|
-
if (last && last !== first) {
|
|
24881
|
-
this.#items[0] = last;
|
|
24882
|
-
this.#positions.set(last.url, 0);
|
|
24883
|
-
this.#bubbleDown(0);
|
|
24884
|
-
}
|
|
24885
|
-
return first;
|
|
24886
|
-
}
|
|
24887
|
-
#swap(left, right) {
|
|
24888
|
-
const leftItem = this.#items[left];
|
|
24889
|
-
const rightItem = this.#items[right];
|
|
24890
|
-
if (!leftItem || !rightItem) return;
|
|
24891
|
-
this.#items[left] = rightItem;
|
|
24892
|
-
this.#items[right] = leftItem;
|
|
24893
|
-
this.#positions.set(rightItem.url, left);
|
|
24894
|
-
this.#positions.set(leftItem.url, right);
|
|
25131
|
+
/**
|
|
25132
|
+
* Rate-limit state for one crawl. After the first 429 the crawl drops to one
|
|
25133
|
+
* request at a time with a per-host interval and never speeds back up.
|
|
25134
|
+
*/
|
|
25135
|
+
var CrawlRateLimit = class {
|
|
25136
|
+
rate;
|
|
25137
|
+
responses = 0;
|
|
25138
|
+
skippedUrls = 0;
|
|
25139
|
+
pendingMs = 0;
|
|
25140
|
+
spentMs = 0;
|
|
25141
|
+
#level = 0;
|
|
25142
|
+
#budgetSpent = false;
|
|
25143
|
+
#retries = /* @__PURE__ */ new Map();
|
|
25144
|
+
constructor(rate) {
|
|
25145
|
+
this.rate = rate;
|
|
25146
|
+
}
|
|
25147
|
+
get limited() {
|
|
25148
|
+
return this.responses > 0;
|
|
25149
|
+
}
|
|
25150
|
+
get intervalMs() {
|
|
25151
|
+
return this.rate.intervalMs ?? LIMITED_INTERVAL_MS;
|
|
25152
|
+
}
|
|
25153
|
+
/** The cool-down before probes that run after the crawl. */
|
|
25154
|
+
get settleMs() {
|
|
25155
|
+
return Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** this.#level);
|
|
24895
25156
|
}
|
|
24896
|
-
|
|
24897
|
-
|
|
24898
|
-
|
|
24899
|
-
|
|
24900
|
-
|
|
24901
|
-
|
|
24902
|
-
|
|
24903
|
-
|
|
24904
|
-
|
|
24905
|
-
|
|
25157
|
+
/**
|
|
25158
|
+
* `retry` requeues the URL after `pendingMs`. `skip` gives it up, and
|
|
25159
|
+
* `stop` gives up the rest of the queue too once the budget is spent.
|
|
25160
|
+
*/
|
|
25161
|
+
observe(url, result, status, nowMs) {
|
|
25162
|
+
const evidence = rateLimitEvidence(result, status, nowMs);
|
|
25163
|
+
if (!evidence) return void 0;
|
|
25164
|
+
this.responses += 1;
|
|
25165
|
+
if (this.pendingMs === 0) this.#level += 1;
|
|
25166
|
+
const step = 2 ** Math.max(0, this.#level - 1);
|
|
25167
|
+
const waitMs = evidence.retryAfterMs ?? Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * step);
|
|
25168
|
+
this.rate = {
|
|
25169
|
+
...this.rate,
|
|
25170
|
+
concurrency: 1,
|
|
25171
|
+
intervalCap: 1,
|
|
25172
|
+
intervalMs: Math.min(LIMITED_MAX_INTERVAL_MS, Math.max(this.rate.intervalMs ?? 0, LIMITED_INTERVAL_MS * step))
|
|
25173
|
+
};
|
|
25174
|
+
const nextMs = Math.max(this.pendingMs, waitMs);
|
|
25175
|
+
if (this.spentMs + nextMs > COOLDOWN_BUDGET_MS) this.#budgetSpent = true;
|
|
25176
|
+
const retries = this.#retries.get(url) ?? 0;
|
|
25177
|
+
if (!this.#budgetSpent && retries < RETRIES_PER_URL) {
|
|
25178
|
+
this.#retries.set(url, retries + 1);
|
|
25179
|
+
this.pendingMs = nextMs;
|
|
25180
|
+
return "retry";
|
|
25181
|
+
}
|
|
25182
|
+
this.skippedUrls += 1;
|
|
25183
|
+
return this.#budgetSpent ? "stop" : "skip";
|
|
25184
|
+
}
|
|
25185
|
+
async settle(sleep, signal) {
|
|
25186
|
+
if (this.pendingMs <= 0) return;
|
|
25187
|
+
await sleep(this.pendingMs, signal);
|
|
25188
|
+
this.spentMs += this.pendingMs;
|
|
25189
|
+
this.pendingMs = 0;
|
|
25190
|
+
}
|
|
25191
|
+
caveatInput(config, readPages) {
|
|
25192
|
+
if (this.skippedUrls === 0) return void 0;
|
|
25193
|
+
return {
|
|
25194
|
+
readPages,
|
|
25195
|
+
skippedUrls: this.skippedUrls,
|
|
25196
|
+
rerunCommand: rateLimitRerunCommand(config, this.intervalMs)
|
|
25197
|
+
};
|
|
24906
25198
|
}
|
|
24907
|
-
|
|
24908
|
-
|
|
24909
|
-
|
|
24910
|
-
|
|
24911
|
-
const right = left + 1;
|
|
24912
|
-
let next = index;
|
|
24913
|
-
const item = this.#items[next];
|
|
24914
|
-
const leftItem = this.#items[left];
|
|
24915
|
-
const rightItem = this.#items[right];
|
|
24916
|
-
if (leftItem && item && before(leftItem, item)) next = left;
|
|
24917
|
-
const nextItem = this.#items[next];
|
|
24918
|
-
if (rightItem && nextItem && before(rightItem, nextItem)) next = right;
|
|
24919
|
-
if (next === index) return;
|
|
24920
|
-
this.#swap(index, next);
|
|
24921
|
-
index = next;
|
|
24922
|
-
}
|
|
25199
|
+
warning() {
|
|
25200
|
+
if (!this.limited) return void 0;
|
|
25201
|
+
const plural = (count, noun) => `${count} ${noun}${count === 1 ? "" : "s"}`;
|
|
25202
|
+
return `The origin rate limited ${plural(this.responses, "request")}. The crawl slowed to one request at a time, at least ${Math.round(this.intervalMs / 1e3)}s apart, and waited ${Math.round(this.spentMs / 1e3)}s in cool-downs${this.skippedUrls > 0 ? `, then left ${plural(this.skippedUrls, "URL")} unread` : " before reading every queued URL"}.`;
|
|
24923
25203
|
}
|
|
24924
25204
|
};
|
|
24925
25205
|
// packages/core/dist/analyze/crawler/external-link-checks.js
|
|
@@ -25154,11 +25434,15 @@ function observationFromPage(requestedUrl, page) {
|
|
|
25154
25434
|
};
|
|
25155
25435
|
}
|
|
25156
25436
|
// packages/core/dist/analyze/crawler/site-checks.js
|
|
25157
|
-
const PROBE_COUNT =
|
|
25437
|
+
const PROBE_COUNT = 3;
|
|
25158
25438
|
function probeUrls(startUrl) {
|
|
25159
25439
|
const origin = new URL(startUrl).origin;
|
|
25160
25440
|
const seed = createHash("sha256").update(origin).digest("hex").slice(0, 12);
|
|
25161
|
-
return
|
|
25441
|
+
return [
|
|
25442
|
+
`/${seed}-not-found`,
|
|
25443
|
+
`/${seed}-not-found/${seed}-missing`,
|
|
25444
|
+
`/.well-known/seo-audit/not-found-${seed}-1`
|
|
25445
|
+
].map((path) => new URL(path, origin).toString());
|
|
25162
25446
|
}
|
|
25163
25447
|
function probeObservation(url, result) {
|
|
25164
25448
|
const request = result.request;
|
|
@@ -25178,29 +25462,40 @@ function probeObservation(url, result) {
|
|
|
25178
25462
|
error: request.outcome === "failure" ? request.error : `Probe skipped: ${request.reason}`
|
|
25179
25463
|
};
|
|
25180
25464
|
}
|
|
25465
|
+
function isConclusive(probe) {
|
|
25466
|
+
return probe.status !== void 0 && !probe.accessBlocked && probe.status !== 429 && probe.status !== 503;
|
|
25467
|
+
}
|
|
25181
25468
|
async function collectCrawlSiteChecks(input) {
|
|
25469
|
+
const probe = async (url) => probeObservation(url, await input.fetchStatusPage(url, {
|
|
25470
|
+
js: "off",
|
|
25471
|
+
refresh: true,
|
|
25472
|
+
timeoutMs: input.timeoutMs,
|
|
25473
|
+
signal: input.signal,
|
|
25474
|
+
writeCache: false,
|
|
25475
|
+
respectRobots: input.respectRobots,
|
|
25476
|
+
...input.rate ? { rate: input.rate } : {}
|
|
25477
|
+
}));
|
|
25182
25478
|
const probes = [];
|
|
25183
25479
|
for (const url of probeUrls(input.startUrl)) {
|
|
25184
25480
|
if (input.signal?.aborted) break;
|
|
25185
|
-
|
|
25186
|
-
|
|
25187
|
-
|
|
25188
|
-
|
|
25189
|
-
|
|
25190
|
-
|
|
25191
|
-
|
|
25192
|
-
}
|
|
25193
|
-
probes.push(probeObservation(url, result));
|
|
25481
|
+
probes.push(await probe(url));
|
|
25482
|
+
}
|
|
25483
|
+
if (input.sleep && !input.signal?.aborted && probes.some((item) => !isConclusive(item))) {
|
|
25484
|
+
await input.sleep(input.retryDelayMs ?? 2e3, input.signal);
|
|
25485
|
+
for (const [index, item] of probes.entries()) {
|
|
25486
|
+
if (input.signal?.aborted) break;
|
|
25487
|
+
if (!isConclusive(item)) probes[index] = await probe(item.url);
|
|
25488
|
+
}
|
|
25194
25489
|
}
|
|
25195
|
-
const conclusive = probes.filter(
|
|
25490
|
+
const conclusive = probes.filter(isConclusive);
|
|
25196
25491
|
const successful = conclusive.filter((probe) => (probe.status ?? 0) >= 200 && (probe.status ?? 0) < 400);
|
|
25197
25492
|
const correctMissing = conclusive.filter((probe) => probe.status === 404 || probe.status === 410);
|
|
25198
|
-
const status =
|
|
25493
|
+
const status = successful.length > 0 ? "warning" : correctMissing.length === PROBE_COUNT ? "pass" : "unknown";
|
|
25199
25494
|
const checks = { soft404: {
|
|
25200
25495
|
status,
|
|
25201
25496
|
probes,
|
|
25202
25497
|
probeLimit: PROBE_COUNT,
|
|
25203
|
-
complete:
|
|
25498
|
+
complete: conclusive.length === PROBE_COUNT
|
|
25204
25499
|
} };
|
|
25205
25500
|
if (status !== "warning") return {
|
|
25206
25501
|
checks,
|
|
@@ -25216,7 +25511,7 @@ async function collectCrawlSiteChecks(input) {
|
|
|
25216
25511
|
category: rule.category,
|
|
25217
25512
|
severity: rule.defaultSeverity,
|
|
25218
25513
|
url: new URL("/", input.startUrl).toString(),
|
|
25219
|
-
detail: `${successful.length}
|
|
25514
|
+
detail: `${successful.length} of ${probes.length} made-up paths returned a successful response or redirect: ${successful.map((probe) => `${new URL(probe.url).pathname} (${probe.status})`).join(", ")}.`,
|
|
25220
25515
|
evidence: checks.soft404
|
|
25221
25516
|
}]
|
|
25222
25517
|
};
|
|
@@ -27648,6 +27943,18 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
27648
27943
|
}[connection.provider](connection);
|
|
27649
27944
|
}
|
|
27650
27945
|
// packages/core/dist/analyze/crawler/site-crawl-dependencies.js
|
|
27946
|
+
function sleep$2(ms, signal) {
|
|
27947
|
+
return new Promise((resolve) => {
|
|
27948
|
+
if (signal?.aborted) return resolve();
|
|
27949
|
+
const done = () => {
|
|
27950
|
+
clearTimeout(timer);
|
|
27951
|
+
signal?.removeEventListener("abort", done);
|
|
27952
|
+
resolve();
|
|
27953
|
+
};
|
|
27954
|
+
const timer = setTimeout(done, ms);
|
|
27955
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
27956
|
+
});
|
|
27957
|
+
}
|
|
27651
27958
|
function resolveCrawlSiteDependencies(dependencies = {}) {
|
|
27652
27959
|
const hasInjectedPerPageSearchProvider = Boolean(dependencies.queryPageMetrics || dependencies.queryPageTopQuery);
|
|
27653
27960
|
const hasInjectedLegacyBulkSearchProvider = Boolean(dependencies.queryPagesMetrics || dependencies.queryPagesTopQueries);
|
|
@@ -27668,6 +27975,7 @@ function resolveCrawlSiteDependencies(dependencies = {}) {
|
|
|
27668
27975
|
fetchLandingPageValues: dependencies.fetchLandingPageValues ?? fetchLandingPageValues,
|
|
27669
27976
|
landingValueForUrl: dependencies.landingValueForUrl ?? landingValueForUrl,
|
|
27670
27977
|
now: dependencies.now ?? (() => /* @__PURE__ */ new Date()),
|
|
27978
|
+
sleep: dependencies.sleep ?? sleep$2,
|
|
27671
27979
|
memoryUsage: dependencies.memoryUsage ?? (() => process.memoryUsage()),
|
|
27672
27980
|
totalMemory: dependencies.totalMemory ?? totalmem
|
|
27673
27981
|
};
|
|
@@ -27883,6 +28191,10 @@ async function crawlSite(input, dependencies) {
|
|
|
27883
28191
|
const queueSafetyDeferredUrls = /* @__PURE__ */ new Set();
|
|
27884
28192
|
const maxTrackedQueueSafetyUrls = Math.min(5e4, Math.max(1e3, config.maxPages * 5));
|
|
27885
28193
|
let queueSafetyInventoryCapped = false;
|
|
28194
|
+
const queryVariantsByPath = /* @__PURE__ */ new Map();
|
|
28195
|
+
const queryVariantSkipped = /* @__PURE__ */ new Set();
|
|
28196
|
+
let renderingFallbackUrls = 0;
|
|
28197
|
+
const rateLimit = new CrawlRateLimit(config.fetchRate);
|
|
27886
28198
|
let failedUrls = 0;
|
|
27887
28199
|
let observedInternalLinks = 0;
|
|
27888
28200
|
let memoryLimited = false;
|
|
@@ -27967,6 +28279,15 @@ async function crawlSite(input, dependencies) {
|
|
|
27967
28279
|
});
|
|
27968
28280
|
return;
|
|
27969
28281
|
}
|
|
28282
|
+
if (isCdnUtilityUrl(normalized)) {
|
|
28283
|
+
recordSkip("cdn-utility-url");
|
|
28284
|
+
emitStatus("url_skipped", {
|
|
28285
|
+
url: normalized,
|
|
28286
|
+
depth,
|
|
28287
|
+
reason: "cdn_utility_url"
|
|
28288
|
+
});
|
|
28289
|
+
return;
|
|
28290
|
+
}
|
|
27970
28291
|
if (isLikelyAsset(normalized)) {
|
|
27971
28292
|
recordSkip("asset-url");
|
|
27972
28293
|
emitStatus("url_skipped", {
|
|
@@ -27985,6 +28306,20 @@ async function crawlSite(input, dependencies) {
|
|
|
27985
28306
|
});
|
|
27986
28307
|
return;
|
|
27987
28308
|
}
|
|
28309
|
+
const parsed = new URL(normalized);
|
|
28310
|
+
const pathKey = parsed.search ? `${parsed.origin}${parsed.pathname}` : "";
|
|
28311
|
+
if (pathKey && (queryVariantsByPath.get(pathKey) ?? 0) >= 5) {
|
|
28312
|
+
if (!queryVariantSkipped.has(normalized) && queryVariantSkipped.size < maxTrackedQueueSafetyUrls) {
|
|
28313
|
+
queryVariantSkipped.add(normalized);
|
|
28314
|
+
recordSkip("query-variant-limit");
|
|
28315
|
+
emitStatus("url_skipped", {
|
|
28316
|
+
url: normalized,
|
|
28317
|
+
depth,
|
|
28318
|
+
reason: "query_variant_limit"
|
|
28319
|
+
});
|
|
28320
|
+
}
|
|
28321
|
+
return;
|
|
28322
|
+
}
|
|
27988
28323
|
if (queue.size + visited.size + inFlight.size >= config.maxPages * 5) {
|
|
27989
28324
|
if (!queueSafetyDeferredUrls.has(normalized)) {
|
|
27990
28325
|
if (queueSafetyDeferredUrls.size < maxTrackedQueueSafetyUrls) {
|
|
@@ -28000,6 +28335,7 @@ async function crawlSite(input, dependencies) {
|
|
|
28000
28335
|
}
|
|
28001
28336
|
return;
|
|
28002
28337
|
}
|
|
28338
|
+
if (pathKey) queryVariantsByPath.set(pathKey, (queryVariantsByPath.get(pathKey) ?? 0) + 1);
|
|
28003
28339
|
discovered.add(normalized);
|
|
28004
28340
|
queuedUrls += 1;
|
|
28005
28341
|
queue.push({
|
|
@@ -28056,7 +28392,7 @@ async function crawlSite(input, dependencies) {
|
|
|
28056
28392
|
js: config.js,
|
|
28057
28393
|
refresh: config.refresh,
|
|
28058
28394
|
timeoutMs: config.timeoutMs,
|
|
28059
|
-
rate:
|
|
28395
|
+
rate: rateLimit.rate,
|
|
28060
28396
|
signal,
|
|
28061
28397
|
writeCache: false,
|
|
28062
28398
|
respectRobots: config.respectRobots,
|
|
@@ -28087,7 +28423,11 @@ async function crawlSite(input, dependencies) {
|
|
|
28087
28423
|
});
|
|
28088
28424
|
};
|
|
28089
28425
|
while (!isCancelled() && (queue.size || inFlight.size) && (!memoryLimited || inFlight.size > 0) && pages.length < config.maxPages) {
|
|
28090
|
-
|
|
28426
|
+
if (rateLimit.pendingMs > 0 && inFlight.size === 0) {
|
|
28427
|
+
await rateLimit.settle(deps.sleep, signal);
|
|
28428
|
+
if (isCancelled()) break;
|
|
28429
|
+
}
|
|
28430
|
+
while (!isCancelled() && !memoryLimited && rateLimit.pendingMs === 0 && inFlight.size < activeConcurrency && pages.length + inFlight.size < config.maxPages) {
|
|
28091
28431
|
const item = nextQueueItem();
|
|
28092
28432
|
if (!item) break;
|
|
28093
28433
|
startTask(item);
|
|
@@ -28107,11 +28447,46 @@ async function crawlSite(input, dependencies) {
|
|
|
28107
28447
|
error: result.warning ?? "Crawler returned no page snapshot.",
|
|
28108
28448
|
extraction: "not-applicable"
|
|
28109
28449
|
});
|
|
28450
|
+
const limited = rateLimit.observe(task.url, result, request.outcome === "response" ? request.status : void 0, deps.now().getTime());
|
|
28451
|
+
if (limited) {
|
|
28452
|
+
activeConcurrency = 1;
|
|
28453
|
+
healthyProbeStreak = 0;
|
|
28454
|
+
}
|
|
28455
|
+
if (limited === "retry") {
|
|
28456
|
+
visited.delete(task.url);
|
|
28457
|
+
if (!queue.has(task.url)) queue.push({
|
|
28458
|
+
url: task.url,
|
|
28459
|
+
depth: task.depth
|
|
28460
|
+
});
|
|
28461
|
+
emitStatus("url_queued", {
|
|
28462
|
+
url: task.url,
|
|
28463
|
+
depth: task.depth,
|
|
28464
|
+
reason: "rate_limited_retry",
|
|
28465
|
+
message: `The origin rate limited ${task.url}. Waiting ${Math.ceil(rateLimit.pendingMs / 1e3)}s, then retrying one request at a time.`
|
|
28466
|
+
});
|
|
28467
|
+
continue;
|
|
28468
|
+
}
|
|
28469
|
+
if (limited) {
|
|
28470
|
+
requests.push(request);
|
|
28471
|
+
recordSkip("origin-backpressure");
|
|
28472
|
+
emitStatus("page_skipped", {
|
|
28473
|
+
url: task.url,
|
|
28474
|
+
depth: task.depth,
|
|
28475
|
+
reason: "origin-backpressure"
|
|
28476
|
+
});
|
|
28477
|
+
if (limited === "stop") {
|
|
28478
|
+
let left = 0;
|
|
28479
|
+
while (queue.take()) left += 1;
|
|
28480
|
+
recordSkips("origin-backpressure", left);
|
|
28481
|
+
rateLimit.skippedUrls += left;
|
|
28482
|
+
}
|
|
28483
|
+
continue;
|
|
28484
|
+
}
|
|
28110
28485
|
requests.push(request);
|
|
28111
28486
|
if (healthStrategy) {
|
|
28112
28487
|
if (request.outcome === "response" && request.status < 400 && !request.accessBlock) {
|
|
28113
28488
|
healthyProbeStreak += 1;
|
|
28114
|
-
activeConcurrency = Math.min(config.concurrency, 1 + Math.floor(healthyProbeStreak / 20));
|
|
28489
|
+
activeConcurrency = Math.min(rateLimit.limited ? 1 : config.concurrency, 1 + Math.floor(healthyProbeStreak / 20));
|
|
28115
28490
|
} else {
|
|
28116
28491
|
healthyProbeStreak = 0;
|
|
28117
28492
|
activeConcurrency = 1;
|
|
@@ -28152,6 +28527,9 @@ async function crawlSite(input, dependencies) {
|
|
|
28152
28527
|
continue;
|
|
28153
28528
|
}
|
|
28154
28529
|
const page = result.page;
|
|
28530
|
+
const rendering = page.fetchDiagnostics?.rendering;
|
|
28531
|
+
if (rendering?.status === "unavailable" && config.js === "on") throw new SeoError("INVALID_INPUT", `--rendering on needs a local Chrome or Chromium browser, and none could start. ${rendering.error ?? ""} Install Chromium, or set SEO_BROWSER_EXECUTABLE_PATH to a Chromium executable, then rerun. Use --rendering auto to accept raw HTML.`.replace(/\s+/g, " ").trim());
|
|
28532
|
+
if (rendering?.status === "unavailable" || rendering?.status === "failed") renderingFallbackUrls += 1;
|
|
28155
28533
|
const finalUrl = normalizeUrl$1(page.finalUrl, config.url);
|
|
28156
28534
|
if (!finalUrl) {
|
|
28157
28535
|
failedUrls += 1;
|
|
@@ -28181,7 +28559,7 @@ async function crawlSite(input, dependencies) {
|
|
|
28181
28559
|
const storeDocument = (urls) => {
|
|
28182
28560
|
const sourceRequest = normalizeUrl$1(task.url, config.url) ?? task.url;
|
|
28183
28561
|
const directlyRequested = sourceRequest === finalUrl;
|
|
28184
|
-
const normalizedLinks = urls.map((url) => normalizeUrl$1(url, finalUrl)).filter((url) => Boolean(url));
|
|
28562
|
+
const normalizedLinks = urls.map((url) => normalizeUrl$1(url, finalUrl)).filter((url) => Boolean(url) && !isCdnUtilityUrl(url ?? ""));
|
|
28185
28563
|
const syncLinkEvidence = (target, links) => {
|
|
28186
28564
|
const sortedLinks = [...links].sort((left, right) => left < right ? -1 : left > right ? 1 : 0);
|
|
28187
28565
|
target.outgoingInternalCount = sortedLinks.length;
|
|
@@ -28260,7 +28638,7 @@ async function crawlSite(input, dependencies) {
|
|
|
28260
28638
|
if (followLinks && task.depth < config.maxDepth) {
|
|
28261
28639
|
if (memoryLimited) for (const value of result.urls) {
|
|
28262
28640
|
const normalized = normalizeUrl$1(value, finalUrl);
|
|
28263
|
-
if (normalized && sameOrigin(normalized, origin) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
|
|
28641
|
+
if (normalized && sameOrigin(normalized, origin) && !isCdnUtilityUrl(normalized) && !isLikelyAsset(normalized) && passesFilters(normalized, config.include, config.exclude) && !visited.has(normalized) && !queue.has(normalized)) {
|
|
28264
28642
|
discovered.add(normalized);
|
|
28265
28643
|
memoryDeferredUrls.add(normalized);
|
|
28266
28644
|
}
|
|
@@ -28285,6 +28663,8 @@ async function crawlSite(input, dependencies) {
|
|
|
28285
28663
|
cancelled = isCancelled();
|
|
28286
28664
|
const robotsDisallowed = skipReasonCounts["robots-disallowed"] ?? 0;
|
|
28287
28665
|
if (robotsDisallowed > 0) warnings.push(`${robotsDisallowed} URL${robotsDisallowed === 1 ? "" : "s"} disallowed ${SEO_CRAWLER_USER_AGENT} in robots.txt. If the audit should reach them, allow the SEO-Skill token on only those public paths.`);
|
|
28666
|
+
const rateLimitWarning = rateLimit.warning();
|
|
28667
|
+
if (rateLimitWarning) warnings.push(rateLimitWarning);
|
|
28288
28668
|
if (cancelled) {
|
|
28289
28669
|
warnings.push("Crawl cancelled; returning a partial report.");
|
|
28290
28670
|
emitStatus("cancelled", {
|
|
@@ -28296,7 +28676,7 @@ async function crawlSite(input, dependencies) {
|
|
|
28296
28676
|
const sitemapEvidencePartial = sitemapDiscovery?.dataStatus === "partial";
|
|
28297
28677
|
const accessEvidencePartial = requests.some((request) => request.outcome === "response" && request.accessBlock);
|
|
28298
28678
|
const requiredSitemapUnavailable = config.mode === "sitemap" && sitemapDiscovery?.dataStatus === "unavailable";
|
|
28299
|
-
const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
|
|
28679
|
+
const partial = cancelled || failedUrls > 0 || pageLimitReached || robotsDeferredUrls > 0 || originBackpressureSkippedUrls > 0 || rateLimit.skippedUrls > 0 || queueSafetySkippedUrls > 0 || queue.size > 0 || inFlight.size > 0 || stoppedForMemory || sitemapEvidencePartial || accessEvidencePartial || requiredSitemapUnavailable;
|
|
28300
28680
|
const failed = !cancelled && pages.length === 0 && (requiredSitemapUnavailable || requests.length > 0 && requests.every((request) => request.outcome === "failure"));
|
|
28301
28681
|
const requestEvidenceStatus = cancelled && inFlight.size > 0 ? "partial" : "available";
|
|
28302
28682
|
const dataSources = await crawlDataSources({
|
|
@@ -28343,7 +28723,10 @@ async function crawlSite(input, dependencies) {
|
|
|
28343
28723
|
timeoutMs: config.timeoutMs,
|
|
28344
28724
|
fetchStatusPage: deps.fetchStatusPage,
|
|
28345
28725
|
respectRobots: config.respectRobots,
|
|
28346
|
-
signal
|
|
28726
|
+
signal,
|
|
28727
|
+
rate: rateLimit.rate,
|
|
28728
|
+
sleep: deps.sleep,
|
|
28729
|
+
retryDelayMs: rateLimit.settleMs
|
|
28347
28730
|
}) : void 0;
|
|
28348
28731
|
for (const page of pages) {
|
|
28349
28732
|
if (!page.geo || !llmsTxt) continue;
|
|
@@ -28374,6 +28757,12 @@ async function crawlSite(input, dependencies) {
|
|
|
28374
28757
|
...startPage.responseHeaders?.["content-signal"] ? { contentSignal: startPage.responseHeaders["content-signal"] } : {}
|
|
28375
28758
|
} } : {}
|
|
28376
28759
|
} : aiSignals?.signals;
|
|
28760
|
+
const reachIssues = followLinks && config.maxDepth > 0 && config.maxPages > 1 && !cancelled ? crawlReachIssues({
|
|
28761
|
+
startUrl: config.url,
|
|
28762
|
+
pages: pages.filter((page) => page.status >= 200 && page.status < 300),
|
|
28763
|
+
crawledPages: pages.length,
|
|
28764
|
+
sitemapUrlsReturned: sitemapDiscovery?.urlsReturned ?? 0
|
|
28765
|
+
}) : [];
|
|
28377
28766
|
const report = createCrawlReport({
|
|
28378
28767
|
config,
|
|
28379
28768
|
site,
|
|
@@ -28388,6 +28777,7 @@ async function crawlSite(input, dependencies) {
|
|
|
28388
28777
|
...externalLinkVerification ? { externalLinkVerification } : {},
|
|
28389
28778
|
...siteCheckResult ? { siteChecks: siteCheckResult.checks } : {},
|
|
28390
28779
|
additionalIssues: [
|
|
28780
|
+
...reachIssues,
|
|
28391
28781
|
...siteCheckResult?.issues ?? [],
|
|
28392
28782
|
...sitemapDiscovery ? sitemapHostIssues(sitemapDiscovery, config.url) : [],
|
|
28393
28783
|
...ai ? agentReadyIssues({
|
|
@@ -28405,7 +28795,12 @@ async function crawlSite(input, dependencies) {
|
|
|
28405
28795
|
queueSafetySkippedUrls,
|
|
28406
28796
|
queueSafetyInventoryCapped,
|
|
28407
28797
|
originBackpressureSkippedUrls,
|
|
28408
|
-
|
|
28798
|
+
rateLimit: rateLimit.caveatInput(config, pages.length),
|
|
28799
|
+
sitemapLimit: sitemapLimitCaveatInput(config, sitemapDiscovery, input.maxPages === void 0),
|
|
28800
|
+
memoryPressureSkippedUrls,
|
|
28801
|
+
queryVariantSkippedUrls: queryVariantSkipped.size,
|
|
28802
|
+
renderingFallbackUrls,
|
|
28803
|
+
reachIssues
|
|
28409
28804
|
}),
|
|
28410
28805
|
stats: {
|
|
28411
28806
|
discoveredUrls: discovered.size,
|
|
@@ -28958,7 +29353,7 @@ function analyzeCtrUnderperformersFromRows(input) {
|
|
|
28958
29353
|
// packages/core/dist/analyze/ctr-underperformers-report.js
|
|
28959
29354
|
const MAX_GSC_ROWS$3 = 1e5;
|
|
28960
29355
|
const DEFAULT_DAYS$3 = 28;
|
|
28961
|
-
const MAX_DAYS$
|
|
29356
|
+
const MAX_DAYS$3 = 548;
|
|
28962
29357
|
function plural$8(count, singular, pluralLabel = `${singular}s`) {
|
|
28963
29358
|
return count === 1 ? singular : pluralLabel;
|
|
28964
29359
|
}
|
|
@@ -28977,12 +29372,12 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
28977
29372
|
maximum: 100,
|
|
28978
29373
|
label: "limit"
|
|
28979
29374
|
});
|
|
28980
|
-
const explicitRange = explicitDateRange(input, MAX_DAYS$
|
|
29375
|
+
const explicitRange = explicitDateRange(input, MAX_DAYS$3);
|
|
28981
29376
|
const days = explicitRange?.days ?? integerOption$1({
|
|
28982
29377
|
value: input.days,
|
|
28983
29378
|
fallback: DEFAULT_DAYS$3,
|
|
28984
29379
|
minimum: 1,
|
|
28985
|
-
maximum: MAX_DAYS$
|
|
29380
|
+
maximum: MAX_DAYS$3,
|
|
28986
29381
|
label: "Days"
|
|
28987
29382
|
});
|
|
28988
29383
|
const now = dependencies.now();
|
|
@@ -32171,7 +32566,7 @@ function quickWinTemplateSummaries(items) {
|
|
|
32171
32566
|
}
|
|
32172
32567
|
// packages/core/dist/analyze/site-diagnostics/quick-wins-report.js
|
|
32173
32568
|
const DEFAULT_DAYS$2 = 28;
|
|
32174
|
-
const MAX_DAYS$
|
|
32569
|
+
const MAX_DAYS$2 = 548;
|
|
32175
32570
|
const DEFAULT_VERIFY_LIMIT$1 = 5;
|
|
32176
32571
|
const MAX_VERIFY_LIMIT = 100;
|
|
32177
32572
|
const MAX_SOURCE_ROWS$2 = 1e5;
|
|
@@ -32189,12 +32584,12 @@ function applyVerification$1(item, coverage) {
|
|
|
32189
32584
|
};
|
|
32190
32585
|
}
|
|
32191
32586
|
async function quickWinsReportWith(input, dependencies) {
|
|
32192
|
-
const explicitRange = explicitDateRange(input, MAX_DAYS$
|
|
32587
|
+
const explicitRange = explicitDateRange(input, MAX_DAYS$2);
|
|
32193
32588
|
const days = explicitRange?.days ?? integerOption$1({
|
|
32194
32589
|
value: input.days,
|
|
32195
32590
|
fallback: DEFAULT_DAYS$2,
|
|
32196
32591
|
minimum: 1,
|
|
32197
|
-
maximum: MAX_DAYS$
|
|
32592
|
+
maximum: MAX_DAYS$2,
|
|
32198
32593
|
label: "Days"
|
|
32199
32594
|
});
|
|
32200
32595
|
const verifyLimit = integerOption$1({
|
|
@@ -32613,7 +33008,7 @@ function analyzeStrikingDistanceRows(input) {
|
|
|
32613
33008
|
}
|
|
32614
33009
|
// packages/core/dist/analyze/striking-distance-report.js
|
|
32615
33010
|
const DEFAULT_DAYS$1 = 28;
|
|
32616
|
-
const MAX_DAYS = 548;
|
|
33011
|
+
const MAX_DAYS$1 = 548;
|
|
32617
33012
|
const DEFAULT_VERIFY_LIMIT = 5;
|
|
32618
33013
|
const MAX_SOURCE_ROWS$1 = 1e5;
|
|
32619
33014
|
function boundedInteger$2(value, fallback, minimum, maximum) {
|
|
@@ -32622,7 +33017,7 @@ function boundedInteger$2(value, fallback, minimum, maximum) {
|
|
|
32622
33017
|
}
|
|
32623
33018
|
function reportDays(value) {
|
|
32624
33019
|
if (value === void 0) return DEFAULT_DAYS$1;
|
|
32625
|
-
if (!Number.isInteger(value) || value < 1 || value > MAX_DAYS) throw new SeoError("INVALID_INPUT", `Days must be a whole number between 1 and ${MAX_DAYS}.`);
|
|
33020
|
+
if (!Number.isInteger(value) || value < 1 || value > MAX_DAYS$1) throw new SeoError("INVALID_INPUT", `Days must be a whole number between 1 and ${MAX_DAYS$1}.`);
|
|
32626
33021
|
return value;
|
|
32627
33022
|
}
|
|
32628
33023
|
function dateRange(days, now) {
|
|
@@ -32662,7 +33057,7 @@ function verdict$1(analysis) {
|
|
|
32662
33057
|
return `${countLabel(analysis.summary.eligibleRows, "eligible query/page row")} found; ${countLabel(analysis.summary.returnedRows, "row")} returned for review.`;
|
|
32663
33058
|
}
|
|
32664
33059
|
async function strikingDistanceWith(input, dependencies) {
|
|
32665
|
-
const explicitRange = explicitDateRange(input, MAX_DAYS);
|
|
33060
|
+
const explicitRange = explicitDateRange(input, MAX_DAYS$1);
|
|
32666
33061
|
const days = explicitRange?.days ?? reportDays(input.days);
|
|
32667
33062
|
const generatedAt = dependencies.now().toISOString();
|
|
32668
33063
|
const range = explicitRange?.range ?? dateRange(days, new Date(generatedAt));
|
|
@@ -41868,6 +42263,7 @@ const wait = (ms) => new Promise((done) => setTimeout(done, ms));
|
|
|
41868
42263
|
function requireInput(input) {
|
|
41869
42264
|
const needsKeywords = input.action === "keyword_metrics" || input.action === "serp";
|
|
41870
42265
|
if (needsKeywords && input.keywords.length === 0) throw new CommandError("INVALID_INPUT", `${input.action} needs keywords.`);
|
|
42266
|
+
if (input.action === "link_gap" && !input.target) throw new CommandError("INVALID_INPUT", "link_gap needs a target domain and competitors.");
|
|
41871
42267
|
if (!needsKeywords && !input.target) throw new CommandError("INVALID_INPUT", `${input.action} needs a target domain or URL.`);
|
|
41872
42268
|
}
|
|
41873
42269
|
async function eachKeyword(keywords, missed, rowsFor) {
|
|
@@ -41885,14 +42281,17 @@ async function eachKeyword(keywords, missed, rowsFor) {
|
|
|
41885
42281
|
}
|
|
41886
42282
|
async function research(provider, headers, input) {
|
|
41887
42283
|
requireInput(input);
|
|
42284
|
+
if (input.action === "link_gap") return linkGap(provider, headers, input);
|
|
41888
42285
|
const keys = need(headers, provider);
|
|
41889
42286
|
if (provider === "dataforseo") return dataForSeo(keys, input);
|
|
41890
42287
|
const missed = [];
|
|
42288
|
+
const rows = await ahrefs(keys[0] ?? "", input, missed);
|
|
41891
42289
|
return {
|
|
41892
|
-
rows
|
|
42290
|
+
rows,
|
|
41893
42291
|
costUsd: null,
|
|
41894
42292
|
partial: false,
|
|
41895
|
-
missed
|
|
42293
|
+
missed,
|
|
42294
|
+
noData: withoutData(input, rows, missed, 100)
|
|
41896
42295
|
};
|
|
41897
42296
|
}
|
|
41898
42297
|
function countryName(code) {
|
|
@@ -41944,7 +42343,8 @@ async function dataForSeoTask([login, password], path, task) {
|
|
|
41944
42343
|
if (result?.status_code === NO_RESULTS) return {
|
|
41945
42344
|
cost: result.cost ?? 0,
|
|
41946
42345
|
result: null,
|
|
41947
|
-
partial: false
|
|
42346
|
+
partial: false,
|
|
42347
|
+
noResults: true
|
|
41948
42348
|
};
|
|
41949
42349
|
if (!result || result.status_code >= 4e4 && result.status_code !== PARTIAL) throw Object.assign(new CommandError("PROVIDER_ERROR", `dataforseo: ${result?.status_message ?? "no task result"}`), { cost: result?.cost ?? 0 });
|
|
41950
42350
|
return {
|
|
@@ -41985,18 +42385,29 @@ async function dataForSeo(keys, input) {
|
|
|
41985
42385
|
partial: false,
|
|
41986
42386
|
missed: []
|
|
41987
42387
|
};
|
|
42388
|
+
const rows = await dataForSeoRows(keys, input, spent);
|
|
41988
42389
|
return {
|
|
41989
|
-
rows
|
|
42390
|
+
rows,
|
|
41990
42391
|
costUsd: Math.round(spent.cost * 1e4) / 1e4,
|
|
41991
42392
|
partial: spent.partial,
|
|
41992
|
-
missed: spent.missed
|
|
42393
|
+
missed: spent.missed,
|
|
42394
|
+
noData: withoutData(input, rows, spent.missed, 700)
|
|
41993
42395
|
};
|
|
41994
42396
|
}
|
|
42397
|
+
function withoutData(input, rows, missed, keywordLimit) {
|
|
42398
|
+
if (input.action === "keyword_metrics" || input.action === "serp") {
|
|
42399
|
+
const max = input.action === "serp" ? 5 : keywordLimit;
|
|
42400
|
+
const answered = new Set(rows.map((row) => String(row.keyword ?? "").toLowerCase()));
|
|
42401
|
+
const failed = new Set(missed);
|
|
42402
|
+
return input.keywords.slice(0, max).filter((keyword) => !failed.has(keyword) && !answered.has(keyword.toLowerCase()));
|
|
42403
|
+
}
|
|
42404
|
+
return [];
|
|
42405
|
+
}
|
|
41995
42406
|
async function dataForSeoRows(keys, input, spent) {
|
|
41996
42407
|
const language = dataForSeoLanguage(input.country);
|
|
41997
42408
|
if (!language && input.action !== "serp" && input.action !== "backlinks") throw new CommandError("INVALID_INPUT", `DataForSEO has no ${input.action} data for country "${input.country}".`);
|
|
41998
42409
|
const market = dataForSeoMarket(input.country, language);
|
|
41999
|
-
const post = async (path, task) => {
|
|
42410
|
+
const post = async (path, task, attempts = 1) => {
|
|
42000
42411
|
const done = await dataForSeoTask(keys, path, {
|
|
42001
42412
|
...market,
|
|
42002
42413
|
...task
|
|
@@ -42006,6 +42417,7 @@ async function dataForSeoRows(keys, input, spent) {
|
|
|
42006
42417
|
});
|
|
42007
42418
|
spent.cost += done.cost;
|
|
42008
42419
|
spent.partial ||= done.partial;
|
|
42420
|
+
if (done.noResults && attempts > 1) return post(path, task, attempts - 1);
|
|
42009
42421
|
return done.result?.items ?? [];
|
|
42010
42422
|
};
|
|
42011
42423
|
const get = (value, path) => path.split(".").reduce((current, key) => current && typeof current === "object" ? current[key] : void 0, value);
|
|
@@ -42035,7 +42447,7 @@ async function dataForSeoRows(keys, input, spent) {
|
|
|
42035
42447
|
return (await post("v3/serp/google/organic/live/advanced", {
|
|
42036
42448
|
keyword,
|
|
42037
42449
|
depth: Math.min(input.limit, 100)
|
|
42038
|
-
})).filter((item) => item.type === "organic").map((item) => {
|
|
42450
|
+
}, 2)).filter((item) => item.type === "organic").map((item) => {
|
|
42039
42451
|
const row = pick(item, {
|
|
42040
42452
|
position: "rank_absolute",
|
|
42041
42453
|
url: "url",
|
|
@@ -42043,9 +42455,11 @@ async function dataForSeoRows(keys, input, spent) {
|
|
|
42043
42455
|
title: "title"
|
|
42044
42456
|
});
|
|
42045
42457
|
if (/^https?:\/\/(www\.)?google\.[a-z.]+\/goto\b/.test(String(row.url))) {
|
|
42046
|
-
const
|
|
42458
|
+
const crumb = String(item.breadcrumb ?? "");
|
|
42459
|
+
const site = /^https?:\/\/([^/\s›]+)/.exec(crumb);
|
|
42047
42460
|
row.url = null;
|
|
42048
42461
|
row.domain = site?.[1] ?? null;
|
|
42462
|
+
row.url_hint = site ? crumb.replace(/^https?:\/\//, "") : null;
|
|
42049
42463
|
}
|
|
42050
42464
|
return {
|
|
42051
42465
|
keyword,
|
|
@@ -42075,6 +42489,93 @@ async function dataForSeoRows(keys, input, spent) {
|
|
|
42075
42489
|
organic_etv: "full_domain_metrics.organic.etv"
|
|
42076
42490
|
}));
|
|
42077
42491
|
}
|
|
42492
|
+
const GAP_DOMAINS = 1e3;
|
|
42493
|
+
const GAP_COMPETITORS = 5;
|
|
42494
|
+
const SHORTENERS = [
|
|
42495
|
+
"bit.ly",
|
|
42496
|
+
"tinyurl.com",
|
|
42497
|
+
"t.co",
|
|
42498
|
+
"ow.ly",
|
|
42499
|
+
"buff.ly",
|
|
42500
|
+
"is.gd",
|
|
42501
|
+
"cutt.ly",
|
|
42502
|
+
"rebrand.ly",
|
|
42503
|
+
"shorturl.at",
|
|
42504
|
+
"rb.gy",
|
|
42505
|
+
"lnkd.in"
|
|
42506
|
+
];
|
|
42507
|
+
const FREE_APP_HOSTS = [
|
|
42508
|
+
"web.app",
|
|
42509
|
+
"firebaseapp.com",
|
|
42510
|
+
"pages.dev"
|
|
42511
|
+
];
|
|
42512
|
+
const LINK_LIST_NAME = /(subdomain|whois|login|signin|coupon|promo|voucher|discount|sitelike|similarsite|sitesimilar|siteprice|sitevalue|siteworth|websiteworth|hypestat|statshow|websiteoutlook|siteinfo|domaininfo|domainstat|domaintools|ipaddress|dnslookup|linkdirectory|webdirectory|alternativeto-)/;
|
|
42513
|
+
function isLinkList(domain) {
|
|
42514
|
+
const within = (name) => domain === name || domain.endsWith(`.${name}`);
|
|
42515
|
+
if (SHORTENERS.some(within)) return true;
|
|
42516
|
+
if (FREE_APP_HOSTS.some((name) => domain.endsWith(`.${name}`))) return true;
|
|
42517
|
+
return LINK_LIST_NAME.test(domain.replace(/[-.]/g, ""));
|
|
42518
|
+
}
|
|
42519
|
+
async function linkGap(provider, headers, input) {
|
|
42520
|
+
const target = bareTarget(input.target).replace(/^www\./, "");
|
|
42521
|
+
const competitors = [...new Set((input.competitors ?? []).map((site) => bareTarget(site).replace(/^www\./, "")))].filter((site) => site && site !== target);
|
|
42522
|
+
if (competitors.length < 2 || competitors.length > GAP_COMPETITORS) throw new CommandError("INVALID_INPUT", `link_gap needs a target and 2 to ${GAP_COMPETITORS} competitors.`);
|
|
42523
|
+
const score = provider === "ahrefs" ? "domain_rating" : "rank";
|
|
42524
|
+
const sites = [target, ...competitors];
|
|
42525
|
+
const lists = [];
|
|
42526
|
+
let cost = 0;
|
|
42527
|
+
let partial = false;
|
|
42528
|
+
for (const site of sites) {
|
|
42529
|
+
const got = await research(provider, headers, {
|
|
42530
|
+
...input,
|
|
42531
|
+
action: "backlinks",
|
|
42532
|
+
target: site,
|
|
42533
|
+
limit: GAP_DOMAINS
|
|
42534
|
+
});
|
|
42535
|
+
cost += got.costUsd ?? 0;
|
|
42536
|
+
partial ||= got.partial;
|
|
42537
|
+
lists.push(got.rows);
|
|
42538
|
+
}
|
|
42539
|
+
const domainOf = (row) => String(row.domain ?? "").toLowerCase().replace(/^www\./, "");
|
|
42540
|
+
const authority = (row) => Number(row[score] ?? 0);
|
|
42541
|
+
const floors = lists.filter((rows) => rows.length >= GAP_DOMAINS).map((rows) => Math.min(...rows.map(authority)));
|
|
42542
|
+
const floor = floors.length ? Math.max(...floors) : null;
|
|
42543
|
+
const own = new Set((lists[0] ?? []).map(domainOf));
|
|
42544
|
+
const found = /* @__PURE__ */ new Map();
|
|
42545
|
+
competitors.forEach((site, index) => {
|
|
42546
|
+
for (const row of lists[index + 1] ?? []) {
|
|
42547
|
+
const domain = domainOf(row);
|
|
42548
|
+
if (!domain || own.has(domain) || domain === target) continue;
|
|
42549
|
+
if (domain.endsWith(`.${target}`)) continue;
|
|
42550
|
+
const entry = found.get(domain) ?? {
|
|
42551
|
+
value: 0,
|
|
42552
|
+
sites: []
|
|
42553
|
+
};
|
|
42554
|
+
entry.value = Math.max(entry.value, authority(row));
|
|
42555
|
+
if (!entry.sites.includes(site)) entry.sites.push(site);
|
|
42556
|
+
found.set(domain, entry);
|
|
42557
|
+
}
|
|
42558
|
+
});
|
|
42559
|
+
const gap = [...found.entries()].filter(([, entry]) => entry.sites.length >= 2 && (floor === null || entry.value >= floor));
|
|
42560
|
+
const linkLists = gap.map(([domain]) => domain).filter(isLinkList).sort((a, b) => a.localeCompare(b));
|
|
42561
|
+
return {
|
|
42562
|
+
rows: gap.filter(([domain]) => !isLinkList(domain)).sort(([a, left], [b, right]) => right.value - left.value || right.sites.length - left.sites.length || a.localeCompare(b)).map(([domain, entry]) => ({
|
|
42563
|
+
domain,
|
|
42564
|
+
[score]: entry.value,
|
|
42565
|
+
competitors_linking: entry.sites.length,
|
|
42566
|
+
links_to: entry.sites.join(" ")
|
|
42567
|
+
})),
|
|
42568
|
+
costUsd: provider === "dataforseo" ? Math.round(cost * 1e4) / 1e4 : null,
|
|
42569
|
+
partial,
|
|
42570
|
+
missed: [],
|
|
42571
|
+
noData: [],
|
|
42572
|
+
...floor !== null ? { floor: {
|
|
42573
|
+
field: score,
|
|
42574
|
+
value: floor
|
|
42575
|
+
} } : {},
|
|
42576
|
+
...linkLists.length ? { leftOut: linkLists } : {}
|
|
42577
|
+
};
|
|
42578
|
+
}
|
|
42078
42579
|
function isPage(target) {
|
|
42079
42580
|
return /\/./.test(bareTarget(target));
|
|
42080
42581
|
}
|
|
@@ -42165,7 +42666,7 @@ const DIMENSIONS = [
|
|
|
42165
42666
|
"device",
|
|
42166
42667
|
"searchAppearance"
|
|
42167
42668
|
];
|
|
42168
|
-
const ROWS = 25e3;
|
|
42669
|
+
const ROWS$1 = 25e3;
|
|
42169
42670
|
async function runRows(input, deps) {
|
|
42170
42671
|
const dimensions = input.dimensions?.length ? input.dimensions : ["query"];
|
|
42171
42672
|
const filters = input.filters ?? [];
|
|
@@ -42174,7 +42675,7 @@ async function runRows(input, deps) {
|
|
|
42174
42675
|
const query = {
|
|
42175
42676
|
dimensions,
|
|
42176
42677
|
type: "web",
|
|
42177
|
-
maxRows: ROWS,
|
|
42678
|
+
maxRows: ROWS$1,
|
|
42178
42679
|
...filters.length ? { dimensionFilterGroups: [{
|
|
42179
42680
|
groupType: "and",
|
|
42180
42681
|
filters
|
|
@@ -42231,9 +42732,9 @@ async function runRows(input, deps) {
|
|
|
42231
42732
|
const sum = (key) => rows.reduce((total, row) => total + Number(row[key] ?? 0), 0);
|
|
42232
42733
|
notes.push(`Totals: clicks ${sum("clicks")}, impressions ${sum("impressions")}; previous period clicks ${sum("previous_clicks")}.`);
|
|
42233
42734
|
}
|
|
42234
|
-
notes.push(`Compared with ${previous.startDate} to ${previous.endDate}${before.rowsFetched >= ROWS ? " (earlier period hit the row limit)" : ""}. 0 clicks with no position means the row had no impressions in the current period.`);
|
|
42735
|
+
notes.push(`Compared with ${previous.startDate} to ${previous.endDate}${before.rowsFetched >= ROWS$1 ? " (earlier period hit the row limit)" : ""}. 0 clicks with no position means the row had no impressions in the current period.`);
|
|
42235
42736
|
if (pageAt >= 0) notes.push("Page URLs with #fragments are merged into their page.");
|
|
42236
|
-
if (current.rowsFetched >= ROWS) notes.push(`Search Console has more than ${ROWS} rows; add a filter to narrow it.`);
|
|
42737
|
+
if (current.rowsFetched >= ROWS$1) notes.push(`Search Console has more than ${ROWS$1} rows; add a filter to narrow it.`);
|
|
42237
42738
|
notes.push("Search Console hides some rare queries for privacy, so query rows do not add up to the site totals.");
|
|
42238
42739
|
const detail = input.detail ?? "summary";
|
|
42239
42740
|
return {
|
|
@@ -42342,6 +42843,22 @@ const hostOf$2 = (url) => {
|
|
|
42342
42843
|
return null;
|
|
42343
42844
|
}
|
|
42344
42845
|
};
|
|
42846
|
+
const NOT_SOURCES = [
|
|
42847
|
+
"dataforseo.com",
|
|
42848
|
+
"images.openai.com",
|
|
42849
|
+
"gstatic.com",
|
|
42850
|
+
"googleusercontent.com",
|
|
42851
|
+
"ggpht.com",
|
|
42852
|
+
"example.com",
|
|
42853
|
+
"example.org",
|
|
42854
|
+
"example.net"
|
|
42855
|
+
];
|
|
42856
|
+
const IMAGE = /\.(png|jpe?g|gif|webp|avif|svg)(\?|#|$)/i;
|
|
42857
|
+
const isSource = (url) => {
|
|
42858
|
+
const host = hostOf$2(url);
|
|
42859
|
+
if (!host || IMAGE.test(url)) return false;
|
|
42860
|
+
return !NOT_SOURCES.some((name) => host === name || host.endsWith(`.${name}`));
|
|
42861
|
+
};
|
|
42345
42862
|
const empty = {
|
|
42346
42863
|
cited_url: null,
|
|
42347
42864
|
sources: [],
|
|
@@ -42370,7 +42887,7 @@ function judge(engine, task, site) {
|
|
|
42370
42887
|
const cited = all.find((url) => onSite(url, site.domain)) ?? null;
|
|
42371
42888
|
const base = {
|
|
42372
42889
|
cited_url: cited,
|
|
42373
|
-
sources: [...new Set(all.filter((url) => !onSite(url, site.domain)).map(hostOf$2).filter((host) => Boolean(host)))].slice(0, MAX_SOURCES),
|
|
42890
|
+
sources: [...new Set(all.filter((url) => !onSite(url, site.domain) && isSource(url)).map(hostOf$2).filter((host) => Boolean(host)))].slice(0, MAX_SOURCES),
|
|
42374
42891
|
model: shown.model,
|
|
42375
42892
|
answered_at
|
|
42376
42893
|
};
|
|
@@ -42423,6 +42940,7 @@ async function runAiAnswers(input, headers, deps, more) {
|
|
|
42423
42940
|
question,
|
|
42424
42941
|
impressions,
|
|
42425
42942
|
task: null,
|
|
42943
|
+
ref: null,
|
|
42426
42944
|
cost_usd: 0,
|
|
42427
42945
|
...empty
|
|
42428
42946
|
});
|
|
@@ -42434,7 +42952,8 @@ async function runAiAnswers(input, headers, deps, more) {
|
|
|
42434
42952
|
...blank("", null),
|
|
42435
42953
|
engine: engine.name,
|
|
42436
42954
|
result: "pending",
|
|
42437
|
-
task: value
|
|
42955
|
+
task: value,
|
|
42956
|
+
ref: value
|
|
42438
42957
|
};
|
|
42439
42958
|
rows.push(row);
|
|
42440
42959
|
waiting.push({
|
|
@@ -42487,6 +43006,7 @@ async function runAiAnswers(input, headers, deps, more) {
|
|
|
42487
43006
|
}
|
|
42488
43007
|
row.result = "pending";
|
|
42489
43008
|
row.task = `${engine.name}:${task.id}`;
|
|
43009
|
+
row.ref = row.task;
|
|
42490
43010
|
waiting.push({
|
|
42491
43011
|
row,
|
|
42492
43012
|
engine,
|
|
@@ -42527,6 +43047,7 @@ async function runAiAnswers(input, headers, deps, more) {
|
|
|
42527
43047
|
const cost = usd(rows.reduce((sum, row) => sum + row.cost_usd, 0));
|
|
42528
43048
|
const count = (result) => rows.filter((row) => row.result === result).length;
|
|
42529
43049
|
const left = rows.filter((row) => row.task).map((row) => row.task);
|
|
43050
|
+
const refs = rows.flatMap((row) => row.ref ? [row.ref] : []);
|
|
42530
43051
|
const cites = /* @__PURE__ */ new Map();
|
|
42531
43052
|
for (const row of rows) for (const host of row.sources) cites.set(host, (cites.get(host) ?? 0) + 1);
|
|
42532
43053
|
const topCited = [...cites].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10);
|
|
@@ -42560,7 +43081,7 @@ async function runAiAnswers(input, headers, deps, more) {
|
|
|
42560
43081
|
limit: 50,
|
|
42561
43082
|
detail: input.detail,
|
|
42562
43083
|
notes,
|
|
42563
|
-
more
|
|
43084
|
+
more: (offset) => more(offset, refs)
|
|
42564
43085
|
}),
|
|
42565
43086
|
data: {
|
|
42566
43087
|
provider: "dataforseo",
|
|
@@ -42574,6 +43095,7 @@ async function runAiAnswers(input, headers, deps, more) {
|
|
|
42574
43095
|
questions: [...new Set(rows.map((row) => row.question))],
|
|
42575
43096
|
costUsd: cost,
|
|
42576
43097
|
pendingTasks: left,
|
|
43098
|
+
tasks: refs,
|
|
42577
43099
|
citedSites: topCited.map(([host, answers]) => ({
|
|
42578
43100
|
host,
|
|
42579
43101
|
answers
|
|
@@ -42688,7 +43210,7 @@ function matchGa4WebStreams(site, candidates) {
|
|
|
42688
43210
|
}
|
|
42689
43211
|
// packages/core/dist/commands/analytics-site.js
|
|
42690
43212
|
const STREAM_CHECKS = 30;
|
|
42691
|
-
const AT_ONCE = 5;
|
|
43213
|
+
const AT_ONCE$1 = 5;
|
|
42692
43214
|
const TIED = 3;
|
|
42693
43215
|
const hostOf$1 = (site) => {
|
|
42694
43216
|
const value = site.replace(/^sc-domain:/, "");
|
|
@@ -42753,8 +43275,8 @@ async function lookProperty(access, site) {
|
|
|
42753
43275
|
}
|
|
42754
43276
|
const order = ranked(site, candidates).slice(0, STREAM_CHECKS);
|
|
42755
43277
|
let failures = 0;
|
|
42756
|
-
for (let start = 0; start < order.length; start += AT_ONCE) {
|
|
42757
|
-
const batch = order.slice(start, start + AT_ONCE);
|
|
43278
|
+
for (let start = 0; start < order.length; start += AT_ONCE$1) {
|
|
43279
|
+
const batch = order.slice(start, start + AT_ONCE$1);
|
|
42758
43280
|
const streams = await Promise.all(batch.map((candidate) => listAnalyticsDataStreams(candidate.google, candidate.property, { signIn: access.signIn }).catch(() => {
|
|
42759
43281
|
failures += 1;
|
|
42760
43282
|
return [];
|
|
@@ -42955,7 +43477,7 @@ async function totals$1(found, access, range, before, pages) {
|
|
|
42955
43477
|
notes
|
|
42956
43478
|
};
|
|
42957
43479
|
}
|
|
42958
|
-
const change$
|
|
43480
|
+
const change$2 = (value) => value === null ? "n/a" : `${value > 0 ? "+" : ""}${value}%`;
|
|
42959
43481
|
function engineText(rows) {
|
|
42960
43482
|
const shown = rows.filter((row) => row.sessions || row.previousSessions);
|
|
42961
43483
|
if (shown.length < 2) return "";
|
|
@@ -42965,7 +43487,7 @@ function aiText(ai) {
|
|
|
42965
43487
|
const list = (rows, key) => rows.slice(0, AI_ROWS).map((row) => `${row[key]} ${row.sessions} (was ${row.previousSessions})`).join(", ");
|
|
42966
43488
|
if (!ai.sessions && !ai.previousSessions) return "Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): none in either period.";
|
|
42967
43489
|
return [
|
|
42968
|
-
`Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): ${ai.sessions} sessions (was ${ai.previousSessions}, ${change$
|
|
43490
|
+
`Visits from AI assistants (ChatGPT, Perplexity, Claude, Gemini, Copilot and others): ${ai.sessions} sessions (was ${ai.previousSessions}, ${change$2(ai.sessionsChangePct)}). By source: ${list(ai.bySource, "source")}.`,
|
|
42969
43491
|
ai.topLandingPages.length ? `Top landing pages from AI assistants: ${list(ai.topLandingPages, "page")}.` : "",
|
|
42970
43492
|
"Only visits that pass a referrer or utm_source count; AI apps that send none show as Direct."
|
|
42971
43493
|
].filter(Boolean).join(" ");
|
|
@@ -43082,7 +43604,7 @@ async function analyticsBeside(access, site, range, before, pages) {
|
|
|
43082
43604
|
const keyEvents = organic.keyEvents || organic.previousKeyEvents ? `, Organic Search key events ${organic.keyEvents} (was ${organic.previousKeyEvents})` : "";
|
|
43083
43605
|
return {
|
|
43084
43606
|
text: [
|
|
43085
|
-
`Google Analytics, property ${data.propertyName} (${data.property}), matched by its web stream ${data.webStream}${pages ? `, landing pages whose path contains "${pages}"` : ""}: Organic Search sessions from Google ${google.sessions} (was ${google.previousSessions}, ${change$
|
|
43607
|
+
`Google Analytics, property ${data.propertyName} (${data.property}), matched by its web stream ${data.webStream}${pages ? `, landing pages whose path contains "${pages}"` : ""}: Organic Search sessions from Google ${google.sessions} (was ${google.previousSessions}, ${change$2(google.sessionsChangePct)}), from every search engine ${organic.sessions} (was ${organic.previousSessions}, ${change$2(organic.sessionsChangePct)})${keyEvents}, sessions from all channels ${all.sessions} (was ${all.previousSessions}, ${change$2(all.sessionsChangePct)}).`,
|
|
43086
43608
|
engineText(data.organicSearchByEngine),
|
|
43087
43609
|
"Sessions are Google Analytics’ own count, not Search Console clicks.",
|
|
43088
43610
|
...data.notes,
|
|
@@ -43309,7 +43831,7 @@ function auditPreview(snapshot) {
|
|
|
43309
43831
|
}
|
|
43310
43832
|
const WEEK_MS = 6048e5;
|
|
43311
43833
|
const day$1 = (ms) => ms === null ? "" : new Date(ms).toISOString().slice(0, 10);
|
|
43312
|
-
const count$
|
|
43834
|
+
const count$3 = (value) => value.toLocaleString("en-US");
|
|
43313
43835
|
function relative$2(value, origin) {
|
|
43314
43836
|
if (!value) return "";
|
|
43315
43837
|
return origin && value.startsWith(origin) ? value.slice(origin.length) || "/" : value;
|
|
@@ -43330,7 +43852,7 @@ function rowsTable(rows, checks) {
|
|
|
43330
43852
|
changed(row) ? row.prev_coverage ?? "" : "",
|
|
43331
43853
|
changed(row) ? day$1(row.changed_at) : "",
|
|
43332
43854
|
day$1(row.checked_at),
|
|
43333
|
-
`${count$
|
|
43855
|
+
`${count$3(row.clicks)} (${count$3(row.previous_clicks)})${row.lost ? " lost" : ""}`,
|
|
43334
43856
|
row.in_sitemap ? "yes" : "no",
|
|
43335
43857
|
row.google_canonical && row.google_canonical !== row.url ? relative$2(row.google_canonical, checks.origin) : "",
|
|
43336
43858
|
row.error ?? ""
|
|
@@ -43338,12 +43860,12 @@ function rowsTable(rows, checks) {
|
|
|
43338
43860
|
}
|
|
43339
43861
|
function head(site, checks) {
|
|
43340
43862
|
const { queued } = checks;
|
|
43341
|
-
const quota = checks.properties.map((item) => `${item.property} (${count$
|
|
43863
|
+
const quota = checks.properties.map((item) => `${item.property} (${count$3(item.urls)} URLs: ${count$3(item.top)} top pages checked daily, the rest every ${item.cycle_days === 1 ? "day" : `${item.cycle_days} days`}; ${count$3(item.used_today)} of ${count$3(checks.per_day)} checks used today)`).join(", ");
|
|
43342
43864
|
return [
|
|
43343
43865
|
`Google indexing of ${site}, from the URL Inspection results this server keeps for it (as of ${new Date(checks.as_of).toISOString().slice(0, 16).replace("T", " ")} UTC).`,
|
|
43344
|
-
checks.dropped_this_week || checks.returned_this_week ? `In the last 7 days ${count$
|
|
43345
|
-
`Covered: ${count$
|
|
43346
|
-
queued.total ? `Queued: ${count$
|
|
43866
|
+
checks.dropped_this_week || checks.returned_this_week ? `In the last 7 days ${count$3(checks.dropped_this_week)} ${checks.dropped_this_week === 1 ? "URL" : "URLs"} dropped out of Google's index and ${count$3(checks.returned_this_week)} came back; they are listed first. A top page's drop counts only after a second check an hour later.` : null,
|
|
43867
|
+
`Covered: ${count$3(checks.covered)} of ${count$3(checks.known)} known URLs have a current result. Known URLs are the sitemap's, the crawl's and the Search Console pages with clicks or impressions. Each property spends its full ${count$3(checks.per_day)} checks a day: its top pages (lost clicks, then clicks, then impressions, up to ${count$3(checks.top_per_day)}) daily, the rest as often as the quota allows, and a page the weekly crawl saw change at once.`,
|
|
43868
|
+
queued.total ? `Queued: ${count$3(queued.total)} (${count$3(queued.lost)} pages that lost their clicks, ${count$3(queued.search)} with clicks or impressions, ${count$3(queued.sitemap)} sitemap URLs never checked, ${count$3(queued.stale)} older or changed), checked in that order. Time left: ${etaText(checks.eta_hours)}, at up to ${count$3(checks.per_day)} checks a day per property, spread over the day.` : "Queued: nothing; every known URL has a current result.",
|
|
43347
43869
|
quota ? `Checked through ${quota}.` : null,
|
|
43348
43870
|
checks.stopped ? `Checks stopped early: ${checks.stopped}` : null,
|
|
43349
43871
|
"A URL without a result has none here, so this is not an all-clear for it."
|
|
@@ -43352,9 +43874,9 @@ function head(site, checks) {
|
|
|
43352
43874
|
function suggestionsText(checks) {
|
|
43353
43875
|
if (!checks.suggestions.length) return null;
|
|
43354
43876
|
return [
|
|
43355
|
-
`The sitemap has more URLs than these properties can check every 7 days. Each Search Console property gets its own ${count$
|
|
43877
|
+
`The sitemap has more URLs than these properties can check every 7 days. Each Search Console property gets its own ${count$3(checks.per_day)} checks a day, so adding these folders as URL-prefix properties (on the same Google account) would check more:`,
|
|
43356
43878
|
"property to add | sitemap URLs in it | more URLs checked a week",
|
|
43357
|
-
...checks.suggestions.map((item) => `${item.property} | ${count$
|
|
43879
|
+
...checks.suggestions.map((item) => `${item.property} | ${count$3(item.urls)} | ${count$3(item.gain)}`),
|
|
43358
43880
|
"The next day the checks find a new property in Search Console and use its quota for the URLs in that folder."
|
|
43359
43881
|
].join("\n");
|
|
43360
43882
|
}
|
|
@@ -43369,7 +43891,7 @@ function runAuditChecks(input, checks, call) {
|
|
|
43369
43891
|
data: checks
|
|
43370
43892
|
};
|
|
43371
43893
|
if (!checks.started) return {
|
|
43372
|
-
text: `${site} is in the weekly site audit, and its Google checks are starting now: pages that lost their clicks first, then pages by clicks and impressions, then sitemap URLs. ${checks.known ? `${count$
|
|
43894
|
+
text: `${site} is in the weekly site audit, and its Google checks are starting now: pages that lost their clicks first, then pages by clicks and impressions, then sitemap URLs. ${checks.known ? `${count$3(checks.known)} URLs are known so far. ` : ""}Run ${call("index", { site })} again in a few minutes for the coverage and the time left.`,
|
|
43373
43895
|
data: checks
|
|
43374
43896
|
};
|
|
43375
43897
|
const more = (offset) => input.paged ? `More: ${call("index", {
|
|
@@ -43393,7 +43915,7 @@ function runAuditChecks(input, checks, call) {
|
|
|
43393
43915
|
const reasons = checks.reasons.length ? [
|
|
43394
43916
|
"Google's reason at each URL's latest check:",
|
|
43395
43917
|
"reason | verdict | urls | changed to it this week",
|
|
43396
|
-
...checks.reasons.map((item) => `${item.coverage ?? ""} | ${item.verdict ?? ""} | ${count$
|
|
43918
|
+
...checks.reasons.map((item) => `${item.coverage ?? ""} | ${item.verdict ?? ""} | ${count$3(item.urls)} | ${count$3(item.changed_this_week)}`)
|
|
43397
43919
|
].join("\n") : "No URL has a result yet.";
|
|
43398
43920
|
const shown = checks.rows.slice(0, 20);
|
|
43399
43921
|
return {
|
|
@@ -43450,7 +43972,7 @@ async function bingFor(deps, property) {
|
|
|
43450
43972
|
missing: site ? `${site.url} is in the connected Bing account but is not verified, so Bing gives no data for it. Verify it at ${BING_HOME}.` : `The connected Bing account does not have ${property.replace(/^sc-domain:/, "")}. Add and verify it at ${BING_HOME} (it can import sites from Search Console). New sites take a few days to show data.`
|
|
43451
43973
|
};
|
|
43452
43974
|
}
|
|
43453
|
-
function days(range) {
|
|
43975
|
+
function days$1(range) {
|
|
43454
43976
|
return Math.round((Date.parse(range.endDate) - Date.parse(range.startDate)) / 864e5) + 1;
|
|
43455
43977
|
}
|
|
43456
43978
|
async function bingTotals(source, range, before) {
|
|
@@ -43463,7 +43985,7 @@ async function bingTotals(source, range, before) {
|
|
|
43463
43985
|
clicks: rows.reduce((total, row) => total + row.clicks, 0),
|
|
43464
43986
|
impressions: rows.reduce((total, row) => total + row.impressions, 0),
|
|
43465
43987
|
days: new Set(rows.map((row) => row.date)).size,
|
|
43466
|
-
expectedDays: days(window)
|
|
43988
|
+
expectedDays: days$1(window)
|
|
43467
43989
|
};
|
|
43468
43990
|
};
|
|
43469
43991
|
return {
|
|
@@ -43473,14 +43995,14 @@ async function bingTotals(source, range, before) {
|
|
|
43473
43995
|
invalidRows: traffic.invalidRows
|
|
43474
43996
|
};
|
|
43475
43997
|
}
|
|
43476
|
-
const change = (now, before) => {
|
|
43998
|
+
const change$1 = (now, before) => {
|
|
43477
43999
|
const value = pct(now, before);
|
|
43478
44000
|
return value === null ? "n/a" : `${value > 0 ? "+" : ""}${value}%`;
|
|
43479
44001
|
};
|
|
43480
44002
|
function enginesText(google, bing) {
|
|
43481
44003
|
if ("missing" in bing) return `Bing: ${bing.missing}`;
|
|
43482
44004
|
const { current: now, previous: then } = bing;
|
|
43483
|
-
const lines = [`Search engines, same dates. Google (Search Console): clicks ${google.clicks} (was ${google.previousClicks}, ${change(google.clicks, google.previousClicks)}), impressions ${google.impressions}. Bing (${bing.site}): clicks ${now.clicks} (was ${then.clicks}, ${change(now.clicks, then.clicks)}), impressions ${now.impressions} (was ${then.impressions}).`];
|
|
44005
|
+
const lines = [`Search engines, same dates. Google (Search Console): clicks ${google.clicks} (was ${google.previousClicks}, ${change$1(google.clicks, google.previousClicks)}), impressions ${google.impressions}. Bing (${bing.site}): clicks ${now.clicks} (was ${then.clicks}, ${change$1(now.clicks, then.clicks)}), impressions ${now.impressions} (was ${then.impressions}).`];
|
|
43484
44006
|
const both = google.clicks + now.clicks;
|
|
43485
44007
|
if (both > 0) lines.push(`Share of these clicks: Google ${Math.round(google.clicks / both * 100)}%, Bing ${Math.round(now.clicks / both * 100)}%.`);
|
|
43486
44008
|
if (now.days < now.expectedDays || then.days < then.expectedDays) lines.push(`Bing has data for ${now.days} of ${now.expectedDays} days (earlier period ${then.days} of ${then.expectedDays}), so its totals are partial.`);
|
|
@@ -43634,7 +44156,7 @@ async function runBingRows(input, deps) {
|
|
|
43634
44156
|
}
|
|
43635
44157
|
};
|
|
43636
44158
|
}
|
|
43637
|
-
function moved(rows, range, before, name, limit) {
|
|
44159
|
+
function moved$1(rows, range, before, name, limit) {
|
|
43638
44160
|
const now = sumWeeks(rows, range);
|
|
43639
44161
|
const then = sumWeeks(rows, before);
|
|
43640
44162
|
const old = new Map(then.rows.map((row) => [row.key, row]));
|
|
@@ -43672,8 +44194,8 @@ async function bingOverview(source, range, before) {
|
|
|
43672
44194
|
]);
|
|
43673
44195
|
if ("missing" in totals) throw new CommandError("NOT_CONNECTED", totals.missing);
|
|
43674
44196
|
const { current: now, previous: then } = totals;
|
|
43675
|
-
const q = moved(queries.rows, range, before, "query", 15);
|
|
43676
|
-
const p = moved(pages.rows, range, before, "page", 15);
|
|
44197
|
+
const q = moved$1(queries.rows, range, before, "query", 15);
|
|
44198
|
+
const p = moved$1(pages.rows, range, before, "page", 15);
|
|
43677
44199
|
return {
|
|
43678
44200
|
bingSite: source.site,
|
|
43679
44201
|
range,
|
|
@@ -43724,7 +44246,7 @@ function bingOverviewText(report, heading, origin, rowsPerList) {
|
|
|
43724
44246
|
};
|
|
43725
44247
|
return [
|
|
43726
44248
|
`${heading}, ${range.startDate} to ${range.endDate} against ${before.startDate} to ${before.endDate}.`,
|
|
43727
|
-
t.days || t.previousDays ? `Bing totals: clicks ${t.clicks} (was ${t.previousClicks}, ${change(t.clicks, t.previousClicks)}), impressions ${t.impressions} (was ${t.previousImpressions}, ${change(t.impressions, t.previousImpressions)}).` : "Bing totals: unknown. Bing has no daily data for these dates.",
|
|
44249
|
+
t.days || t.previousDays ? `Bing totals: clicks ${t.clicks} (was ${t.previousClicks}, ${change$1(t.clicks, t.previousClicks)}), impressions ${t.impressions} (was ${t.previousImpressions}, ${change$1(t.impressions, t.previousImpressions)}).` : "Bing totals: unknown. Bing has no daily data for these dates.",
|
|
43728
44250
|
group("pages", "page"),
|
|
43729
44251
|
group("queries", "query"),
|
|
43730
44252
|
`Limits:\n${report.caveats.map((line) => `- ${line}`).join("\n")}`
|
|
@@ -43801,13 +44323,13 @@ function clickyRange(input) {
|
|
|
43801
44323
|
endDate: input.endDate
|
|
43802
44324
|
};
|
|
43803
44325
|
}
|
|
43804
|
-
const count$
|
|
44326
|
+
const count$2 = (item) => {
|
|
43805
44327
|
const value = Number(item.value);
|
|
43806
44328
|
return Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
43807
44329
|
};
|
|
43808
44330
|
const totals = (items) => items.map((item) => ({
|
|
43809
44331
|
name: item.title ?? "",
|
|
43810
|
-
visits: count$
|
|
44332
|
+
visits: count$2(item)
|
|
43811
44333
|
})).filter((item) => Boolean(item.name) && item.visits !== null);
|
|
43812
44334
|
const line = (items) => {
|
|
43813
44335
|
const top = items.slice(0, 10).map((item) => `${item.name} ${item.visits}`);
|
|
@@ -43838,7 +44360,7 @@ async function runClickyRows(input, keys, options) {
|
|
|
43838
44360
|
const entrances = stats["pages-entrance"] ?? [];
|
|
43839
44361
|
const visitsByPage = /* @__PURE__ */ new Map();
|
|
43840
44362
|
for (const item of entrances) {
|
|
43841
|
-
const visits = count$
|
|
44363
|
+
const visits = count$2(item);
|
|
43842
44364
|
if (visits === null || !item.url) continue;
|
|
43843
44365
|
let page;
|
|
43844
44366
|
try {
|
|
@@ -43887,7 +44409,8 @@ async function runClickyRows(input, keys, options) {
|
|
|
43887
44409
|
// packages/core/dist/commands/domain-rating.js
|
|
43888
44410
|
const DR_ATTRIBUTION = "domain_rating: Domain Rating by Ahrefs (https://ahrefs.com/).";
|
|
43889
44411
|
const ENDPOINT = "https://api.ahrefs.com/v3/public/domain-rating-free";
|
|
43890
|
-
const MAX_DOMAINS =
|
|
44412
|
+
const MAX_DOMAINS = 50;
|
|
44413
|
+
const AT_ONCE = 25;
|
|
43891
44414
|
const TTL_SECONDS = 86400;
|
|
43892
44415
|
const DOMAIN_FIELDS = [
|
|
43893
44416
|
"domain",
|
|
@@ -43918,23 +44441,26 @@ async function lookup(domain, key) {
|
|
|
43918
44441
|
}
|
|
43919
44442
|
return body.domain_rating?.domain_rating ?? null;
|
|
43920
44443
|
}
|
|
43921
|
-
async function domainRatings(domains, key) {
|
|
44444
|
+
async function domainRatings(domains, key, max = MAX_DOMAINS) {
|
|
43922
44445
|
const found = /* @__PURE__ */ new Map();
|
|
43923
44446
|
if (!key) return found;
|
|
43924
|
-
const unique = [...new Set(domains.map(bareDomain).filter(Boolean))].slice(0,
|
|
43925
|
-
|
|
43926
|
-
|
|
43927
|
-
|
|
43928
|
-
|
|
44447
|
+
const unique = [...new Set(domains.map(bareDomain).filter(Boolean))].slice(0, max);
|
|
44448
|
+
for (let start = 0; start < unique.length; start += AT_ONCE) {
|
|
44449
|
+
const batch = unique.slice(start, start + AT_ONCE);
|
|
44450
|
+
const values = await Promise.all(batch.map((domain) => lookup(domain, key).catch(() => null)));
|
|
44451
|
+
batch.forEach((domain, index) => {
|
|
44452
|
+
found.set(domain, values[index] ?? null);
|
|
44453
|
+
});
|
|
44454
|
+
}
|
|
43929
44455
|
return found;
|
|
43930
44456
|
}
|
|
43931
|
-
async function withDomainRating(rows, key) {
|
|
44457
|
+
async function withDomainRating(rows, key, start = 0) {
|
|
43932
44458
|
const field = DOMAIN_FIELDS.find((name) => rows.some((row) => typeof row[name] === "string"));
|
|
43933
44459
|
if (!key || !field || rows.some((row) => "domain_rating" in row)) return {
|
|
43934
44460
|
rows,
|
|
43935
44461
|
added: false
|
|
43936
44462
|
};
|
|
43937
|
-
const ratings = await domainRatings(rows.map((row) => String(row[field] ?? "")), key);
|
|
44463
|
+
const ratings = await domainRatings(rows.slice(start).map((row) => String(row[field] ?? "")), key);
|
|
43938
44464
|
return {
|
|
43939
44465
|
rows: rows.map((row) => {
|
|
43940
44466
|
const next = {};
|
|
@@ -45167,6 +45693,265 @@ function temporaryRedirectNote(pages) {
|
|
|
45167
45693
|
if (!temporary.length) return null;
|
|
45168
45694
|
return `Note: 302 and 307 are temporary redirects; a permanent move usually uses 301 or 308. Temporary redirects here: ${temporary.map((page) => `${page.url} (${page.status} to ${page.redirects_to})`).join("; ")}.`;
|
|
45169
45695
|
}
|
|
45696
|
+
// packages/core/dist/commands/updates.js
|
|
45697
|
+
const UPDATES_SECTION = "google-updates";
|
|
45698
|
+
const UPDATES_SECTION_TEXT = "One Google ranking update (core, spam and similar, from Google’s Search Status Dashboard) laid over the site’s Search Console data: clicks and impressions from the day it started against the same number of days just before (whole weeks, up to 28 days), whether that change is outside the site’s own movement in the 8 comparisons before it (a heuristic), and the folders, pages and queries that gained or lost most. The latest update by default; with end_date, the latest update that started on or before it. Timing only, not cause.";
|
|
45699
|
+
const MAX_DAYS = 28;
|
|
45700
|
+
const BASELINE = 8;
|
|
45701
|
+
const ROWS = 25e3;
|
|
45702
|
+
const LISTED = 10;
|
|
45703
|
+
const SHOWN = 5;
|
|
45704
|
+
const MIN_CLICKS = 50;
|
|
45705
|
+
const days = (from, to) => Math.round((Date.parse(to) - Date.parse(from)) / 864e5) + 1;
|
|
45706
|
+
const asImpactUpdate = (update) => ({
|
|
45707
|
+
id: update.id,
|
|
45708
|
+
name: update.name,
|
|
45709
|
+
start: update.start,
|
|
45710
|
+
end: update.end ?? null,
|
|
45711
|
+
url: update.sourceUrl
|
|
45712
|
+
});
|
|
45713
|
+
function sum(rows, from, to) {
|
|
45714
|
+
let clicks = 0;
|
|
45715
|
+
let impressions = 0;
|
|
45716
|
+
for (const row of rows) {
|
|
45717
|
+
const day = row.keys[0] ?? "";
|
|
45718
|
+
if (day < from || day > to) continue;
|
|
45719
|
+
clicks += row.clicks;
|
|
45720
|
+
impressions += row.impressions;
|
|
45721
|
+
}
|
|
45722
|
+
return {
|
|
45723
|
+
clicks,
|
|
45724
|
+
impressions
|
|
45725
|
+
};
|
|
45726
|
+
}
|
|
45727
|
+
const change = (now, before) => before > 0 ? (now - before) / before * 100 : null;
|
|
45728
|
+
function joined(before, after) {
|
|
45729
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
45730
|
+
const at = (key) => {
|
|
45731
|
+
let row = byKey.get(key);
|
|
45732
|
+
if (!row) {
|
|
45733
|
+
row = {
|
|
45734
|
+
key,
|
|
45735
|
+
clicks: 0,
|
|
45736
|
+
previous: 0,
|
|
45737
|
+
impressions: 0,
|
|
45738
|
+
previous_impressions: 0
|
|
45739
|
+
};
|
|
45740
|
+
byKey.set(key, row);
|
|
45741
|
+
}
|
|
45742
|
+
return row;
|
|
45743
|
+
};
|
|
45744
|
+
for (const row of before) {
|
|
45745
|
+
const item = at(row.keys[0] ?? "");
|
|
45746
|
+
item.previous += row.clicks;
|
|
45747
|
+
item.previous_impressions += row.impressions;
|
|
45748
|
+
}
|
|
45749
|
+
for (const row of after) {
|
|
45750
|
+
const item = at(row.keys[0] ?? "");
|
|
45751
|
+
item.clicks += row.clicks;
|
|
45752
|
+
item.impressions += row.impressions;
|
|
45753
|
+
}
|
|
45754
|
+
return [...byKey.values()];
|
|
45755
|
+
}
|
|
45756
|
+
const moved = (row) => row.clicks - row.previous;
|
|
45757
|
+
const byKey = (a, b) => a.key < b.key ? -1 : a.key > b.key ? 1 : 0;
|
|
45758
|
+
function extremes(rows) {
|
|
45759
|
+
return {
|
|
45760
|
+
lost: rows.filter((row) => moved(row) < 0).sort((a, b) => moved(a) - moved(b) || byKey(a, b)).slice(0, LISTED),
|
|
45761
|
+
gained: rows.filter((row) => moved(row) > 0).sort((a, b) => moved(b) - moved(a) || byKey(a, b)).slice(0, LISTED)
|
|
45762
|
+
};
|
|
45763
|
+
}
|
|
45764
|
+
function folderOf(url) {
|
|
45765
|
+
try {
|
|
45766
|
+
const parts = new URL(url).pathname.split("/").filter(Boolean);
|
|
45767
|
+
return parts.length > 1 ? `/${parts[0]}/` : "/";
|
|
45768
|
+
} catch {
|
|
45769
|
+
return "/";
|
|
45770
|
+
}
|
|
45771
|
+
}
|
|
45772
|
+
function folders(pages) {
|
|
45773
|
+
const byFolder = /* @__PURE__ */ new Map();
|
|
45774
|
+
for (const page of pages) {
|
|
45775
|
+
const key = folderOf(page.key);
|
|
45776
|
+
const row = byFolder.get(key) ?? {
|
|
45777
|
+
key,
|
|
45778
|
+
clicks: 0,
|
|
45779
|
+
previous: 0,
|
|
45780
|
+
impressions: 0,
|
|
45781
|
+
previous_impressions: 0
|
|
45782
|
+
};
|
|
45783
|
+
row.clicks += page.clicks;
|
|
45784
|
+
row.previous += page.previous;
|
|
45785
|
+
row.impressions += page.impressions;
|
|
45786
|
+
row.previous_impressions += page.previous_impressions;
|
|
45787
|
+
byFolder.set(key, row);
|
|
45788
|
+
}
|
|
45789
|
+
return [...byFolder.values()].filter((row) => moved(row) !== 0).sort((a, b) => Math.abs(moved(b)) - Math.abs(moved(a)) || byKey(a, b)).slice(0, LISTED);
|
|
45790
|
+
}
|
|
45791
|
+
function windowDays(update, latest) {
|
|
45792
|
+
const start = update.start.slice(0, 10);
|
|
45793
|
+
const end = update.end?.slice(0, 10) ?? null;
|
|
45794
|
+
const wanted = Math.min(MAX_DAYS, Math.max(7, Math.ceil(days(start, end ?? latest) / 7) * 7));
|
|
45795
|
+
const covered = Math.floor(Math.max(0, days(start, latest)) / 7) * 7;
|
|
45796
|
+
return {
|
|
45797
|
+
start,
|
|
45798
|
+
end,
|
|
45799
|
+
wanted,
|
|
45800
|
+
n: Math.min(wanted, covered)
|
|
45801
|
+
};
|
|
45802
|
+
}
|
|
45803
|
+
async function updateImpact(site, update, read, now) {
|
|
45804
|
+
const { start, end, wanted, n } = windowDays(update, latestFinalGscDate(now));
|
|
45805
|
+
const empty = {
|
|
45806
|
+
update,
|
|
45807
|
+
site,
|
|
45808
|
+
before: null,
|
|
45809
|
+
after: null,
|
|
45810
|
+
normal_range: null,
|
|
45811
|
+
verdict: "not-enough-data",
|
|
45812
|
+
capped: {
|
|
45813
|
+
pages: false,
|
|
45814
|
+
queries: false
|
|
45815
|
+
},
|
|
45816
|
+
folders: [],
|
|
45817
|
+
pages: {
|
|
45818
|
+
lost: [],
|
|
45819
|
+
gained: []
|
|
45820
|
+
},
|
|
45821
|
+
queries: {
|
|
45822
|
+
lost: [],
|
|
45823
|
+
gained: []
|
|
45824
|
+
}
|
|
45825
|
+
};
|
|
45826
|
+
if (n < 7) return {
|
|
45827
|
+
...empty,
|
|
45828
|
+
window: "waiting"
|
|
45829
|
+
};
|
|
45830
|
+
const after = {
|
|
45831
|
+
startDate: start,
|
|
45832
|
+
endDate: shiftIsoDate(start, n - 1)
|
|
45833
|
+
};
|
|
45834
|
+
const before = {
|
|
45835
|
+
startDate: shiftIsoDate(start, -n),
|
|
45836
|
+
endDate: shiftIsoDate(start, -1)
|
|
45837
|
+
};
|
|
45838
|
+
const first = Math.ceil(n / 7);
|
|
45839
|
+
const pairs = Array.from({ length: BASELINE }, (_, i) => {
|
|
45840
|
+
const back = 7 * (first + i);
|
|
45841
|
+
return {
|
|
45842
|
+
after: [shiftIsoDate(start, -back), shiftIsoDate(start, n - 1 - back)],
|
|
45843
|
+
before: [shiftIsoDate(start, -back - n), shiftIsoDate(start, -back - 1)]
|
|
45844
|
+
};
|
|
45845
|
+
});
|
|
45846
|
+
const shared = sharedReader(read);
|
|
45847
|
+
const call = (range, dimension) => shared(site, {
|
|
45848
|
+
...range,
|
|
45849
|
+
dimensions: [dimension],
|
|
45850
|
+
type: "web",
|
|
45851
|
+
dataState: "final",
|
|
45852
|
+
...dimension === "date" ? {} : { maxRows: ROWS }
|
|
45853
|
+
});
|
|
45854
|
+
const [daily, pagesBefore, pagesAfter, queriesBefore, queriesAfter] = await Promise.all([
|
|
45855
|
+
call({
|
|
45856
|
+
startDate: pairs.at(-1)?.before[0],
|
|
45857
|
+
endDate: after.endDate
|
|
45858
|
+
}, "date"),
|
|
45859
|
+
call(before, "page"),
|
|
45860
|
+
call(after, "page"),
|
|
45861
|
+
call(before, "query"),
|
|
45862
|
+
call(after, "query")
|
|
45863
|
+
]);
|
|
45864
|
+
const totals = (range) => ({
|
|
45865
|
+
...range,
|
|
45866
|
+
...sum(daily.rows, range.startDate, range.endDate)
|
|
45867
|
+
});
|
|
45868
|
+
const was = totals(before);
|
|
45869
|
+
const is = totals(after);
|
|
45870
|
+
const seen = pairs.map((pair) => change(sum(daily.rows, pair.after[0], pair.after[1]).clicks, sum(daily.rows, pair.before[0], pair.before[1]).clicks)).filter((value) => value !== null);
|
|
45871
|
+
const normal = seen.length ? {
|
|
45872
|
+
comparisons: seen.length,
|
|
45873
|
+
clicks_pct: Math.round(Math.max(...seen.map((v) => Math.abs(v))) * 10) / 10
|
|
45874
|
+
} : null;
|
|
45875
|
+
const moves = change(is.clicks, was.clicks);
|
|
45876
|
+
const verdict = !normal || normal.comparisons < BASELINE / 2 || was.clicks < MIN_CLICKS || moves === null ? "not-enough-data" : Math.abs(moves) > normal.clicks_pct ? "outside-normal" : "within-normal";
|
|
45877
|
+
const pageRows = joined(pagesBefore.rows, pagesAfter.rows);
|
|
45878
|
+
return {
|
|
45879
|
+
...empty,
|
|
45880
|
+
window: end && n === wanted ? "full" : "partial",
|
|
45881
|
+
before: was,
|
|
45882
|
+
after: is,
|
|
45883
|
+
normal_range: normal,
|
|
45884
|
+
verdict,
|
|
45885
|
+
capped: {
|
|
45886
|
+
pages: pagesBefore.rows.length >= ROWS || pagesAfter.rows.length >= ROWS,
|
|
45887
|
+
queries: queriesBefore.rows.length >= ROWS || queriesAfter.rows.length >= ROWS
|
|
45888
|
+
},
|
|
45889
|
+
folders: folders(pageRows),
|
|
45890
|
+
pages: extremes(pageRows),
|
|
45891
|
+
queries: extremes(joined(queriesBefore.rows, queriesAfter.rows))
|
|
45892
|
+
};
|
|
45893
|
+
}
|
|
45894
|
+
const count$1 = (n) => n.toLocaleString("en-US");
|
|
45895
|
+
const dayLabel$1 = (date) => (/* @__PURE__ */ new Date(`${date}T00:00:00Z`)).toLocaleDateString("en-GB", {
|
|
45896
|
+
day: "numeric",
|
|
45897
|
+
month: "short",
|
|
45898
|
+
year: "numeric",
|
|
45899
|
+
timeZone: "UTC"
|
|
45900
|
+
});
|
|
45901
|
+
function trend(now, before) {
|
|
45902
|
+
const pct = change(now, before);
|
|
45903
|
+
if (pct === null) return `from ${count$1(before)}`;
|
|
45904
|
+
const rounded = Math.round(pct);
|
|
45905
|
+
return rounded === 0 ? `level with ${count$1(before)}` : `${rounded > 0 ? "up" : "down"} ${Math.abs(rounded)}% on ${count$1(before)}`;
|
|
45906
|
+
}
|
|
45907
|
+
function rowText(row, origin) {
|
|
45908
|
+
const name = row.key.startsWith(origin) ? row.key.slice(origin.length) || "/" : row.key;
|
|
45909
|
+
const diff = moved(row);
|
|
45910
|
+
return `${name} ${count$1(row.clicks)} (${diff > 0 ? "+" : ""}${count$1(diff)})`;
|
|
45911
|
+
}
|
|
45912
|
+
function impactLines(impact) {
|
|
45913
|
+
const { update, before, after } = impact;
|
|
45914
|
+
const origin = originOf(impact.site);
|
|
45915
|
+
const dates = `${dayLabel$1(update.start.slice(0, 10))} to ${update.end ? dayLabel$1(update.end.slice(0, 10)) : "now, still rolling out"}`;
|
|
45916
|
+
const lines = [`${update.name} (${dates}). [Google’s notice](${update.url})`];
|
|
45917
|
+
if (!before || !after) return [...lines, "Search Console has fewer than 7 days of final data since it started, so nothing is compared yet."];
|
|
45918
|
+
const n = days(after.startDate, after.endDate);
|
|
45919
|
+
lines.push(`Compared: ${dayLabel$1(after.startDate)} to ${dayLabel$1(after.endDate)} against the ${n} days before (same weekdays)${update.end && update.end.slice(0, 10) > after.endDate && impact.window === "full" ? `, the first ${n} days of the update` : ""}.${impact.window === "partial" ? " Final data does not cover the whole update yet, so this is part of it." : ""}`, `Clicks ${count$1(after.clicks)}, ${trend(after.clicks, before.clicks)}. Impressions ${count$1(after.impressions)}, ${trend(after.impressions, before.impressions)}.`);
|
|
45920
|
+
const normal = impact.normal_range;
|
|
45921
|
+
lines.push(impact.verdict === "not-enough-data" ? `Not enough data to say whether this is unusual for the site${before.clicks < MIN_CLICKS ? ` (${count$1(before.clicks)} clicks before the update)` : ""}.` : `${impact.verdict === "outside-normal" ? "Outside" : "Within"} the site’s normal movement: in the ${normal?.comparisons} earlier ${n}-day comparisons, clicks moved by at most ${normal?.clicks_pct}% (a heuristic).`);
|
|
45922
|
+
const list = (title, rows) => rows.length ? [`${title}: ${rows.slice(0, SHOWN).map((row) => rowText(row, origin)).join("; ")}.`] : [];
|
|
45923
|
+
lines.push(...list("Folders that moved most, by clicks", impact.folders), ...list("Pages that lost most", impact.pages.lost), ...list("Pages that gained most", impact.pages.gained), ...list("Queries that lost most", impact.queries.lost), ...list("Queries that gained most", impact.queries.gained));
|
|
45924
|
+
const capped = [impact.capped.pages ? "pages" : null, impact.capped.queries ? "queries" : null].filter(Boolean);
|
|
45925
|
+
if (capped.length) lines.push(`Search Console returned the row limit for ${capped.join(" and ")}, so those lists may miss rows.`);
|
|
45926
|
+
lines.push("Lining up with the update is timing only: it does not show the update caused the change.");
|
|
45927
|
+
return lines;
|
|
45928
|
+
}
|
|
45929
|
+
async function runUpdates(input, deps) {
|
|
45930
|
+
const now = (deps.now ?? (() => /* @__PURE__ */ new Date()))();
|
|
45931
|
+
if (input.endDate && !realIsoDate(input.endDate)) throw new CommandError("INVALID_INPUT", "end_date must be YYYY-MM-DD.");
|
|
45932
|
+
const cutoff = input.endDate ?? latestFinalGscDate(now);
|
|
45933
|
+
const updates = await rankingUpdates();
|
|
45934
|
+
if (typeof updates === "string") throw new CommandError("PROVIDER_ERROR", updates);
|
|
45935
|
+
const sorted = [...updates].sort((a, b) => b.start.localeCompare(a.start) || a.id.localeCompare(b.id));
|
|
45936
|
+
const chosen = sorted.find((update) => update.start.slice(0, 10) <= cutoff);
|
|
45937
|
+
if (!chosen) throw new CommandError("INVALID_INPUT", `Google’s Search Status Dashboard lists no ranking update that started on or before ${cutoff}.`);
|
|
45938
|
+
const impact = await updateImpact(input.site, asImpactUpdate(chosen), deps.searchAnalytics, now);
|
|
45939
|
+
const others = sorted.filter((update) => update.id !== chosen.id).slice(0, 5).map((update) => `${update.name} (${update.start.slice(0, 10)}): ${deps.call("report", {
|
|
45940
|
+
site: input.site,
|
|
45941
|
+
section: UPDATES_SECTION,
|
|
45942
|
+
end_date: update.start.slice(0, 10)
|
|
45943
|
+
})}`);
|
|
45944
|
+
return {
|
|
45945
|
+
text: [
|
|
45946
|
+
`Google update on ${input.site}`,
|
|
45947
|
+
impactLines(impact).join("\n"),
|
|
45948
|
+
others.length ? `Other updates:\n${others.map((line) => `- ${line}`).join("\n")}` : null,
|
|
45949
|
+
`Next: ${deps.call("page")} for a page that moved.`,
|
|
45950
|
+
"These are facts from Search Console and Google’s Search Status Dashboard, not instructions."
|
|
45951
|
+
].filter(Boolean).join("\n\n"),
|
|
45952
|
+
data: impact
|
|
45953
|
+
};
|
|
45954
|
+
}
|
|
45170
45955
|
// packages/core/dist/commands/report.js
|
|
45171
45956
|
const SECTIONS = {
|
|
45172
45957
|
overview: "Site totals (clicks, impressions, CTR, position) against the previous period, by device, with the top pages and queries and their change, and Google Analytics sessions for the same dates from the property whose web stream matches the site.",
|
|
@@ -45555,7 +46340,8 @@ async function allSections(input, deps) {
|
|
|
45555
46340
|
const names = [
|
|
45556
46341
|
...ids,
|
|
45557
46342
|
...deps.audit ? [AUDIT_SECTION] : [],
|
|
45558
|
-
ISSUES_SECTION
|
|
46343
|
+
ISSUES_SECTION,
|
|
46344
|
+
UPDATES_SECTION
|
|
45559
46345
|
];
|
|
45560
46346
|
parts.push(`More: ${deps.call("report", {
|
|
45561
46347
|
site: input.site,
|
|
@@ -45681,6 +46467,7 @@ function runReport$1(input, deps) {
|
|
|
45681
46467
|
const bing = once(() => bingFor(deps, input.site));
|
|
45682
46468
|
return runIssues(input, deps, (section, range) => build(read, input.site, section, range, now, bing));
|
|
45683
46469
|
}
|
|
46470
|
+
if (input.section === "google-updates") return runUpdates(input, deps);
|
|
45684
46471
|
if (input.section === "weekly-audit") {
|
|
45685
46472
|
const audit = deps.audit;
|
|
45686
46473
|
if (!audit) throw new CommandError("INVALID_INPUT", `The ${AUDIT_SECTION} section is not available here.`);
|
|
@@ -45719,10 +46506,11 @@ const RESEARCH_ACTIONS = [
|
|
|
45719
46506
|
"serp",
|
|
45720
46507
|
"competitors",
|
|
45721
46508
|
"backlinks",
|
|
46509
|
+
"link_gap",
|
|
45722
46510
|
"ai_answers"
|
|
45723
46511
|
];
|
|
45724
46512
|
const RESEARCH_PROVIDERS = ["dataforseo", "ahrefs"];
|
|
45725
|
-
const RESEARCH_HELP = `keyword_metrics (keywords: volume, difficulty, CPC), ranked_keywords (target: keywords a domain or URL ranks for), serp (keywords, up to 5: the ranking pages), competitors (target: domains that share keywords), backlinks (target: referring domains), ai_answers (DataForSEO only; target: the site. Without keywords it takes the site’s top 10 chat-style and long questions from Search Console’s AI-shaped queries itself. With up to 10 questions in keywords it asks those instead, and the target can be any domain, such as a competitor's. Each answer row also lists the other sites it cites, and the notes rank them. Answers come from DataForSEO's queue: answers not ready after about 40 seconds are returned as tasks, already paid for; call again with the same target and those tasks to read them at no cost. It asks ChatGPT search, Google AI Mode and Google AI Overviews each one and says whether the answer cites or mentions the site. DataForSEO’s queue prices are about $0.0072 a question across the three engines, so about $0.07 a run: tell the person before running it). Rows with a domain, and the target, get a free Ahrefs domain_rating column. Numbers are provider estimates, not Search Console data.`;
|
|
46513
|
+
const RESEARCH_HELP = `keyword_metrics (keywords: volume, difficulty, CPC), ranked_keywords (target: keywords a domain or URL ranks for), serp (keywords, up to 5: the ranking pages), competitors (target: domains that share keywords), backlinks (target: referring domains), link_gap (target and 2 to 5 competitors: referring domains that link to two or more competitors and not the target, highest authority first, from each site's top 1,000 referring domains; costs one backlinks call per site), ai_answers (DataForSEO only; target: the site. Without keywords it takes the site’s top 10 chat-style and long questions from Search Console’s AI-shaped queries itself. With up to 10 questions in keywords it asks those instead, and the target can be any domain, such as a competitor's. Each answer row also lists the other sites it cites, and the notes rank them. Answers come from DataForSEO's queue: answers not ready after about 40 seconds are returned as tasks, already paid for; call again with the same target and those tasks to read them at no cost. It asks ChatGPT search, Google AI Mode and Google AI Overviews each one and says whether the answer cites or mentions the site. DataForSEO’s queue prices are about $0.0072 a question across the three engines, so about $0.07 a run: tell the person before running it). Rows with a domain, and the target, get a free Ahrefs domain_rating column. Numbers are provider estimates, not Search Console data.`;
|
|
45726
46514
|
const GENERIC_PLATFORMS = [
|
|
45727
46515
|
"youtube.com",
|
|
45728
46516
|
"facebook.com",
|
|
@@ -45766,10 +46554,11 @@ async function runResearch(input, keys, options) {
|
|
|
45766
46554
|
const offset = input.offset ?? 0;
|
|
45767
46555
|
const language = input.provider === "dataforseo" ? dataForSeoLanguage(country) : null;
|
|
45768
46556
|
if (input.action === "ai_answers") throw new CommandError("INVALID_INPUT", "ai_answers runs on its own.");
|
|
45769
|
-
const { rows: found, costUsd, partial, missed } = await research(input.provider, keys, {
|
|
46557
|
+
const { rows: found, costUsd, partial, missed, noData, floor, leftOut } = await research(input.provider, keys, {
|
|
45770
46558
|
action: input.action,
|
|
45771
46559
|
keywords: input.keywords ?? [],
|
|
45772
46560
|
target,
|
|
46561
|
+
competitors: input.competitors ?? [],
|
|
45773
46562
|
country,
|
|
45774
46563
|
limit: Math.min(1e3, Math.max(100, offset + 50))
|
|
45775
46564
|
});
|
|
@@ -45777,8 +46566,10 @@ async function runResearch(input, keys, options) {
|
|
|
45777
46566
|
rows: found,
|
|
45778
46567
|
note: null
|
|
45779
46568
|
};
|
|
46569
|
+
const unresolved = rows.filter((row) => row.url === null && row.url_hint !== void 0).length;
|
|
45780
46570
|
const drKey = keys.get("x-ahrefs-key") ?? options.drKey;
|
|
45781
|
-
const
|
|
46571
|
+
const byDr = input.action === "link_gap" && input.provider === "dataforseo" && drKey;
|
|
46572
|
+
const rated = byDr ? await sortedByDomainRating(rows, drKey) : await withDomainRating(rows, drKey, offset);
|
|
45782
46573
|
const targetDr = target ? (await domainRatings([target], drKey)).get(bareDomain(target)) : void 0;
|
|
45783
46574
|
const hasTargetDr = targetDr !== void 0 && targetDr !== null;
|
|
45784
46575
|
return {
|
|
@@ -45793,6 +46584,11 @@ async function runResearch(input, keys, options) {
|
|
|
45793
46584
|
...costUsd !== null ? [`DataForSEO cost: $${costUsd}.`] : [],
|
|
45794
46585
|
...partial ? ["Partial: DataForSEO could not fetch some pages of results, so rows may be missing."] : [],
|
|
45795
46586
|
...missed.length ? [`Partial: ${input.provider} failed for ${missed.join(", ")}, so those keywords have no rows. Ask for them again.`] : [],
|
|
46587
|
+
...noData.length ? [noDataNote(input, noData)] : [],
|
|
46588
|
+
...byDr ? [rows.length > GAP_RATED ? `Sorted by Domain Rating for the top ${GAP_RATED} rows by DataForSEO rank; the ${rows.length - GAP_RATED} rows after them follow by rank, with no Domain Rating.` : "Sorted by Domain Rating, then DataForSEO rank."] : [],
|
|
46589
|
+
...leftOut?.length ? [`Left out ${leftOut.length} link lists, URL shorteners and directories that link to most sites (a guess from the host name): ${leftOut.join(", ")}.`] : [],
|
|
46590
|
+
...floor ? [`Cut at ${floor.field} ${floor.value}: at least one site has more than 1,000 referring domains, and only its top 1,000 were read, so domains below ${floor.field} ${floor.value} are left out rather than guessed.`] : [],
|
|
46591
|
+
...unresolved ? [unresolvedNote(input, unresolved, rows.length)] : [],
|
|
45796
46592
|
...note ? [note] : [],
|
|
45797
46593
|
...hasTargetDr ? [`${bareDomain(target)} domain_rating: ${targetDr}.`] : [],
|
|
45798
46594
|
...rated.added || hasTargetDr ? [DR_ATTRIBUTION] : []
|
|
@@ -45811,12 +46607,44 @@ async function runResearch(input, keys, options) {
|
|
|
45811
46607
|
costUsd,
|
|
45812
46608
|
partial,
|
|
45813
46609
|
missedKeywords: missed,
|
|
46610
|
+
noDataKeywords: noData,
|
|
46611
|
+
...input.action === "link_gap" ? {
|
|
46612
|
+
competitors: input.competitors ?? [],
|
|
46613
|
+
cutAt: floor ?? null,
|
|
46614
|
+
leftOut: leftOut ?? []
|
|
46615
|
+
} : {},
|
|
45814
46616
|
total: rated.rows.length,
|
|
45815
46617
|
offset,
|
|
45816
46618
|
rows: rated.rows.slice(offset, offset + 50)
|
|
45817
46619
|
}
|
|
45818
46620
|
};
|
|
45819
46621
|
}
|
|
46622
|
+
function noDataNote(input, keywords) {
|
|
46623
|
+
const list = keywords.join(", ");
|
|
46624
|
+
if (input.action === "serp") return `No results: ${input.provider} got an empty results page for ${list}${input.provider === "dataforseo" ? ", twice" : ""}. Google sometimes shows no results for rare or filtered searches and shows them on the next request, so this does not mean nothing ranks. Ask again later${input.provider === "dataforseo" ? ", or use provider ahrefs" : ""}.`;
|
|
46625
|
+
return `No data: ${input.provider} has no metrics for ${keywords.length} of the keywords asked for (${list}). Their volume is unknown, not zero.`;
|
|
46626
|
+
}
|
|
46627
|
+
function unresolvedNote(input, count, total) {
|
|
46628
|
+
return `${count} of ${total} results have no exact page: ${input.provider} returned Google redirect links for them. Their domain and url_hint come from Google's breadcrumb, which is shortened.${input.provider === "dataforseo" ? " For exact pages, use provider ahrefs." : ""}`;
|
|
46629
|
+
}
|
|
46630
|
+
const GAP_RATED = 200;
|
|
46631
|
+
async function sortedByDomainRating(rows, key) {
|
|
46632
|
+
const top = rows.slice(0, GAP_RATED);
|
|
46633
|
+
const ratings = await domainRatings(top.map((row) => String(row.domain ?? "")), key, GAP_RATED);
|
|
46634
|
+
const withDr = (row, rated) => {
|
|
46635
|
+
const { domain = null, ...rest } = row;
|
|
46636
|
+
return {
|
|
46637
|
+
domain,
|
|
46638
|
+
domain_rating: rated ? ratings.get(bareDomain(String(domain ?? ""))) ?? null : null,
|
|
46639
|
+
...rest
|
|
46640
|
+
};
|
|
46641
|
+
};
|
|
46642
|
+
const dr = (row) => Number(row.domain_rating ?? -1);
|
|
46643
|
+
return {
|
|
46644
|
+
rows: [...top.map((row) => withDr(row, true)).sort((a, b) => dr(b) - dr(a) || Number(b.rank ?? 0) - Number(a.rank ?? 0) || String(a.domain).localeCompare(String(b.domain))), ...rows.slice(GAP_RATED).map((row) => withDr(row, false))],
|
|
46645
|
+
added: true
|
|
46646
|
+
};
|
|
46647
|
+
}
|
|
45820
46648
|
// packages/core/dist/commands/sitemap-urls.js
|
|
45821
46649
|
const MAX_FILES = 20;
|
|
45822
46650
|
const MAX_URLS = 5e4;
|
|
@@ -46395,6 +47223,23 @@ function siteLines(site, startMs, endMs) {
|
|
|
46395
47223
|
} else lines.push("Search clicks: not available this week.");
|
|
46396
47224
|
return lines;
|
|
46397
47225
|
}
|
|
47226
|
+
function updateLines(site) {
|
|
47227
|
+
return (site.google_updates ?? []).map((update) => {
|
|
47228
|
+
const when = update.end ? "ran" : "has been rolling out";
|
|
47229
|
+
const result = update.result;
|
|
47230
|
+
if (!result && update.ready) return `- Google update: the ${update.name} ran ${update.start.slice(0, 10)} to ${update.end?.slice(0, 10) ?? "now"}. Ask for the report's google-updates section on this site for what moved.`;
|
|
47231
|
+
if (!result) return `- Google update: the ${update.name} ${when}${update.end ? ` (${update.start.slice(0, 10)} to ${update.end.slice(0, 10)})` : ` since ${update.start.slice(0, 10)}`}. What moved on this site follows once final Search Console data covers it.`;
|
|
47232
|
+
const diff = result.clicks - result.previous;
|
|
47233
|
+
const pct = result.previous > 0 ? Math.round(diff / result.previous * 100) : null;
|
|
47234
|
+
const moved = pct === null ? `${count(result.clicks)} clicks from ${count(result.previous)}` : pct === 0 ? `clicks level at ${count(result.clicks)}` : `clicks ${pct > 0 ? "up" : "down"} ${Math.abs(pct)}% (${count(result.clicks)} from ${count(result.previous)})`;
|
|
47235
|
+
const verdict = {
|
|
47236
|
+
"outside-normal": "more than this site usually moves",
|
|
47237
|
+
"within-normal": "within this site’s normal movement",
|
|
47238
|
+
"not-enough-data": "too little data to say if that is unusual"
|
|
47239
|
+
}[result.verdict];
|
|
47240
|
+
return `- Google update: the ${update.name}, ${moved} over the ${result.days} days from its start against the ${result.days} before, ${verdict} (timing only, not cause).`;
|
|
47241
|
+
});
|
|
47242
|
+
}
|
|
46398
47243
|
function opportunityLines(site) {
|
|
46399
47244
|
const found = site.opportunities;
|
|
46400
47245
|
if (!found) return [];
|
|
@@ -46447,7 +47292,7 @@ function weekMarkdown(data) {
|
|
|
46447
47292
|
"",
|
|
46448
47293
|
attention.length ? total === 1 ? "Your site had changes worth a look this week." : `${attention.length} of your ${total} sites had changes worth a look this week.` : quiet.length ? total === 1 ? "Your site had a quiet week." : quiet.length === total ? `All ${total} of your sites had a quiet week.` : `${quiet.length} of your ${total} sites had a quiet week.` : "Your sites are still on their first checks."
|
|
46449
47294
|
];
|
|
46450
|
-
for (const { site, found } of rows) out.push("", `## ${siteName(site.property)}${found.length ? `: ${list(found.map((item) => item.heading))}` : ""}`, "", ...siteLines(site, startMs, endMs).map((line) => `- ${line}`), ...opportunityLines(site), ...measuredLines(site), ...site.link ? [`- [View site](${site.link})`] : []);
|
|
47295
|
+
for (const { site, found } of rows) out.push("", `## ${siteName(site.property)}${found.length ? `: ${list(found.map((item) => item.heading))}` : ""}`, "", ...siteLines(site, startMs, endMs).map((line) => `- ${line}`), ...opportunityLines(site), ...measuredLines(site), ...updateLines(site), ...site.link ? [`- [View site](${site.link})`] : []);
|
|
46451
47296
|
return out.join("\n");
|
|
46452
47297
|
}
|
|
46453
47298
|
async function runAllSites(deps, sites) {
|
|
@@ -46550,10 +47395,14 @@ const SHARED_TOOLS = [
|
|
|
46550
47395
|
name: "seo_report",
|
|
46551
47396
|
command: "report",
|
|
46552
47397
|
title: "Search Console report",
|
|
46553
|
-
description: `Start here. Without site: a short summary of every site in the person's weekly site audit (beta) for last week, with each site's Google index, weekly crawl and search clicks, sites with changes first, ending with the site to ask about next; an account outside that beta gets its site list to pick from. With site: the main report for that Search Console property, the overview plus each section's row count and top rows. With site and section: that section's rows; page sections also check their top 20 URLs live. Sections:\n${Object.entries(SECTIONS).map(([id, text]) => `- ${id}: ${text}`).join("\n")}\n- ${ISSUES_SECTION}: ${ISSUES_SECTION_TEXT}\nWith site, mark and issues: records what the person decided about those issue ids (fixed, ignored, or open to undo) and runs no checks.`,
|
|
47398
|
+
description: `Start here. Without site: a short summary of every site in the person's weekly site audit (beta) for last week, with each site's Google index, weekly crawl and search clicks, sites with changes first, ending with the site to ask about next; an account outside that beta gets its site list to pick from. With site: the main report for that Search Console property, the overview plus each section's row count and top rows. With site and section: that section's rows; page sections also check their top 20 URLs live. Sections:\n${Object.entries(SECTIONS).map(([id, text]) => `- ${id}: ${text}`).join("\n")}\n- ${ISSUES_SECTION}: ${ISSUES_SECTION_TEXT}\n- ${UPDATES_SECTION}: ${UPDATES_SECTION_TEXT}\nWith site, mark and issues: records what the person decided about those issue ids (fixed, ignored, or open to undo) and runs no checks.`,
|
|
46554
47399
|
input: z.object({
|
|
46555
47400
|
site: site.optional().describe("Leave it out for the summary of every audited site (weekly site audit beta). Otherwise a Search Console property or site, such as sc-domain:example.com, https://example.com/ or example.com."),
|
|
46556
|
-
section: z.enum([
|
|
47401
|
+
section: z.enum([
|
|
47402
|
+
...SECTION_IDS,
|
|
47403
|
+
ISSUES_SECTION,
|
|
47404
|
+
UPDATES_SECTION
|
|
47405
|
+
]).optional().describe("Optional, needs site. Leave it out first; then ask for one section."),
|
|
46557
47406
|
...dates,
|
|
46558
47407
|
...detail,
|
|
46559
47408
|
...marks
|
|
@@ -46670,6 +47519,7 @@ const SHARED_TOOLS = [
|
|
|
46670
47519
|
action: z.enum(RESEARCH_ACTIONS),
|
|
46671
47520
|
keywords: z.array(z.string().min(1)).max(100).default([]),
|
|
46672
47521
|
target: z.string().default("").describe("Domain or URL."),
|
|
47522
|
+
competitors: z.array(z.string().min(1)).max(5).default([]).describe("link_gap only: 2 to 5 competitor domains."),
|
|
46673
47523
|
tasks: z.array(z.string().regex(AI_ANSWER_TASK_PATTERN)).max(30).default([]).describe("ai_answers only: the tasks an earlier run returned as not ready, to read back at no cost."),
|
|
46674
47524
|
country: z.string().regex(/^[a-z]{2}$/).default("us").describe("Two-letter country code."),
|
|
46675
47525
|
...detail
|
|
@@ -46685,7 +47535,7 @@ const SHARED_TOOLS = [
|
|
|
46685
47535
|
offset: input.offset,
|
|
46686
47536
|
questions: input.keywords,
|
|
46687
47537
|
tasks: input.tasks
|
|
46688
|
-
}, await surface.keys(), surface.deps,
|
|
47538
|
+
}, await surface.keys(), surface.deps, (offset, tasks) => `seo_research with the same provider, action and target, tasks ${JSON.stringify(tasks)}, detail "rows", offset ${offset}. Passing the tasks reads these answers back at no cost; without them every question is asked and paid for again`);
|
|
46689
47539
|
}
|
|
46690
47540
|
return runResearch(input, await surface.keys(), {
|
|
46691
47541
|
drKey: surface.drKey,
|
|
@@ -46816,6 +47666,7 @@ SHARED_TOOLS.map((shared) => shared.name === "seo_index" ? indexFromStore(shared
|
|
|
46816
47666
|
input: shared.input.extend({ section: z.enum([
|
|
46817
47667
|
...SECTION_IDS,
|
|
46818
47668
|
ISSUES_SECTION,
|
|
47669
|
+
UPDATES_SECTION,
|
|
46819
47670
|
AUDIT_SECTION
|
|
46820
47671
|
]).optional().describe("Optional. Leave it out first; then ask for one section.") })
|
|
46821
47672
|
} : shared);
|