seo 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +23 -5
- package/README.md +1 -1
- package/dist/cli.js +82 -82
- package/dist/cli.js.map +1 -1
- package/dist/{dist-B3L_4U8E.js → dist-BL_yf1fw.js} +1 -1
- package/dist/dist-DqfdgVVU.js +361 -0
- package/dist/dist-DqfdgVVU.js.map +1 -0
- package/dist/{esm-gP6ovS6b.js → esm-ZQwQYL0U.js} +2 -2
- package/dist/{esm-gP6ovS6b.js.map → esm-ZQwQYL0U.js.map} +1 -1
- package/dist/index.d.ts +368 -210
- package/dist/index.js +1953 -962
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +2083 -994
- package/dist/mcp.js.map +1 -1
- package/package.json +1 -1
- package/skills/seo/SKILL.md +1 -1
- package/dist/dist-BdZyUFiw.js +0 -356
- package/dist/dist-BdZyUFiw.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { n as version, t as load } from "./esm-
|
|
2
|
+
import { n as version, t as load } from "./esm-ZQwQYL0U.js";
|
|
3
3
|
import { SEO_PROVIDER_API_VERSION } from "./provider-sdk.js";
|
|
4
4
|
import crypto, { createHash, randomUUID, sign } from "node:crypto";
|
|
5
5
|
import { z } from "zod";
|
|
6
|
-
import { arch, homedir, hostname, platform, totalmem } from "node:os";
|
|
6
|
+
import { arch, homedir, hostname, platform, tmpdir, totalmem } from "node:os";
|
|
7
7
|
import { basename, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
8
8
|
import open from "open";
|
|
9
9
|
import { chmodSync, createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
@@ -22,11 +22,12 @@ import { chromium } from "playwright-core";
|
|
|
22
22
|
import { gunzipSync } from "node:zlib";
|
|
23
23
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
24
24
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
25
|
-
import { mkdir, open as open$1, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
25
|
+
import { mkdir, mkdtemp, open as open$1, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
26
26
|
import { TextDecoder as TextDecoder$1, promisify } from "node:util";
|
|
27
27
|
import { execFile } from "node:child_process";
|
|
28
28
|
import pino from "pino";
|
|
29
29
|
import pretty from "pino-pretty";
|
|
30
|
+
import { request } from "node:https";
|
|
30
31
|
//#region packages/core/src/analytics/providers.ts
|
|
31
32
|
const ANALYTICS_PROVIDER_IDS = ["google", "clicky"];
|
|
32
33
|
const ANALYTICS_PROVIDER_DETAILS = {
|
|
@@ -1458,12 +1459,12 @@ var SeoError = class extends Error {
|
|
|
1458
1459
|
this.retryable = errorMetadata[code].retryable;
|
|
1459
1460
|
}
|
|
1460
1461
|
};
|
|
1461
|
-
function errorMessage$
|
|
1462
|
+
function errorMessage$1(error) {
|
|
1462
1463
|
return error instanceof Error ? error.message : String(error);
|
|
1463
1464
|
}
|
|
1464
1465
|
function toSeoError(error) {
|
|
1465
1466
|
if (error instanceof SeoError) return error;
|
|
1466
|
-
const message = errorMessage$
|
|
1467
|
+
const message = errorMessage$1(error);
|
|
1467
1468
|
if (/not logged in/i.test(message)) return new SeoError("AUTH_REQUIRED", message);
|
|
1468
1469
|
if (/oauth client config.*missing|shared seo google app/i.test(message)) return new SeoError("AUTH_CONFIG_REQUIRED", message);
|
|
1469
1470
|
if (/invalid_grant|refresh token.*no longer valid/i.test(message)) return new SeoError("AUTH_EXPIRED", message);
|
|
@@ -1701,7 +1702,7 @@ async function runAnalyticsReport(gfetch, propertyId, body, options) {
|
|
|
1701
1702
|
//#endregion
|
|
1702
1703
|
//#region packages/core/src/bing/api.ts
|
|
1703
1704
|
const BASE = "https://ssl.bing.com/webmaster/api.svc/json";
|
|
1704
|
-
const MAX_BYTES$
|
|
1705
|
+
const MAX_BYTES$3 = 8e6;
|
|
1705
1706
|
function bingCall(auth, options) {
|
|
1706
1707
|
return async (method, params = {}) => {
|
|
1707
1708
|
const url = new URL(`${BASE}/${method}`);
|
|
@@ -1726,7 +1727,7 @@ function bingCall(auth, options) {
|
|
|
1726
1727
|
if (response.status === 429 || /throttle/i.test(detail)) throw new SeoError("RATE_LIMITED", "Bing Webmaster is limiting requests. Try again in a few minutes.");
|
|
1727
1728
|
throw new SeoError("OPTIONAL_PROVIDER_UNAVAILABLE", `Bing Webmaster answered ${response.status}. ${detail}`.trim());
|
|
1728
1729
|
}
|
|
1729
|
-
return (await readJson(response, "Bing Webmaster", { maxBytes: MAX_BYTES$
|
|
1730
|
+
return (await readJson(response, "Bing Webmaster", { maxBytes: MAX_BYTES$3 })).d;
|
|
1730
1731
|
};
|
|
1731
1732
|
}
|
|
1732
1733
|
//#endregion
|
|
@@ -1844,6 +1845,262 @@ function decayComparisonRange(current, comparison = "previous-period") {
|
|
|
1844
1845
|
};
|
|
1845
1846
|
}
|
|
1846
1847
|
//#endregion
|
|
1848
|
+
//#region packages/core/src/updates/ranking-history.ts
|
|
1849
|
+
const RANKING_HISTORY = [
|
|
1850
|
+
{
|
|
1851
|
+
id: "XhUDXP7A67iHCD2kmbVu",
|
|
1852
|
+
begin: "2026-09-24T16:15:00+00:00",
|
|
1853
|
+
end: "2026-10-08T08:00:00+00:00",
|
|
1854
|
+
external_desc: "September 2026 spam update"
|
|
1855
|
+
},
|
|
1856
|
+
{
|
|
1857
|
+
id: "LEubPCm2octf2uMqCFKE",
|
|
1858
|
+
begin: "2026-08-18T16:27:00+00:00",
|
|
1859
|
+
end: "2026-08-21T08:49:00+00:00",
|
|
1860
|
+
external_desc: "August 2026 spam update"
|
|
1861
|
+
},
|
|
1862
|
+
{
|
|
1863
|
+
id: "YUX1peHev5a4fkxLDiUQ",
|
|
1864
|
+
begin: "2026-06-24T16:00:00+00:00",
|
|
1865
|
+
end: "2026-06-26T17:00:00+00:00",
|
|
1866
|
+
external_desc: "June 2026 spam update"
|
|
1867
|
+
},
|
|
1868
|
+
{
|
|
1869
|
+
id: "wdAXJk6LRRihEjpzEeWE",
|
|
1870
|
+
begin: "2026-05-21T15:40:00+00:00",
|
|
1871
|
+
end: "2026-06-02T12:40:00+00:00",
|
|
1872
|
+
external_desc: "May 2026 core update"
|
|
1873
|
+
},
|
|
1874
|
+
{
|
|
1875
|
+
id: "7eTbAa2jWdToLkraZj5y",
|
|
1876
|
+
begin: "2026-03-27T09:00:00+00:00",
|
|
1877
|
+
end: "2026-04-08T13:00:00+00:00",
|
|
1878
|
+
external_desc: "March 2026 core update"
|
|
1879
|
+
},
|
|
1880
|
+
{
|
|
1881
|
+
id: "VbnSXAH4SmEcxPtx4YSD",
|
|
1882
|
+
begin: "2026-03-24T19:00:00+00:00",
|
|
1883
|
+
end: "2026-03-25T14:30:00+00:00",
|
|
1884
|
+
external_desc: "March 2026 spam update"
|
|
1885
|
+
},
|
|
1886
|
+
{
|
|
1887
|
+
id: "mYbNTqV1ytDc2fA8hUz4",
|
|
1888
|
+
begin: "2026-02-05T17:00:00+00:00",
|
|
1889
|
+
end: "2026-02-27T10:00:00+00:00",
|
|
1890
|
+
external_desc: "February 2026 Discover update"
|
|
1891
|
+
},
|
|
1892
|
+
{
|
|
1893
|
+
id: "DsirqJ1gpPRgVQeccPRv",
|
|
1894
|
+
begin: "2025-12-11T17:25:00+00:00",
|
|
1895
|
+
end: "2025-12-29T19:00:00+00:00",
|
|
1896
|
+
external_desc: "December 2025 core update"
|
|
1897
|
+
},
|
|
1898
|
+
{
|
|
1899
|
+
id: "a7Aainy6E9rZsmfq82xt",
|
|
1900
|
+
begin: "2025-08-26T16:00:00+00:00",
|
|
1901
|
+
end: "2025-09-22T07:00:00+00:00",
|
|
1902
|
+
external_desc: "August 2025 spam update"
|
|
1903
|
+
},
|
|
1904
|
+
{
|
|
1905
|
+
id: "riq1AuqETW46NfBCe5NT",
|
|
1906
|
+
begin: "2025-06-30T14:30:00+00:00",
|
|
1907
|
+
end: "2025-07-17T08:00:00+00:00",
|
|
1908
|
+
external_desc: "June 2025 core update"
|
|
1909
|
+
},
|
|
1910
|
+
{
|
|
1911
|
+
id: "zpmwuSwifjDjfrVdaZUx",
|
|
1912
|
+
begin: "2025-03-13T16:00:00+00:00",
|
|
1913
|
+
end: "2025-03-27T12:35:00+00:00",
|
|
1914
|
+
external_desc: "March 2025 core update"
|
|
1915
|
+
},
|
|
1916
|
+
{
|
|
1917
|
+
id: "UUq2WSouY7PhSm8zvtD1",
|
|
1918
|
+
begin: "2024-12-19T17:00:00+00:00",
|
|
1919
|
+
end: "2024-12-26T19:00:00+00:00",
|
|
1920
|
+
external_desc: "December 2024 spam update"
|
|
1921
|
+
},
|
|
1922
|
+
{
|
|
1923
|
+
id: "V9nDKuo6nWKh2ThBALgA",
|
|
1924
|
+
begin: "2024-12-12T15:30:00+00:00",
|
|
1925
|
+
end: "2024-12-18T19:00:00+00:00",
|
|
1926
|
+
external_desc: "December 2024 core update"
|
|
1927
|
+
},
|
|
1928
|
+
{
|
|
1929
|
+
id: "G7rp11wqTaTGn6JjiPF3",
|
|
1930
|
+
begin: "2024-11-11T19:00:00+00:00",
|
|
1931
|
+
end: "2024-12-05T08:00:00+00:00",
|
|
1932
|
+
external_desc: "November 2024 core update"
|
|
1933
|
+
},
|
|
1934
|
+
{
|
|
1935
|
+
id: "42XCLGtpuCZkzqe4dc6S",
|
|
1936
|
+
begin: "2024-08-16T00:15:00+00:00",
|
|
1937
|
+
end: "2024-08-20T11:30:00+00:00",
|
|
1938
|
+
external_desc: "Ranking is experiencing an ongoing issue."
|
|
1939
|
+
},
|
|
1940
|
+
{
|
|
1941
|
+
id: "gVx6b2o78zke7GrMidGy",
|
|
1942
|
+
begin: "2024-08-15T14:58:00+00:00",
|
|
1943
|
+
end: "2024-09-03T19:00:00+00:00",
|
|
1944
|
+
external_desc: "August 2024 core update"
|
|
1945
|
+
},
|
|
1946
|
+
{
|
|
1947
|
+
id: "QdUeCQx3LRVbzQ3E7FiD",
|
|
1948
|
+
begin: "2024-06-20T15:12:00+00:00",
|
|
1949
|
+
end: "2024-06-27T16:11:00+00:00",
|
|
1950
|
+
external_desc: "June 2024 spam update"
|
|
1951
|
+
},
|
|
1952
|
+
{
|
|
1953
|
+
id: "iXz2PJfodvyjaVUeqxZE",
|
|
1954
|
+
begin: "2024-03-05T17:03:00+00:00",
|
|
1955
|
+
end: "2024-03-20T14:09:00+00:00",
|
|
1956
|
+
external_desc: "March 2024 spam update"
|
|
1957
|
+
},
|
|
1958
|
+
{
|
|
1959
|
+
id: "1jW2F89qC2NxJBWGiKxE",
|
|
1960
|
+
begin: "2024-03-05T17:02:00+00:00",
|
|
1961
|
+
end: "2024-04-19T17:00:00+00:00",
|
|
1962
|
+
external_desc: "March 2024 core update"
|
|
1963
|
+
},
|
|
1964
|
+
{
|
|
1965
|
+
id: "VqRTcmyQCwfpuYeaVNfn",
|
|
1966
|
+
begin: "2023-11-08T16:58:00+00:00",
|
|
1967
|
+
end: "2023-12-07T17:00:00+00:00",
|
|
1968
|
+
external_desc: "November 2023 reviews update"
|
|
1969
|
+
},
|
|
1970
|
+
{
|
|
1971
|
+
id: "WtY1CQgfqrr2cjxKbyUk",
|
|
1972
|
+
begin: "2023-11-02T19:09:00+00:00",
|
|
1973
|
+
end: "2023-11-28T16:30:00+00:00",
|
|
1974
|
+
external_desc: "November 2023 core update"
|
|
1975
|
+
},
|
|
1976
|
+
{
|
|
1977
|
+
id: "VKyoS53ULWzSp8BoDxqk",
|
|
1978
|
+
begin: "2023-10-05T16:00:00+00:00",
|
|
1979
|
+
end: "2023-10-19T15:00:00+00:00",
|
|
1980
|
+
external_desc: "October 2023 core update"
|
|
1981
|
+
},
|
|
1982
|
+
{
|
|
1983
|
+
id: "fCUAy6TvbDMkkLAGcwkj",
|
|
1984
|
+
begin: "2023-10-05T07:00:00+00:00",
|
|
1985
|
+
end: "2023-10-31T07:00:00+00:00",
|
|
1986
|
+
external_desc: "Ranking is experiencing an ongoing issue"
|
|
1987
|
+
},
|
|
1988
|
+
{
|
|
1989
|
+
id: "NzcEhGMDhbQEdXCS35xL",
|
|
1990
|
+
begin: "2023-10-04T16:51:00+00:00",
|
|
1991
|
+
end: "2023-10-20T05:15:00+00:00",
|
|
1992
|
+
external_desc: "October 2023 spam update"
|
|
1993
|
+
},
|
|
1994
|
+
{
|
|
1995
|
+
id: "53diuQvcEsgzqXTPBb8p",
|
|
1996
|
+
begin: "2023-09-14T20:19:00+00:00",
|
|
1997
|
+
end: "2023-09-28T07:07:00+00:00",
|
|
1998
|
+
external_desc: "September 2023 helpful content update"
|
|
1999
|
+
},
|
|
2000
|
+
{
|
|
2001
|
+
id: "nBtYtBeex4GYBbdDS2LX",
|
|
2002
|
+
begin: "2023-08-22T17:30:00+00:00",
|
|
2003
|
+
end: "2023-09-07T20:03:00+00:00",
|
|
2004
|
+
external_desc: "August 2023 core update"
|
|
2005
|
+
},
|
|
2006
|
+
{
|
|
2007
|
+
id: "5XRfC46rorevFt8yN8iR",
|
|
2008
|
+
begin: "2023-04-12T16:00:00+00:00",
|
|
2009
|
+
end: "2023-04-25T18:24:00+00:00",
|
|
2010
|
+
external_desc: "April 2023 reviews update"
|
|
2011
|
+
},
|
|
2012
|
+
{
|
|
2013
|
+
id: "Cou8Tr74r7EXNthuEsaG",
|
|
2014
|
+
begin: "2023-03-15T07:00:00+00:00",
|
|
2015
|
+
end: "2023-03-28T14:26:00+00:00",
|
|
2016
|
+
external_desc: "March 2023 core update"
|
|
2017
|
+
},
|
|
2018
|
+
{
|
|
2019
|
+
id: "NDjZ11MEUJb5i62hy5aN",
|
|
2020
|
+
begin: "2023-02-21T08:00:00+00:00",
|
|
2021
|
+
end: "2023-03-07T08:00:00+00:00",
|
|
2022
|
+
external_desc: "February 2023 product reviews update"
|
|
2023
|
+
},
|
|
2024
|
+
{
|
|
2025
|
+
id: "CxXUEK28vCKqkNocPhKR",
|
|
2026
|
+
begin: "2022-12-14T08:00:00+00:00",
|
|
2027
|
+
end: "2023-01-12T08:00:00+00:00",
|
|
2028
|
+
external_desc: "December 2022 link spam update"
|
|
2029
|
+
},
|
|
2030
|
+
{
|
|
2031
|
+
id: "XMdX5PmbMvKW4fv3CRvn",
|
|
2032
|
+
begin: "2022-12-05T08:00:00+00:00",
|
|
2033
|
+
end: "2023-01-12T08:00:00+00:00",
|
|
2034
|
+
external_desc: "December 2022 helpful content update"
|
|
2035
|
+
},
|
|
2036
|
+
{
|
|
2037
|
+
id: "SAeYuxVSo39HHotQNXHr",
|
|
2038
|
+
begin: "2022-10-19T07:00:00+00:00",
|
|
2039
|
+
end: "2022-10-21T07:00:00+00:00",
|
|
2040
|
+
external_desc: "October 2022 spam update"
|
|
2041
|
+
},
|
|
2042
|
+
{
|
|
2043
|
+
id: "ZYwuBXDhkwaoop2V5epM",
|
|
2044
|
+
begin: "2022-09-20T07:00:00+00:00",
|
|
2045
|
+
end: "2022-09-26T07:00:00+00:00",
|
|
2046
|
+
external_desc: "September 2022 product reviews update"
|
|
2047
|
+
},
|
|
2048
|
+
{
|
|
2049
|
+
id: "Y2Cg5k2xLJvfNqR58kEt",
|
|
2050
|
+
begin: "2022-09-12T07:00:00+00:00",
|
|
2051
|
+
end: "2022-09-26T07:00:00+00:00",
|
|
2052
|
+
external_desc: "September 2022 core update"
|
|
2053
|
+
},
|
|
2054
|
+
{
|
|
2055
|
+
id: "sbKc4YSptrSxcUZFDKwU",
|
|
2056
|
+
begin: "2022-08-25T07:00:00+00:00",
|
|
2057
|
+
end: "2022-09-09T07:00:00+00:00",
|
|
2058
|
+
external_desc: "August 2022 helpful content update"
|
|
2059
|
+
},
|
|
2060
|
+
{
|
|
2061
|
+
id: "Xiid524od7fpCak86sNS",
|
|
2062
|
+
begin: "2022-07-27T07:00:00+00:00",
|
|
2063
|
+
end: "2022-08-02T07:00:00+00:00",
|
|
2064
|
+
external_desc: "July 2022 product reviews update"
|
|
2065
|
+
},
|
|
2066
|
+
{
|
|
2067
|
+
id: "GHwLLtDS64a7nDng8xhD",
|
|
2068
|
+
begin: "2022-05-25T07:00:00+00:00",
|
|
2069
|
+
end: "2022-06-09T07:00:00+00:00",
|
|
2070
|
+
external_desc: "May 2022 core update"
|
|
2071
|
+
},
|
|
2072
|
+
{
|
|
2073
|
+
id: "iYWoNu4QXRsQUrzxrbLv",
|
|
2074
|
+
begin: "2022-03-23T07:00:00+00:00",
|
|
2075
|
+
end: "2022-04-06T07:00:00+00:00",
|
|
2076
|
+
external_desc: "March 2022 product reviews update"
|
|
2077
|
+
},
|
|
2078
|
+
{
|
|
2079
|
+
id: "zDCQR5mxLU3ZrQeQvwqN",
|
|
2080
|
+
begin: "2022-02-22T08:00:00+00:00",
|
|
2081
|
+
end: "2022-03-03T08:00:00+00:00",
|
|
2082
|
+
external_desc: "Page experience update for desktop"
|
|
2083
|
+
},
|
|
2084
|
+
{
|
|
2085
|
+
id: "hgwP7ei5H1uKbYSeZrBZ",
|
|
2086
|
+
begin: "2021-12-01T08:00:00+00:00",
|
|
2087
|
+
end: "2021-12-21T08:00:00+00:00",
|
|
2088
|
+
external_desc: "December 2021 product reviews update"
|
|
2089
|
+
},
|
|
2090
|
+
{
|
|
2091
|
+
id: "FncPfR7zRSZ3diemah4o",
|
|
2092
|
+
begin: "2021-11-17T08:00:00+00:00",
|
|
2093
|
+
end: "2021-11-30T08:00:00+00:00",
|
|
2094
|
+
external_desc: "November 2021 core update"
|
|
2095
|
+
},
|
|
2096
|
+
{
|
|
2097
|
+
id: "AFdXceQVsh1GEfKFvHyD",
|
|
2098
|
+
begin: "2021-11-03T07:00:00+00:00",
|
|
2099
|
+
end: "2021-11-11T08:00:00+00:00",
|
|
2100
|
+
external_desc: "November 2021 spam update"
|
|
2101
|
+
}
|
|
2102
|
+
];
|
|
2103
|
+
//#endregion
|
|
1847
2104
|
//#region packages/core/src/updates/search-updates.ts
|
|
1848
2105
|
function classifyUpdate(name, product) {
|
|
1849
2106
|
const lower = `${name} ${product}`.toLowerCase();
|
|
@@ -1871,6 +2128,18 @@ function searchUpdatesFromIncidents(incidents, input = {}) {
|
|
|
1871
2128
|
};
|
|
1872
2129
|
}).slice(0, input.limit ?? 25);
|
|
1873
2130
|
}
|
|
2131
|
+
function withRankingHistory(live) {
|
|
2132
|
+
const byId = /* @__PURE__ */ new Map();
|
|
2133
|
+
for (const saved of RANKING_HISTORY) byId.set(saved.id, {
|
|
2134
|
+
...saved,
|
|
2135
|
+
service_name: "Ranking",
|
|
2136
|
+
status_impact: "SERVICE_INFORMATION",
|
|
2137
|
+
severity: "low",
|
|
2138
|
+
uri: `incidents/${saved.id}`
|
|
2139
|
+
});
|
|
2140
|
+
for (const incident of live) byId.set(incident.id, incident);
|
|
2141
|
+
return [...byId.values()].sort((left, right) => Date.parse(right.begin) - Date.parse(left.begin) || (left.id < right.id ? -1 : 1));
|
|
2142
|
+
}
|
|
1874
2143
|
function findOverlappingSearchUpdates(input) {
|
|
1875
2144
|
const paddingMs = (input.paddingDays ?? 3) * 864e5;
|
|
1876
2145
|
const windowStart = new Date(input.startDate).getTime() - paddingMs;
|
|
@@ -4346,10 +4615,12 @@ function initDb(database, isNewDatabase) {
|
|
|
4346
4615
|
database.exec("DROP TABLE IF EXISTS provider_spend_notices");
|
|
4347
4616
|
ensureColumn(database, "http_cache", "metadata_json", "TEXT");
|
|
4348
4617
|
}).immediate();
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
4618
|
+
try {
|
|
4619
|
+
runCacheMaintenance(database, {
|
|
4620
|
+
compact: true,
|
|
4621
|
+
allowFullVacuum: true
|
|
4622
|
+
});
|
|
4623
|
+
} catch {}
|
|
4353
4624
|
}
|
|
4354
4625
|
function getDb() {
|
|
4355
4626
|
if (db) return db;
|
|
@@ -4360,12 +4631,41 @@ function getDb() {
|
|
|
4360
4631
|
mode: 448
|
|
4361
4632
|
});
|
|
4362
4633
|
ensurePrivateDatabaseFiles(path);
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4366
|
-
|
|
4634
|
+
const opened = new SqliteDatabase(path);
|
|
4635
|
+
try {
|
|
4636
|
+
ensurePrivateDatabaseFiles(path);
|
|
4637
|
+
initDb(opened, isNewDatabase);
|
|
4638
|
+
ensurePrivateDatabaseFiles(path);
|
|
4639
|
+
} catch (error) {
|
|
4640
|
+
opened.close();
|
|
4641
|
+
throw error;
|
|
4642
|
+
}
|
|
4643
|
+
db = opened;
|
|
4367
4644
|
return db;
|
|
4368
4645
|
}
|
|
4646
|
+
let memoryDb;
|
|
4647
|
+
function memoryCache() {
|
|
4648
|
+
if (!memoryDb) {
|
|
4649
|
+
memoryDb = new SqliteDatabase(":memory:");
|
|
4650
|
+
memoryDb.exec(CREATE_SQL);
|
|
4651
|
+
}
|
|
4652
|
+
return memoryDb;
|
|
4653
|
+
}
|
|
4654
|
+
function cacheDb() {
|
|
4655
|
+
if (memoryDb) return memoryDb;
|
|
4656
|
+
try {
|
|
4657
|
+
return getDb();
|
|
4658
|
+
} catch {
|
|
4659
|
+
return memoryCache();
|
|
4660
|
+
}
|
|
4661
|
+
}
|
|
4662
|
+
function writeToCache(write) {
|
|
4663
|
+
try {
|
|
4664
|
+
write(cacheDb());
|
|
4665
|
+
} catch {
|
|
4666
|
+
memoryCache();
|
|
4667
|
+
}
|
|
4668
|
+
}
|
|
4369
4669
|
function noteCacheWrite(sizeBytes = 0, database = getDb()) {
|
|
4370
4670
|
const counter = cacheMaintenanceCounters.get(database) ?? {
|
|
4371
4671
|
writes: 0,
|
|
@@ -4447,7 +4747,7 @@ function ga4RequestCanUseCache(body) {
|
|
|
4447
4747
|
}
|
|
4448
4748
|
async function runGa4Report(propertyId, body, opts = {}) {
|
|
4449
4749
|
const { client, identity } = await createGoogleAccessTokenClient(opts.accountEmail);
|
|
4450
|
-
const db =
|
|
4750
|
+
const db = cacheDb();
|
|
4451
4751
|
const queryHash = hashKey([
|
|
4452
4752
|
propertyId,
|
|
4453
4753
|
identity,
|
|
@@ -4460,10 +4760,12 @@ async function runGa4Report(propertyId, body, opts = {}) {
|
|
|
4460
4760
|
if (cacheable) {
|
|
4461
4761
|
const requestJson = JSON.stringify(body);
|
|
4462
4762
|
const responseJson = JSON.stringify(result);
|
|
4463
|
-
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
4763
|
+
writeToCache((database) => {
|
|
4764
|
+
database.prepare(`INSERT OR REPLACE INTO ga4_cache
|
|
4765
|
+
(property_id, query_hash, request_json, response_json, row_count, fetched_at, expires_at)
|
|
4766
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`).run(propertyId, queryHash, requestJson, responseJson, result.rowCount ?? result.rows?.length ?? 0, Date.now(), Date.now() + 864e5);
|
|
4767
|
+
noteCacheWrite(Buffer.byteLength(requestJson) + Buffer.byteLength(responseJson), database);
|
|
4768
|
+
});
|
|
4467
4769
|
}
|
|
4468
4770
|
return result;
|
|
4469
4771
|
}
|
|
@@ -6922,7 +7224,7 @@ async function fetchRobotsSource(origin, refresh, options) {
|
|
|
6922
7224
|
const robotsUrl = new URL("/robots.txt", origin).toString();
|
|
6923
7225
|
const key = hashKey(["robots", robotsUrl]);
|
|
6924
7226
|
if (!refresh) {
|
|
6925
|
-
const cached =
|
|
7227
|
+
const cached = cacheDb().prepare("SELECT status, body_blob, expires_at FROM http_cache WHERE url_hash = ?").get(key);
|
|
6926
7228
|
if (cached?.body_blob && cached.status !== void 0 && cacheableStatus(cached.status) && cached.expires_at && cached.expires_at > Date.now()) return {
|
|
6927
7229
|
robotsUrl,
|
|
6928
7230
|
status: cached.status,
|
|
@@ -6941,12 +7243,12 @@ async function fetchRobotsSource(origin, refresh, options) {
|
|
|
6941
7243
|
signal: controller.signal
|
|
6942
7244
|
});
|
|
6943
7245
|
const text = await readBoundedResponseText(response, MAX_ROBOTS_RESPONSE_BYTES, "robots.txt response");
|
|
6944
|
-
if (options.writeCache !== false && cacheableStatus(response.status)) {
|
|
6945
|
-
|
|
6946
|
-
|
|
6947
|
-
|
|
6948
|
-
noteCacheWrite(Buffer.byteLength(text));
|
|
6949
|
-
}
|
|
7246
|
+
if (options.writeCache !== false && cacheableStatus(response.status)) writeToCache((database) => {
|
|
7247
|
+
database.prepare(`INSERT OR REPLACE INTO http_cache
|
|
7248
|
+
(url_hash, url, status, headers_json, body_blob, etag, fetched_at, expires_at)
|
|
7249
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(key, robotsUrl, response.status, "{}", Buffer.from(text), null, Date.now(), Date.now() + 864e5);
|
|
7250
|
+
noteCacheWrite(Buffer.byteLength(text), database);
|
|
7251
|
+
});
|
|
6950
7252
|
return {
|
|
6951
7253
|
robotsUrl,
|
|
6952
7254
|
status: response.status,
|
|
@@ -6984,6 +7286,12 @@ function createRobotsSession(input = {}) {
|
|
|
6984
7286
|
};
|
|
6985
7287
|
return {
|
|
6986
7288
|
resolve: async (origin, targetUrl) => resultFromSource(await sourceFor(origin), targetUrl),
|
|
7289
|
+
crawlDelay: async (origin) => {
|
|
7290
|
+
const source = await sourceFor(origin);
|
|
7291
|
+
if (source.status === void 0 || source.status >= 300 || !source.text) return;
|
|
7292
|
+
const delay = parseRobots$1(source.robotsUrl, source.text).getCrawlDelay(SEO_CRAWLER_USER_AGENT);
|
|
7293
|
+
return delay !== void 0 && Number.isFinite(delay) && delay > 0 ? delay : void 0;
|
|
7294
|
+
},
|
|
6987
7295
|
sitemapUrls: async (origin) => {
|
|
6988
7296
|
const source = await sourceFor(origin);
|
|
6989
7297
|
if (source.status === void 0 || source.status >= 300 || !source.text) return [];
|
|
@@ -7047,7 +7355,6 @@ function decodePageFetchCacheEvidence(value) {
|
|
|
7047
7355
|
}
|
|
7048
7356
|
async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, respectRobots = false, writeCache = true) {
|
|
7049
7357
|
const startedAt = Date.now();
|
|
7050
|
-
const db = getDb();
|
|
7051
7358
|
const key = hashKey(["page", url]);
|
|
7052
7359
|
const host = new URL(url).host;
|
|
7053
7360
|
let robots = respectRobots ? await fetchRobots(new URL(url).origin, url, refresh, {
|
|
@@ -7055,7 +7362,7 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
|
|
|
7055
7362
|
signal
|
|
7056
7363
|
}) : void 0;
|
|
7057
7364
|
if (robots && robots.allowed !== true) throw new RobotsAccessError(robots.allowed === false ? "robots-disallowed" : "robots-deferred", robots);
|
|
7058
|
-
const cached =
|
|
7365
|
+
const cached = cacheDb().prepare("SELECT status, headers_json, body_blob, metadata_json, expires_at FROM http_cache WHERE url_hash = ? AND expires_at > ?").get(key, Date.now());
|
|
7059
7366
|
if (!refresh && cached) {
|
|
7060
7367
|
const evidence = decodePageFetchCacheEvidence(cached.metadata_json);
|
|
7061
7368
|
return {
|
|
@@ -7157,12 +7464,12 @@ async function fetchPlain(url, refresh = false, timeoutMs = 2e4, rate, signal, r
|
|
|
7157
7464
|
warnings: [...robots.allowed === null ? [`robots.txt availability is unknown${robots.status ? ` after HTTP ${robots.status}` : ""}; this fetch does not prove crawler access is allowed.`] : [], ...accessBlock ? [`${accessBlock.guidance.summary} Identify the crawler as ${accessBlock.crawler.userAgent}.`] : []],
|
|
7158
7465
|
robotsTxt: pageRobotsEvidence(robots)
|
|
7159
7466
|
};
|
|
7160
|
-
if (!accessBlock && writeCache) {
|
|
7161
|
-
|
|
7162
|
-
|
|
7163
|
-
|
|
7164
|
-
noteCacheWrite(Buffer.byteLength(html));
|
|
7165
|
-
}
|
|
7467
|
+
if (!accessBlock && writeCache) writeToCache((database) => {
|
|
7468
|
+
database.prepare(`INSERT OR REPLACE INTO http_cache
|
|
7469
|
+
(url_hash, url, status, headers_json, body_blob, metadata_json, etag, fetched_at, expires_at)
|
|
7470
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(key, url, response.response.status, JSON.stringify(headerMap), Buffer.from(html), encodePageFetchCacheEvidence(result), response.response.headers.get("etag"), Date.now(), Date.now() + 36e5);
|
|
7471
|
+
noteCacheWrite(Buffer.byteLength(html), database);
|
|
7472
|
+
});
|
|
7166
7473
|
return result;
|
|
7167
7474
|
}
|
|
7168
7475
|
//#endregion
|
|
@@ -7944,7 +8251,7 @@ function finalizeUrlInspectionQuota(input) {
|
|
|
7944
8251
|
function credentialKey(input) {
|
|
7945
8252
|
return createHash("sha256").update(`${input.clientId}\n${input.accountEmail.toLowerCase()}`).digest("base64url");
|
|
7946
8253
|
}
|
|
7947
|
-
function retryAfter(response, now) {
|
|
8254
|
+
function retryAfter$1(response, now) {
|
|
7948
8255
|
const value = response.headers.get("retry-after");
|
|
7949
8256
|
if (!value) return void 0;
|
|
7950
8257
|
const seconds = Number(value);
|
|
@@ -7987,7 +8294,7 @@ async function inspectUrl(input) {
|
|
|
7987
8294
|
}
|
|
7988
8295
|
if (response.status === 429) {
|
|
7989
8296
|
const now = /* @__PURE__ */ new Date();
|
|
7990
|
-
const blockedUntil = retryAfter(response, now);
|
|
8297
|
+
const blockedUntil = retryAfter$1(response, now);
|
|
7991
8298
|
finalizeUrlInspectionQuota({
|
|
7992
8299
|
reservation,
|
|
7993
8300
|
outcome: "exhausted",
|
|
@@ -8020,7 +8327,7 @@ function resolveSearchDateWindow(input = 28) {
|
|
|
8020
8327
|
}
|
|
8021
8328
|
async function querySearchAnalytics(site, body, opts = {}) {
|
|
8022
8329
|
const { client, identity } = await getAuthorized(opts.accountEmail ?? await cloudAccountFor(site));
|
|
8023
|
-
const db =
|
|
8330
|
+
const db = cacheDb();
|
|
8024
8331
|
const queryHash = hashKey([
|
|
8025
8332
|
site,
|
|
8026
8333
|
identity,
|
|
@@ -8040,10 +8347,12 @@ async function querySearchAnalytics(site, body, opts = {}) {
|
|
|
8040
8347
|
const calls = result.calls;
|
|
8041
8348
|
const requestJson = JSON.stringify(body);
|
|
8042
8349
|
const responseJson = JSON.stringify(allRows);
|
|
8043
|
-
|
|
8044
|
-
|
|
8045
|
-
|
|
8046
|
-
|
|
8350
|
+
writeToCache((database) => {
|
|
8351
|
+
database.prepare(`INSERT OR REPLACE INTO gsc_cache
|
|
8352
|
+
(site_url, query_hash, request_json, response_json, row_count, fetched_at, expires_at)
|
|
8353
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`).run(site, queryHash, requestJson, responseJson, allRows.length, Date.now(), Date.now() + 864e5);
|
|
8354
|
+
noteCacheWrite(Buffer.byteLength(requestJson) + Buffer.byteLength(responseJson), database);
|
|
8355
|
+
});
|
|
8047
8356
|
return {
|
|
8048
8357
|
rows: allRows,
|
|
8049
8358
|
calls,
|
|
@@ -8179,8 +8488,10 @@ async function queryPagesTopQueriesBatch(site, pageUrls, range = 28) {
|
|
|
8179
8488
|
//#region packages/core/src/gsc/client/sites.ts
|
|
8180
8489
|
async function listSites(_refresh = false, accountEmail) {
|
|
8181
8490
|
const entries = !accountEmail && cloudLinked() ? await allCloudSites() : await listSearchConsoleSites(googleFetchFor((await getAuthorized(accountEmail)).client), LOCAL_SIGN_IN);
|
|
8182
|
-
|
|
8183
|
-
|
|
8491
|
+
writeToCache((database) => {
|
|
8492
|
+
const insert = database.prepare("INSERT OR REPLACE INTO sites (site_url, display_name, permission, added_at, is_default) VALUES (?, ?, ?, ?, COALESCE((SELECT is_default FROM sites WHERE site_url = ?), 0))");
|
|
8493
|
+
for (const site of entries) insert.run(site.siteUrl, site.siteUrl, site.permissionLevel ?? null, Date.now(), site.siteUrl);
|
|
8494
|
+
});
|
|
8184
8495
|
return entries;
|
|
8185
8496
|
}
|
|
8186
8497
|
async function allCloudSites() {
|
|
@@ -8372,7 +8683,7 @@ const AUDIT_PAGE_RULE_IDS = [
|
|
|
8372
8683
|
"canonical_streamed_in_body",
|
|
8373
8684
|
"canonical_mismatch"
|
|
8374
8685
|
];
|
|
8375
|
-
const defaultDependencies$
|
|
8686
|
+
const defaultDependencies$12 = {
|
|
8376
8687
|
fetchPage,
|
|
8377
8688
|
queryPageMetrics,
|
|
8378
8689
|
now: () => /* @__PURE__ */ new Date()
|
|
@@ -8403,7 +8714,7 @@ function buildRecommendations(report) {
|
|
|
8403
8714
|
});
|
|
8404
8715
|
return recommendations;
|
|
8405
8716
|
}
|
|
8406
|
-
async function auditPage(input, dependencies = defaultDependencies$
|
|
8717
|
+
async function auditPage(input, dependencies = defaultDependencies$12) {
|
|
8407
8718
|
const fetched = await dependencies.fetchPage(input.url, {
|
|
8408
8719
|
js: input.js,
|
|
8409
8720
|
refresh: input.refresh
|
|
@@ -8522,7 +8833,7 @@ async function auditPage(input, dependencies = defaultDependencies$13) {
|
|
|
8522
8833
|
}
|
|
8523
8834
|
//#endregion
|
|
8524
8835
|
//#region packages/core/src/analyze/content-coverage-analysis.ts
|
|
8525
|
-
const STOPWORDS
|
|
8836
|
+
const STOPWORDS = /* @__PURE__ */ new Set([
|
|
8526
8837
|
"a",
|
|
8527
8838
|
"an",
|
|
8528
8839
|
"and",
|
|
@@ -8551,7 +8862,7 @@ function hasNonAscii(value) {
|
|
|
8551
8862
|
}
|
|
8552
8863
|
function queryTerms$1(query) {
|
|
8553
8864
|
const tokens = normalizeForCoverage(query).split(" ").filter((token) => token.length > 1 || hasNonAscii(token));
|
|
8554
|
-
const meaningful = tokens.filter((token) => (token.length > 2 || hasNonAscii(token)) && !STOPWORDS
|
|
8865
|
+
const meaningful = tokens.filter((token) => (token.length > 2 || hasNonAscii(token)) && !STOPWORDS.has(token));
|
|
8555
8866
|
return [...new Set(meaningful.length ? meaningful : tokens)];
|
|
8556
8867
|
}
|
|
8557
8868
|
function countPhrase$1(query, text) {
|
|
@@ -9858,7 +10169,7 @@ const actionPriority = new Map([
|
|
|
9858
10169
|
"unverified",
|
|
9859
10170
|
"covered"
|
|
9860
10171
|
].map((type, index) => [type, index]));
|
|
9861
|
-
function plural$
|
|
10172
|
+
function plural$6(count, singular, pluralLabel) {
|
|
9862
10173
|
return count === 1 ? singular : pluralLabel;
|
|
9863
10174
|
}
|
|
9864
10175
|
function actionable(item) {
|
|
@@ -9892,7 +10203,7 @@ function pageOpportunityVerdict(input) {
|
|
|
9892
10203
|
if (focus === "unverified") return "GSC visibility exists, but the page was not verified and no specific change is justified yet.";
|
|
9893
10204
|
if (focus === "covered") return "The verified page covers the sampled queries and CTR is not materially below its directional benchmark.";
|
|
9894
10205
|
const count = analysis.summary.opportunities;
|
|
9895
|
-
const prefix = `${count} ${plural$
|
|
10206
|
+
const prefix = `${count} ${plural$6(count, "actionable observation", "actionable observations")} found.`;
|
|
9896
10207
|
if (focus === "content-gap") return `${prefix} Verified body coverage is the main issue; confirm intent before expanding the page.`;
|
|
9897
10208
|
if (focus === "serp-framing") return `${prefix} The body is relevant, but its title, meta description, or H1 framing is weak.`;
|
|
9898
10209
|
if (focus === "ranking") return `${prefix} Most upside requires stronger relevance or authority; no CTR-only lift is claimed for page-two queries.`;
|
|
@@ -9907,7 +10218,7 @@ function pageOpportunityRecommendations(analysis) {
|
|
|
9907
10218
|
}
|
|
9908
10219
|
const recommendations = [`Start with "${top.query}". ${top.recommendation}`];
|
|
9909
10220
|
const technical = actions.filter((item) => item.opportunityType === "technical-check").length;
|
|
9910
|
-
if (technical) recommendations.push(`Resolve the technical evidence affecting ${technical} ${plural$
|
|
10221
|
+
if (technical) recommendations.push(`Resolve the technical evidence affecting ${technical} ${plural$6(technical, "row", "rows")} before content work.`);
|
|
9911
10222
|
const gaps = actions.filter((item) => item.opportunityType === "content-gap").length;
|
|
9912
10223
|
if (gaps >= 3) recommendations.push(`Group the ${gaps} verified content gaps into useful sections rather than adding one thin paragraph per query.`);
|
|
9913
10224
|
return recommendations;
|
|
@@ -9915,13 +10226,13 @@ function pageOpportunityRecommendations(analysis) {
|
|
|
9915
10226
|
//#endregion
|
|
9916
10227
|
//#region packages/core/src/analyze/page-opportunities.ts
|
|
9917
10228
|
const MAX_BENCHMARK_ROWS = 1e5;
|
|
9918
|
-
const defaultDependencies$
|
|
10229
|
+
const defaultDependencies$11 = {
|
|
9919
10230
|
searchAnalytics: querySearchAnalytics,
|
|
9920
10231
|
fetch: fetchPage,
|
|
9921
10232
|
extract: extractPage,
|
|
9922
10233
|
now: () => /* @__PURE__ */ new Date()
|
|
9923
10234
|
};
|
|
9924
|
-
function errorMessage
|
|
10235
|
+
function errorMessage(error) {
|
|
9925
10236
|
return error instanceof Error ? error.message : String(error);
|
|
9926
10237
|
}
|
|
9927
10238
|
function unique$3(values) {
|
|
@@ -9942,7 +10253,7 @@ async function verifyPage(input) {
|
|
|
9942
10253
|
});
|
|
9943
10254
|
warnings.push(...fetched.warnings);
|
|
9944
10255
|
} catch (error) {
|
|
9945
|
-
const reason = `Page fetch failed: ${errorMessage
|
|
10256
|
+
const reason = `Page fetch failed: ${errorMessage(error)}`;
|
|
9946
10257
|
return {
|
|
9947
10258
|
verification: {
|
|
9948
10259
|
status: "failed",
|
|
@@ -9964,7 +10275,7 @@ async function verifyPage(input) {
|
|
|
9964
10275
|
warnings: unique$3(warnings)
|
|
9965
10276
|
};
|
|
9966
10277
|
} catch (error) {
|
|
9967
|
-
const reason = `Page extraction failed: ${errorMessage
|
|
10278
|
+
const reason = `Page extraction failed: ${errorMessage(error)}`;
|
|
9968
10279
|
return {
|
|
9969
10280
|
fetched,
|
|
9970
10281
|
verification: {
|
|
@@ -9987,7 +10298,7 @@ function reportCaveats$1(input) {
|
|
|
9987
10298
|
input.benchmarkTruncated ? `The site-wide benchmark reached its ${MAX_BENCHMARK_ROWS.toLocaleString("en-US")}-row cap, so provenance is complete but the peer sample is truncated.` : ""
|
|
9988
10299
|
].filter(Boolean);
|
|
9989
10300
|
}
|
|
9990
|
-
async function pageOpportunitiesReport(input, dependencies = defaultDependencies$
|
|
10301
|
+
async function pageOpportunitiesReport(input, dependencies = defaultDependencies$11) {
|
|
9991
10302
|
const url = assertUrlMatchesGscProperty(input.site, input.url);
|
|
9992
10303
|
const days = reportDays$1(input.days);
|
|
9993
10304
|
const range = defaultDateRange$1(days);
|
|
@@ -15291,7 +15602,7 @@ function taskIds(value) {
|
|
|
15291
15602
|
}
|
|
15292
15603
|
}
|
|
15293
15604
|
function readProviderCache(key, schema, options = {}) {
|
|
15294
|
-
const database = options.database ??
|
|
15605
|
+
const database = options.database ?? cacheDb();
|
|
15295
15606
|
const now = options.now ?? Date.now();
|
|
15296
15607
|
const hash = requestHash(key, stableProviderRequestJson(key.request));
|
|
15297
15608
|
const row = database.prepare(`SELECT response_json, row_count, source_cost_micros, task_ids_json,
|
|
@@ -15312,9 +15623,11 @@ function readProviderCache(key, schema, options = {}) {
|
|
|
15312
15623
|
taskIds: taskIds(row.task_ids_json)
|
|
15313
15624
|
};
|
|
15314
15625
|
} catch {
|
|
15315
|
-
|
|
15316
|
-
|
|
15317
|
-
AND
|
|
15626
|
+
try {
|
|
15627
|
+
database.prepare(`DELETE FROM provider_cache
|
|
15628
|
+
WHERE provider = ? AND credential_scope = ? AND operation = ?
|
|
15629
|
+
AND request_hash = ?`).run(key.provider, key.credentialScope, key.operation, hash);
|
|
15630
|
+
} catch {}
|
|
15318
15631
|
return null;
|
|
15319
15632
|
}
|
|
15320
15633
|
}
|
|
@@ -15325,19 +15638,22 @@ function writeProviderCache(key, input, options = {}) {
|
|
|
15325
15638
|
code: "configuration",
|
|
15326
15639
|
message: `Provider cache TTL must be between 1 and ${CACHE_MAX_AGE_MS} milliseconds.`
|
|
15327
15640
|
});
|
|
15328
|
-
const database = options.database ?? getDb();
|
|
15329
15641
|
const now = options.now ?? Date.now();
|
|
15330
15642
|
const expiresAt = now + input.ttlMs;
|
|
15331
15643
|
const requestJson = stableProviderRequestJson(key.request);
|
|
15332
15644
|
const responseJson = JSON.stringify(input.data);
|
|
15333
15645
|
const normalizedTaskIds = [...new Set(input.taskIds)].filter((id) => id.length > 0 && id.length <= 100).sort((left, right) => left < right ? -1 : left > right ? 1 : 0).slice(0, 20);
|
|
15334
15646
|
const taskIdsJson = JSON.stringify(normalizedTaskIds);
|
|
15335
|
-
|
|
15336
|
-
|
|
15337
|
-
|
|
15338
|
-
|
|
15339
|
-
|
|
15340
|
-
|
|
15647
|
+
const save = (database) => {
|
|
15648
|
+
database.prepare(`INSERT OR REPLACE INTO provider_cache (
|
|
15649
|
+
provider, credential_scope, operation, request_hash, request_json,
|
|
15650
|
+
response_json, row_count, source_cost_micros, task_ids_json,
|
|
15651
|
+
fetched_at, expires_at
|
|
15652
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(key.provider, key.credentialScope, key.operation, requestHash(key, requestJson), requestJson, responseJson, input.rowCount, input.sourceCostMicros, taskIdsJson, now, expiresAt);
|
|
15653
|
+
noteCacheWrite(Buffer.byteLength(requestJson) + Buffer.byteLength(responseJson) + Buffer.byteLength(taskIdsJson), database);
|
|
15654
|
+
};
|
|
15655
|
+
if (options.database) save(options.database);
|
|
15656
|
+
else writeToCache(save);
|
|
15341
15657
|
return {
|
|
15342
15658
|
data: input.data,
|
|
15343
15659
|
storedAt: new Date(now).toISOString(),
|
|
@@ -16050,9 +16366,9 @@ async function loadInstalledProviderExtensions(options = {}) {
|
|
|
16050
16366
|
//#endregion
|
|
16051
16367
|
//#region packages/core/src/provider-extensions/runtime.ts
|
|
16052
16368
|
const DEFAULT_MAX_REQUESTS = 8;
|
|
16053
|
-
const MAX_RESPONSE_BYTES = 10 * 1024 * 1024;
|
|
16369
|
+
const MAX_RESPONSE_BYTES$1 = 10 * 1024 * 1024;
|
|
16054
16370
|
const MAX_REQUEST_BODY_BYTES = 64 * 1024;
|
|
16055
|
-
const TIMEOUT_MS$
|
|
16371
|
+
const TIMEOUT_MS$2 = 15e3;
|
|
16056
16372
|
function normalizeProviderExtensionAccount(provider, account) {
|
|
16057
16373
|
const fields = provider.connection.fields.filter((field) => field.kind === "account");
|
|
16058
16374
|
const allowed = new Set(fields.map((field) => field.id));
|
|
@@ -16098,8 +16414,8 @@ function runtimeForProvider(provider, options = {}) {
|
|
|
16098
16414
|
...input.body ? { body: input.body } : {}
|
|
16099
16415
|
},
|
|
16100
16416
|
fetch: providerFetch,
|
|
16101
|
-
maxResponseBytes: MAX_RESPONSE_BYTES,
|
|
16102
|
-
timeoutMs: TIMEOUT_MS$
|
|
16417
|
+
maxResponseBytes: MAX_RESPONSE_BYTES$1,
|
|
16418
|
+
timeoutMs: TIMEOUT_MS$2,
|
|
16103
16419
|
retry: (input.method ?? "GET") === "GET" ? "safe" : "never",
|
|
16104
16420
|
schema: z.unknown()
|
|
16105
16421
|
});
|
|
@@ -16203,7 +16519,7 @@ async function runAnalyticsProviderLandingPages(input) {
|
|
|
16203
16519
|
}
|
|
16204
16520
|
//#endregion
|
|
16205
16521
|
//#region packages/core/src/analyze/workflows/analytics-value.ts
|
|
16206
|
-
const defaultDependencies$
|
|
16522
|
+
const defaultDependencies$10 = {
|
|
16207
16523
|
runGa4Report,
|
|
16208
16524
|
analyticsLandingPages: runAnalyticsProviderLandingPages
|
|
16209
16525
|
};
|
|
@@ -16274,7 +16590,7 @@ function landingPageRankingPolicy(input) {
|
|
|
16274
16590
|
warnings
|
|
16275
16591
|
};
|
|
16276
16592
|
}
|
|
16277
|
-
async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
16593
|
+
async function fetchLandingPageValues(input, dependencies = defaultDependencies$10) {
|
|
16278
16594
|
const retainedRowLimit = input.limit ?? 5e3;
|
|
16279
16595
|
const connection = input.connection ?? (input.propertyId ? {
|
|
16280
16596
|
provider: "google",
|
|
@@ -16293,7 +16609,7 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
16293
16609
|
}
|
|
16294
16610
|
};
|
|
16295
16611
|
if (connection.provider === "extension") try {
|
|
16296
|
-
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$
|
|
16612
|
+
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$10.analyticsLandingPages)({
|
|
16297
16613
|
providerId: connection.providerId,
|
|
16298
16614
|
account: connection.account,
|
|
16299
16615
|
startDate: input.startDate,
|
|
@@ -16333,7 +16649,7 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
16333
16649
|
clicky: async (selected) => {
|
|
16334
16650
|
if (selected.provider !== "clicky") throw new Error("Clicky analytics connection mismatch.");
|
|
16335
16651
|
try {
|
|
16336
|
-
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$
|
|
16652
|
+
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$10.analyticsLandingPages)({
|
|
16337
16653
|
providerId: "clicky",
|
|
16338
16654
|
account: { siteId: selected.siteId },
|
|
16339
16655
|
startDate: input.startDate,
|
|
@@ -16372,7 +16688,7 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
16372
16688
|
google: async (selected) => {
|
|
16373
16689
|
if (selected.provider !== "google") throw new Error("Google Analytics connection mismatch.");
|
|
16374
16690
|
try {
|
|
16375
|
-
const result = await (dependencies.runGa4Report ?? defaultDependencies$
|
|
16691
|
+
const result = await (dependencies.runGa4Report ?? defaultDependencies$10.runGa4Report)(selected.propertyId, {
|
|
16376
16692
|
dateRanges: [{
|
|
16377
16693
|
startDate: input.startDate,
|
|
16378
16694
|
endDate: input.endDate
|
|
@@ -17153,7 +17469,7 @@ const NAVIGATIONAL_STEM_THRESHOLDS = {
|
|
|
17153
17469
|
function compact(value) {
|
|
17154
17470
|
return normalizeBrandText(value).replace(/\s+/g, "");
|
|
17155
17471
|
}
|
|
17156
|
-
function round$
|
|
17472
|
+
function round$2(value, digits) {
|
|
17157
17473
|
return Number(value.toFixed(digits));
|
|
17158
17474
|
}
|
|
17159
17475
|
function stemQueryEvidence(rows, stem) {
|
|
@@ -17180,8 +17496,8 @@ function stemQueryEvidence(rows, stem) {
|
|
|
17180
17496
|
query,
|
|
17181
17497
|
clicks,
|
|
17182
17498
|
impressions,
|
|
17183
|
-
ctr: round$
|
|
17184
|
-
position: round$
|
|
17499
|
+
ctr: round$2(clicks / impressions, 4),
|
|
17500
|
+
position: round$2(weightedPosition / impressions, 2),
|
|
17185
17501
|
sourceRows
|
|
17186
17502
|
};
|
|
17187
17503
|
}
|
|
@@ -17678,9 +17994,9 @@ function analyzeCtrUnderperformersFromRows(input) {
|
|
|
17678
17994
|
//#endregion
|
|
17679
17995
|
//#region packages/core/src/analyze/ctr-underperformers-report.ts
|
|
17680
17996
|
const MAX_GSC_ROWS$3 = 1e5;
|
|
17681
|
-
const DEFAULT_DAYS$
|
|
17997
|
+
const DEFAULT_DAYS$3 = 28;
|
|
17682
17998
|
const MAX_DAYS$2 = 548;
|
|
17683
|
-
function plural$
|
|
17999
|
+
function plural$5(count, singular, pluralLabel = `${singular}s`) {
|
|
17684
18000
|
return count === 1 ? singular : pluralLabel;
|
|
17685
18001
|
}
|
|
17686
18002
|
async function ctrUnderperformersReportWith(input, dependencies) {
|
|
@@ -17701,7 +18017,7 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
17701
18017
|
const explicitRange = explicitDateRange(input, MAX_DAYS$2);
|
|
17702
18018
|
const days = explicitRange?.days ?? integerOption$1({
|
|
17703
18019
|
value: input.days,
|
|
17704
|
-
fallback: DEFAULT_DAYS$
|
|
18020
|
+
fallback: DEFAULT_DAYS$3,
|
|
17705
18021
|
minimum: 1,
|
|
17706
18022
|
maximum: MAX_DAYS$2,
|
|
17707
18023
|
label: "Days"
|
|
@@ -17783,11 +18099,11 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
17783
18099
|
minImpressions,
|
|
17784
18100
|
limit,
|
|
17785
18101
|
brandFiltering: input.includeBrand ? "included" : "excluded",
|
|
17786
|
-
verdict: top ? `${selection.eligibleUnderperformers} CTR ${plural$
|
|
18102
|
+
verdict: top ? `${selection.eligibleUnderperformers} CTR ${plural$5(selection.eligibleUnderperformers, "underperformer")} found in retained rows, with a calculated heuristic shortfall of ${totalClickShortfall.toFixed(0)} clicks for this window. Start review with "${top.query}" because it has the largest gap.` : dataStatus === "partial" ? "No material CTR underperformers remained in the validated retained rows, but partial evidence prevents an all-clear." : `No high-impression rows at average position below 4 in the retained rows are materially below the heuristic CTR benchmark.`
|
|
17787
18103
|
},
|
|
17788
18104
|
items,
|
|
17789
18105
|
caveats: [
|
|
17790
|
-
`Date window: ${range.startDate} to ${range.endDate} (${days} ${plural$
|
|
18106
|
+
`Date window: ${range.startDate} to ${range.endDate} (${days} ${plural$5(days, "day")}), using final GSC data where available.`,
|
|
17791
18107
|
`Brand queries: ${input.includeBrand ? "included" : "excluded"}. Brand terms used: ${brand.terms.join(", ") || "none"} (${brand.source === "provided" ? "provided" : "host forms, plus the bare domain stem only when Search Console shows the site is its navigational answer"}).`,
|
|
17792
18108
|
`Only rows at average position below 4 with at least ${minImpressions} impressions were checked. Rows at position 4 to 10 belong to quick-wins, which uses the same site CTR curve.`,
|
|
17793
18109
|
minImpressionsCaveat({
|
|
@@ -17795,13 +18111,13 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
17795
18111
|
belowMinimumRows: selection.belowMinimumRows
|
|
17796
18112
|
}),
|
|
17797
18113
|
"Expected CTR uses the site CTR curve from all page-one rows (75th percentile of qualified URL CTRs per position) when enough peer data exists, otherwise the fallback position curve.",
|
|
17798
|
-
...selection.fragmentRows ? [`${selection.fragmentRows} source ${plural$
|
|
17799
|
-
...selection.flaggedRows ? [`${selection.flaggedRows} eligible ${plural$
|
|
18114
|
+
...selection.fragmentRows ? [`${selection.fragmentRows} source ${plural$5(selection.fragmentRows, "row")} with a #fragment URL ${selection.fragmentRows === 1 ? "was" : "were"} merged into the base URL.`] : [],
|
|
18115
|
+
...selection.flaggedRows ? [`${selection.flaggedRows} eligible ${plural$5(selection.flaggedRows, "row")} carry heuristic query flags (likely-automated or navigational-other) and rank after unflagged rows.`] : [],
|
|
17800
18116
|
"CTR benchmarks and calculated click shortfalls are directional heuristics, not forecasts or proof that clicks are available. Validate the search intent and live SERP before editing.",
|
|
17801
|
-
...selection.invalidRows ? [`${selection.invalidRows} invalid provider ${plural$
|
|
17802
|
-
...selection.duplicateRows ? [`${selection.duplicateRows} repeated query/page ${plural$
|
|
18117
|
+
...selection.invalidRows ? [`${selection.invalidRows} invalid provider ${plural$5(selection.invalidRows, "row")} ${selection.invalidRows === 1 ? "was" : "were"} excluded, so this report is partial.`] : [],
|
|
18118
|
+
...selection.duplicateRows ? [`${selection.duplicateRows} repeated query/page ${plural$5(selection.duplicateRows, "row")} ${selection.duplicateRows === 1 ? "was" : "were"} aggregated before CTR and average position were calculated.`] : [],
|
|
17803
18119
|
...possiblyTruncated ? [`The retained-row limit of ${MAX_GSC_ROWS$3} was reached, so lower-ranked source rows may be absent.`] : [],
|
|
17804
|
-
...selection.limitedUnderperformers ? [`${selection.limitedUnderperformers} eligible ${plural$
|
|
18120
|
+
...selection.limitedUnderperformers ? [`${selection.limitedUnderperformers} eligible ${plural$5(selection.limitedUnderperformers, "underperformer")} ${selection.limitedUnderperformers === 1 ? "was" : "were"} omitted by the output limit.`] : []
|
|
17805
18121
|
],
|
|
17806
18122
|
recommendations: top ? [
|
|
17807
18123
|
`Review the live SERP and displayed snippet for "${top.query}" first. Test the title or meta description only if the observed framing does not match the query intent.`,
|
|
@@ -17812,11 +18128,11 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
17812
18128
|
}
|
|
17813
18129
|
//#endregion
|
|
17814
18130
|
//#region packages/core/src/analyze/ctr-underperformers.ts
|
|
17815
|
-
const defaultDependencies$
|
|
18131
|
+
const defaultDependencies$9 = {
|
|
17816
18132
|
searchAnalytics: querySearchAnalytics,
|
|
17817
18133
|
now: () => /* @__PURE__ */ new Date()
|
|
17818
18134
|
};
|
|
17819
|
-
function ctrUnderperformersReport(input, dependencies = defaultDependencies$
|
|
18135
|
+
function ctrUnderperformersReport(input, dependencies = defaultDependencies$9) {
|
|
17820
18136
|
return ctrUnderperformersReportWith(input, dependencies);
|
|
17821
18137
|
}
|
|
17822
18138
|
//#endregion
|
|
@@ -18284,7 +18600,7 @@ function unavailableSegmentImpactReport(input) {
|
|
|
18284
18600
|
//#endregion
|
|
18285
18601
|
//#region packages/core/src/analyze/segment-impact.ts
|
|
18286
18602
|
const DEFAULT_MAX_ROWS = 1e5;
|
|
18287
|
-
const defaultDependencies$
|
|
18603
|
+
const defaultDependencies$8 = {
|
|
18288
18604
|
searchAnalytics: querySearchAnalytics,
|
|
18289
18605
|
now: () => /* @__PURE__ */ new Date()
|
|
18290
18606
|
};
|
|
@@ -18324,7 +18640,7 @@ function compareSegmentRows(input) {
|
|
|
18324
18640
|
generatedAt: input.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString()
|
|
18325
18641
|
});
|
|
18326
18642
|
}
|
|
18327
|
-
async function segmentImpact(input, dependencies = defaultDependencies$
|
|
18643
|
+
async function segmentImpact(input, dependencies = defaultDependencies$8) {
|
|
18328
18644
|
validateSegmentSite(input.site);
|
|
18329
18645
|
const now = dependencies.now();
|
|
18330
18646
|
const ranges = resolveSegmentRanges({
|
|
@@ -20182,11 +20498,11 @@ async function cannibalReportWith(input, dependencies) {
|
|
|
20182
20498
|
}
|
|
20183
20499
|
//#endregion
|
|
20184
20500
|
//#region packages/core/src/analyze/site-diagnostics/cannibal.ts
|
|
20185
|
-
const defaultDependencies$
|
|
20501
|
+
const defaultDependencies$7 = {
|
|
20186
20502
|
searchAnalytics: querySearchAnalytics,
|
|
20187
20503
|
now: () => /* @__PURE__ */ new Date()
|
|
20188
20504
|
};
|
|
20189
|
-
async function cannibalReport(input, dependencies = defaultDependencies$
|
|
20505
|
+
async function cannibalReport(input, dependencies = defaultDependencies$7) {
|
|
20190
20506
|
if (input.researchFiles) {
|
|
20191
20507
|
const conflicts = [
|
|
20192
20508
|
["days", input.days],
|
|
@@ -20644,11 +20960,11 @@ async function decayingReportWith(input, dependencies) {
|
|
|
20644
20960
|
}
|
|
20645
20961
|
//#endregion
|
|
20646
20962
|
//#region packages/core/src/analyze/site-diagnostics/decaying.ts
|
|
20647
|
-
const defaultDependencies$
|
|
20963
|
+
const defaultDependencies$6 = {
|
|
20648
20964
|
searchAnalytics: querySearchAnalytics,
|
|
20649
20965
|
now: () => /* @__PURE__ */ new Date()
|
|
20650
20966
|
};
|
|
20651
|
-
function decayingReport(input, dependencies = defaultDependencies$
|
|
20967
|
+
function decayingReport(input, dependencies = defaultDependencies$6) {
|
|
20652
20968
|
return decayingReportWith(input, dependencies);
|
|
20653
20969
|
}
|
|
20654
20970
|
//#endregion
|
|
@@ -20958,7 +21274,7 @@ function quickWinTemplateSummaries(items) {
|
|
|
20958
21274
|
}
|
|
20959
21275
|
//#endregion
|
|
20960
21276
|
//#region packages/core/src/analyze/site-diagnostics/quick-wins-report.ts
|
|
20961
|
-
const DEFAULT_DAYS$
|
|
21277
|
+
const DEFAULT_DAYS$2 = 28;
|
|
20962
21278
|
const MAX_DAYS$1 = 548;
|
|
20963
21279
|
const DEFAULT_VERIFY_LIMIT$1 = 5;
|
|
20964
21280
|
const MAX_VERIFY_LIMIT = 100;
|
|
@@ -20980,7 +21296,7 @@ async function quickWinsReportWith(input, dependencies) {
|
|
|
20980
21296
|
const explicitRange = explicitDateRange(input, MAX_DAYS$1);
|
|
20981
21297
|
const days = explicitRange?.days ?? integerOption$1({
|
|
20982
21298
|
value: input.days,
|
|
20983
|
-
fallback: DEFAULT_DAYS$
|
|
21299
|
+
fallback: DEFAULT_DAYS$2,
|
|
20984
21300
|
minimum: 1,
|
|
20985
21301
|
maximum: MAX_DAYS$1,
|
|
20986
21302
|
label: "Days"
|
|
@@ -21145,13 +21461,13 @@ async function quickWinsReportWith(input, dependencies) {
|
|
|
21145
21461
|
}
|
|
21146
21462
|
//#endregion
|
|
21147
21463
|
//#region packages/core/src/analyze/site-diagnostics/quick-wins.ts
|
|
21148
|
-
const defaultDependencies$
|
|
21464
|
+
const defaultDependencies$5 = {
|
|
21149
21465
|
searchAnalytics: querySearchAnalytics,
|
|
21150
21466
|
fetch: fetchPage,
|
|
21151
21467
|
extract: extractPage,
|
|
21152
21468
|
now: () => /* @__PURE__ */ new Date()
|
|
21153
21469
|
};
|
|
21154
|
-
function quickWinsReport(input, dependencies = defaultDependencies$
|
|
21470
|
+
function quickWinsReport(input, dependencies = defaultDependencies$5) {
|
|
21155
21471
|
return quickWinsReportWith(input, dependencies);
|
|
21156
21472
|
}
|
|
21157
21473
|
//#endregion
|
|
@@ -21404,7 +21720,7 @@ function analyzeStrikingDistanceRows(input) {
|
|
|
21404
21720
|
}
|
|
21405
21721
|
//#endregion
|
|
21406
21722
|
//#region packages/core/src/analyze/striking-distance-report.ts
|
|
21407
|
-
const DEFAULT_DAYS$
|
|
21723
|
+
const DEFAULT_DAYS$1 = 28;
|
|
21408
21724
|
const MAX_DAYS = 548;
|
|
21409
21725
|
const DEFAULT_VERIFY_LIMIT = 5;
|
|
21410
21726
|
const MAX_SOURCE_ROWS$1 = 1e5;
|
|
@@ -21413,7 +21729,7 @@ function boundedInteger$2(value, fallback, minimum, maximum) {
|
|
|
21413
21729
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
21414
21730
|
}
|
|
21415
21731
|
function reportDays(value) {
|
|
21416
|
-
if (value === void 0) return DEFAULT_DAYS$
|
|
21732
|
+
if (value === void 0) return DEFAULT_DAYS$1;
|
|
21417
21733
|
if (!Number.isInteger(value) || value < 1 || value > MAX_DAYS) throw new SeoError("INVALID_INPUT", `Days must be a whole number between 1 and ${MAX_DAYS}.`);
|
|
21418
21734
|
return value;
|
|
21419
21735
|
}
|
|
@@ -21560,12 +21876,12 @@ async function strikingDistanceWith(input, dependencies) {
|
|
|
21560
21876
|
}
|
|
21561
21877
|
//#endregion
|
|
21562
21878
|
//#region packages/core/src/analyze/striking-distance.ts
|
|
21563
|
-
const defaultDependencies$
|
|
21879
|
+
const defaultDependencies$4 = {
|
|
21564
21880
|
searchAnalytics: querySearchAnalytics,
|
|
21565
21881
|
verifyContent: verifyQueryContent,
|
|
21566
21882
|
now: () => /* @__PURE__ */ new Date()
|
|
21567
21883
|
};
|
|
21568
|
-
function strikingDistance(input, dependencies = defaultDependencies$
|
|
21884
|
+
function strikingDistance(input, dependencies = defaultDependencies$4) {
|
|
21569
21885
|
return strikingDistanceWith(input, dependencies);
|
|
21570
21886
|
}
|
|
21571
21887
|
//#endregion
|
|
@@ -21577,7 +21893,7 @@ async function fetchSearchStatusIncidents() {
|
|
|
21577
21893
|
return readJson(response, "Google Search Status", { maxBytes: 10 * 1024 * 1024 });
|
|
21578
21894
|
}
|
|
21579
21895
|
async function listSearchUpdates(input = {}) {
|
|
21580
|
-
return searchUpdatesFromIncidents(await fetchSearchStatusIncidents(), input);
|
|
21896
|
+
return searchUpdatesFromIncidents(withRankingHistory(await fetchSearchStatusIncidents()), input);
|
|
21581
21897
|
}
|
|
21582
21898
|
//#endregion
|
|
21583
21899
|
//#region packages/core/src/analyze/traffic-anomaly-analysis.ts
|
|
@@ -21636,7 +21952,7 @@ function anomalyEvidence(anomaly) {
|
|
|
21636
21952
|
if (anomaly.direction === "normal") return `${formatMetric(anomaly.metric)} changed ${changePhrase} (${anomaly.baselineMean.toLocaleString("en-GB")} -> ${anomaly.comparisonMean.toLocaleString("en-GB")} per observed day${anomaly.zScore === null ? "; the baseline had no observed variance" : `, z ${anomaly.zScore}`}).`;
|
|
21637
21953
|
return `${formatMetric(anomaly.metric)} ${direction} ${changePhrase} (${anomaly.baselineMean.toLocaleString("en-GB")} -> ${anomaly.comparisonMean.toLocaleString("en-GB")} per observed day${anomaly.zScore === null ? "; outside the flat observed baseline" : `, z ${anomaly.zScore}`}).`;
|
|
21638
21954
|
}
|
|
21639
|
-
function plural$
|
|
21955
|
+
function plural$4(count, singular, pluralLabel = `${singular}s`) {
|
|
21640
21956
|
return count === 1 ? singular : pluralLabel;
|
|
21641
21957
|
}
|
|
21642
21958
|
function strongestSignal(anomalies) {
|
|
@@ -21692,13 +22008,13 @@ function interpretUpdateCorrelation(input) {
|
|
|
21692
22008
|
summary,
|
|
21693
22009
|
evidence: [
|
|
21694
22010
|
...input.anomalies.map(anomalyEvidence),
|
|
21695
|
-
hasUpdate ? `${input.overlappingUpdates.length} official Google Ranking update ${plural$
|
|
22011
|
+
hasUpdate ? `${input.overlappingUpdates.length} official Google Ranking update ${plural$4(input.overlappingUpdates.length, "window")} overlapped the change date or comparison window: ${updateNames.join(", ")}.` : "No official Google Ranking update windows overlapped the change date or comparison window.",
|
|
21696
22012
|
largeMovement ? "Both clicks and impressions moved sharply in the same direction." : "Clicks and impressions did not both meet the sharp-movement evidence threshold.",
|
|
21697
22013
|
hasConfounders ? `Known overlapping site changes: ${confounders.map((confounder) => confounder.title).join("; ")}.` : "No overlapping site changes were supplied."
|
|
21698
22014
|
],
|
|
21699
22015
|
caveats: [
|
|
21700
22016
|
`GSC window: ${input.days} days; recent comparison window: ${input.recentDays} days.`,
|
|
21701
|
-
`Update overlap padding: ${input.paddingDays} ${plural$
|
|
22017
|
+
`Update overlap padding: ${input.paddingDays} ${plural$4(input.paddingDays, "day")} either side of the step-change date, or of the recent comparison window when no step change was found.`,
|
|
21702
22018
|
input.refresh ? "Data freshness: cache bypassed for GSC data." : "Data freshness: local GSC cache may have been used.",
|
|
21703
22019
|
"Source: official Google Search Status Dashboard Ranking incidents only; third-party volatility and unconfirmed chatter are not included.",
|
|
21704
22020
|
"An update-window overlap cannot establish causation, even when movement is large and no known site change was recorded.",
|
|
@@ -21866,7 +22182,7 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
21866
22182
|
if (step && changeDate) {
|
|
21867
22183
|
const previousDate = aggregated.rows[step.index - 1]?.keys[0];
|
|
21868
22184
|
const gap = previousDate ? calendarDays(previousDate, changeDate) - 2 : 0;
|
|
21869
|
-
stepCaveats.push(`${formatMetric(metric)} changed level on ${changeDate}, so its baseline is every observed day before that date and its comparison is every observed day from it.`, ...gap > 0 ? [`The ${metric} change date follows ${gap} missing ${plural$
|
|
22185
|
+
stepCaveats.push(`${formatMetric(metric)} changed level on ${changeDate}, so its baseline is every observed day before that date and its comparison is every observed day from it.`, ...gap > 0 ? [`The ${metric} change date follows ${gap} missing ${plural$4(gap, "day")}; the change may have started on one of them.`] : []);
|
|
21870
22186
|
return compareSegments({
|
|
21871
22187
|
site: input.site,
|
|
21872
22188
|
metric,
|
|
@@ -21897,9 +22213,9 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
21897
22213
|
const comparisonMissingDays = input.recentDays - comparisonRows.length;
|
|
21898
22214
|
const missingDays = expectedDays - aggregated.rows.length;
|
|
21899
22215
|
const caveats = [
|
|
21900
|
-
...missingDays > 0 ? [`${missingDays} requested calendar ${plural$
|
|
21901
|
-
...aggregated.invalidRows > 0 ? [`${aggregated.invalidRows} invalid provider ${plural$
|
|
21902
|
-
...aggregated.duplicateRows > 0 ? [`${aggregated.duplicateRows} duplicate date ${plural$
|
|
22216
|
+
...missingDays > 0 ? [`${missingDays} requested calendar ${plural$4(missingDays, "day")} had no returned date aggregate and ${missingDays === 1 ? "was" : "were"} not filled with zeros.`] : [],
|
|
22217
|
+
...aggregated.invalidRows > 0 ? [`${aggregated.invalidRows} invalid provider ${plural$4(aggregated.invalidRows, "row")} ${aggregated.invalidRows === 1 ? "was" : "were"} excluded.`] : [],
|
|
22218
|
+
...aggregated.duplicateRows > 0 ? [`${aggregated.duplicateRows} duplicate date ${plural$4(aggregated.duplicateRows, "row")} ${aggregated.duplicateRows === 1 ? "was" : "were"} aggregated before analysis.`] : [],
|
|
21903
22219
|
...stepCaveats
|
|
21904
22220
|
];
|
|
21905
22221
|
return {
|
|
@@ -21982,11 +22298,11 @@ async function trafficAnomalyWith(input, dependencies) {
|
|
|
21982
22298
|
}
|
|
21983
22299
|
//#endregion
|
|
21984
22300
|
//#region packages/core/src/analyze/traffic-anomaly.ts
|
|
21985
|
-
const defaultDependencies$
|
|
22301
|
+
const defaultDependencies$3 = {
|
|
21986
22302
|
searchAnalytics: querySearchAnalytics,
|
|
21987
22303
|
now: () => /* @__PURE__ */ new Date()
|
|
21988
22304
|
};
|
|
21989
|
-
function trafficAnomaly(input, dependencies = defaultDependencies$
|
|
22305
|
+
function trafficAnomaly(input, dependencies = defaultDependencies$3) {
|
|
21990
22306
|
return trafficAnomalyWith(input, dependencies);
|
|
21991
22307
|
}
|
|
21992
22308
|
async function updateCorrelation(input) {
|
|
@@ -21999,7 +22315,7 @@ async function updateCorrelation(input) {
|
|
|
21999
22315
|
const comparisonEnd = window?.end;
|
|
22000
22316
|
const updates = await listSearchUpdates({
|
|
22001
22317
|
product: "Ranking",
|
|
22002
|
-
limit:
|
|
22318
|
+
limit: Number.POSITIVE_INFINITY
|
|
22003
22319
|
});
|
|
22004
22320
|
const overlappingUpdates = comparisonStart && comparisonEnd ? findOverlappingSearchUpdates({
|
|
22005
22321
|
updates,
|
|
@@ -22344,7 +22660,7 @@ function emptyQuickWins(input) {
|
|
|
22344
22660
|
warnings: []
|
|
22345
22661
|
};
|
|
22346
22662
|
}
|
|
22347
|
-
const defaultDependencies$
|
|
22663
|
+
const defaultDependencies$2 = {
|
|
22348
22664
|
trafficAnomaly,
|
|
22349
22665
|
updateCorrelation,
|
|
22350
22666
|
segmentImpact,
|
|
@@ -22355,7 +22671,7 @@ const defaultDependencies$3 = {
|
|
|
22355
22671
|
};
|
|
22356
22672
|
async function diagnoseProperty(input, dependencies = {}) {
|
|
22357
22673
|
const providers = {
|
|
22358
|
-
...defaultDependencies$
|
|
22674
|
+
...defaultDependencies$2,
|
|
22359
22675
|
...dependencies
|
|
22360
22676
|
};
|
|
22361
22677
|
const limit = input.limit ?? 10;
|
|
@@ -22566,298 +22882,281 @@ async function diagnoseProperty(input, dependencies = {}) {
|
|
|
22566
22882
|
};
|
|
22567
22883
|
}
|
|
22568
22884
|
//#endregion
|
|
22569
|
-
//#region packages/core/src/
|
|
22570
|
-
|
|
22571
|
-
|
|
22572
|
-
|
|
22573
|
-
|
|
22574
|
-
|
|
22575
|
-
|
|
22576
|
-
|
|
22577
|
-
|
|
22578
|
-
"for",
|
|
22579
|
-
"free",
|
|
22580
|
-
"from",
|
|
22581
|
-
"guide",
|
|
22582
|
-
"how",
|
|
22583
|
-
"in",
|
|
22584
|
-
"is",
|
|
22585
|
-
"of",
|
|
22586
|
-
"on",
|
|
22587
|
-
"online",
|
|
22588
|
-
"or",
|
|
22589
|
-
"page",
|
|
22590
|
-
"seo",
|
|
22591
|
-
"site",
|
|
22592
|
-
"the",
|
|
22593
|
-
"to",
|
|
22594
|
-
"tool",
|
|
22595
|
-
"use",
|
|
22596
|
-
"website",
|
|
22597
|
-
"what",
|
|
22598
|
-
"when",
|
|
22599
|
-
"where",
|
|
22600
|
-
"which",
|
|
22601
|
-
"with"
|
|
22602
|
-
]);
|
|
22603
|
-
function compareInternalLinkText(left, right) {
|
|
22604
|
-
return left < right ? -1 : left > right ? 1 : 0;
|
|
22605
|
-
}
|
|
22606
|
-
function normalizeInternalLinkQuery(value) {
|
|
22607
|
-
return value.normalize("NFKC").replace(/[’']/g, "'").toLowerCase().replace(/[^\p{L}\p{N}']+/gu, " ").replace(/\s+/g, " ").trim();
|
|
22608
|
-
}
|
|
22609
|
-
function normalizeInternalLinkUrl(value) {
|
|
22610
|
-
try {
|
|
22611
|
-
const url = new URL(value);
|
|
22612
|
-
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
22613
|
-
url.hash = "";
|
|
22614
|
-
return url.toString();
|
|
22615
|
-
} catch {
|
|
22616
|
-
return;
|
|
22885
|
+
//#region packages/core/src/decision-model.ts
|
|
22886
|
+
var DecisionModelError = class extends Error {
|
|
22887
|
+
kind;
|
|
22888
|
+
retryAfterMs;
|
|
22889
|
+
constructor(kind, message, retryAfterMs) {
|
|
22890
|
+
super(message);
|
|
22891
|
+
this.kind = kind;
|
|
22892
|
+
this.retryAfterMs = retryAfterMs;
|
|
22893
|
+
this.name = "DecisionModelError";
|
|
22617
22894
|
}
|
|
22618
|
-
|
|
22619
|
-
|
|
22620
|
-
|
|
22621
|
-
|
|
22622
|
-
|
|
22623
|
-
|
|
22624
|
-
|
|
22625
|
-
|
|
22626
|
-
|
|
22627
|
-
|
|
22628
|
-
|
|
22629
|
-
|
|
22630
|
-
|
|
22631
|
-
|
|
22632
|
-
|
|
22633
|
-
|
|
22634
|
-
|
|
22635
|
-
|
|
22636
|
-
|
|
22637
|
-
|
|
22638
|
-
|
|
22639
|
-
|
|
22640
|
-
|
|
22641
|
-
|
|
22642
|
-
|
|
22643
|
-
|
|
22644
|
-
|
|
22645
|
-
|
|
22646
|
-
|
|
22647
|
-
|
|
22648
|
-
|
|
22649
|
-
|
|
22650
|
-
|
|
22651
|
-
|
|
22652
|
-
|
|
22653
|
-
|
|
22654
|
-
|
|
22655
|
-
|
|
22656
|
-
|
|
22657
|
-
|
|
22658
|
-
|
|
22659
|
-
|
|
22660
|
-
|
|
22661
|
-
|
|
22662
|
-
|
|
22663
|
-
|
|
22664
|
-
|
|
22665
|
-
|
|
22666
|
-
|
|
22667
|
-
|
|
22668
|
-
|
|
22895
|
+
get retryable() {
|
|
22896
|
+
return this.kind === "rate-limit" || this.kind === "unavailable";
|
|
22897
|
+
}
|
|
22898
|
+
};
|
|
22899
|
+
const TYPESAFE_URL = "https://api.typesafe.ai/v1/systemone";
|
|
22900
|
+
const JEV_PRICE_PER_MILLION = .042;
|
|
22901
|
+
const REQUEST_TIMEOUT_MS = 6e4;
|
|
22902
|
+
const MAX_RESPONSE_BYTES = 1024 * 1024;
|
|
22903
|
+
function retryAfter(response) {
|
|
22904
|
+
const value = Number(response.headers.get("retry-after"));
|
|
22905
|
+
return Number.isFinite(value) && value > 0 ? Math.min(value, 30) * 1e3 : void 0;
|
|
22906
|
+
}
|
|
22907
|
+
function typesafeDecisionModel(input) {
|
|
22908
|
+
const call = input.fetch ?? fetch;
|
|
22909
|
+
const model = input.model ?? "jev-latest";
|
|
22910
|
+
return {
|
|
22911
|
+
provider: "typesafe",
|
|
22912
|
+
name: model,
|
|
22913
|
+
pricePerMillionInputTokens: JEV_PRICE_PER_MILLION,
|
|
22914
|
+
async ask(request, signal) {
|
|
22915
|
+
const controller = new AbortController();
|
|
22916
|
+
const abort = () => controller.abort();
|
|
22917
|
+
const timer = setTimeout(abort, REQUEST_TIMEOUT_MS);
|
|
22918
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
22919
|
+
let response;
|
|
22920
|
+
try {
|
|
22921
|
+
response = await call(TYPESAFE_URL, {
|
|
22922
|
+
method: "POST",
|
|
22923
|
+
headers: {
|
|
22924
|
+
authorization: `Bearer ${input.apiKey}`,
|
|
22925
|
+
"content-type": "application/json"
|
|
22926
|
+
},
|
|
22927
|
+
body: JSON.stringify({
|
|
22928
|
+
model,
|
|
22929
|
+
...request
|
|
22930
|
+
}),
|
|
22931
|
+
signal: controller.signal
|
|
22932
|
+
});
|
|
22933
|
+
} catch (error) {
|
|
22934
|
+
throw new DecisionModelError("unavailable", `TypeSafe could not be reached: ${error instanceof Error ? error.message : String(error)}`);
|
|
22935
|
+
} finally {
|
|
22936
|
+
clearTimeout(timer);
|
|
22937
|
+
signal?.removeEventListener("abort", abort);
|
|
22938
|
+
}
|
|
22939
|
+
if (response.status === 401 || response.status === 403) {
|
|
22940
|
+
await response.body?.cancel();
|
|
22941
|
+
throw new DecisionModelError("auth", "TypeSafe rejected the API key. Replace it at https://seoskill.dev/account#keys or in TYPESAFE_API_KEY.");
|
|
22942
|
+
}
|
|
22943
|
+
if (response.status === 429 || response.status === 529) {
|
|
22944
|
+
await response.body?.cancel();
|
|
22945
|
+
throw new DecisionModelError("rate-limit", `TypeSafe returned HTTP ${response.status}.`, retryAfter(response));
|
|
22946
|
+
}
|
|
22947
|
+
if (response.status >= 500) {
|
|
22948
|
+
await response.body?.cancel();
|
|
22949
|
+
throw new DecisionModelError("unavailable", `TypeSafe returned HTTP ${response.status}.`);
|
|
22950
|
+
}
|
|
22951
|
+
if (!response.ok) {
|
|
22952
|
+
await response.body?.cancel();
|
|
22953
|
+
throw new DecisionModelError("invalid", `TypeSafe did not accept the request (HTTP ${response.status}).`);
|
|
22954
|
+
}
|
|
22955
|
+
const body = await readJson(response, "TypeSafe", { maxBytes: MAX_RESPONSE_BYTES });
|
|
22956
|
+
return {
|
|
22957
|
+
model: body.model ?? model,
|
|
22958
|
+
answers: body.answers ?? {},
|
|
22959
|
+
inputTokens: body.usage?.input_tokens ?? 0
|
|
22960
|
+
};
|
|
22961
|
+
}
|
|
22669
22962
|
};
|
|
22670
22963
|
}
|
|
22671
|
-
|
|
22672
|
-
|
|
22673
|
-
const
|
|
22674
|
-
const
|
|
22675
|
-
|
|
22676
|
-
|
|
22677
|
-
|
|
22678
|
-
|
|
22679
|
-
|
|
22680
|
-
|
|
22681
|
-
|
|
22682
|
-
|
|
22683
|
-
|
|
22684
|
-
|
|
22685
|
-
|
|
22686
|
-
|
|
22687
|
-
|
|
22688
|
-
|
|
22689
|
-
|
|
22690
|
-
impressions: 0
|
|
22691
|
-
};
|
|
22692
|
-
current.clicks += row.clicks;
|
|
22693
|
-
current.impressions += row.impressions;
|
|
22694
|
-
if (compareInternalLinkText(query, current.query) < 0) current.query = query;
|
|
22695
|
-
map.set(key, current);
|
|
22696
|
-
}
|
|
22697
|
-
function bestMatch(input) {
|
|
22698
|
-
const sourceKey = normalizeInternalLinkQuery(input.source.query);
|
|
22699
|
-
const exact = input.exactTargets.get(sourceKey);
|
|
22700
|
-
if (exact) return {
|
|
22701
|
-
sourceQuery: input.source.query,
|
|
22702
|
-
targetQuery: exact.query,
|
|
22703
|
-
kind: "exact-query",
|
|
22704
|
-
relevanceScore: 1,
|
|
22705
|
-
impressions: input.source.impressions,
|
|
22706
|
-
sharedTerms: internalLinkTokens(input.source.query)
|
|
22707
|
-
};
|
|
22708
|
-
return input.lexicalTargets.flatMap((target) => {
|
|
22709
|
-
const score = input.scoreLexical(input.source.query, target.query);
|
|
22710
|
-
return score ? [{
|
|
22711
|
-
sourceQuery: input.source.query,
|
|
22712
|
-
targetQuery: target.query,
|
|
22713
|
-
kind: "lexical-review",
|
|
22714
|
-
relevanceScore: score.score,
|
|
22715
|
-
impressions: input.source.impressions,
|
|
22716
|
-
sharedTerms: score.sharedTerms
|
|
22717
|
-
}] : [];
|
|
22718
|
-
}).sort((left, right) => right.relevanceScore - left.relevanceScore || compareInternalLinkText(left.targetQuery, right.targetQuery))[0];
|
|
22719
|
-
}
|
|
22720
|
-
function candidatesFromMatches(matchesByUrl) {
|
|
22721
|
-
return [...matchesByUrl.entries()].map(([sourceUrl, matches]) => {
|
|
22722
|
-
const ordered = [...matches].sort((left, right) => (left.kind === right.kind ? 0 : left.kind === "exact-query" ? -1 : 1) || right.impressions - left.impressions || right.relevanceScore - left.relevanceScore || compareInternalLinkText(left.sourceQuery, right.sourceQuery));
|
|
22964
|
+
const MAX_CONCURRENCY = 32;
|
|
22965
|
+
const MIN_CONCURRENCY = 2;
|
|
22966
|
+
const MAX_ATTEMPTS = 5;
|
|
22967
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
22968
|
+
var DecisionRunner = class {
|
|
22969
|
+
model;
|
|
22970
|
+
limit = MAX_CONCURRENCY;
|
|
22971
|
+
active = 0;
|
|
22972
|
+
streak = 0;
|
|
22973
|
+
waiting = [];
|
|
22974
|
+
served = "";
|
|
22975
|
+
requests = 0;
|
|
22976
|
+
failed = 0;
|
|
22977
|
+
tokens = 0;
|
|
22978
|
+
authError = null;
|
|
22979
|
+
constructor(model) {
|
|
22980
|
+
this.model = model;
|
|
22981
|
+
}
|
|
22982
|
+
usage() {
|
|
22723
22983
|
return {
|
|
22724
|
-
|
|
22725
|
-
|
|
22726
|
-
|
|
22727
|
-
|
|
22728
|
-
|
|
22729
|
-
|
|
22730
|
-
matches: ordered.slice(0, 10)
|
|
22984
|
+
provider: this.model.provider,
|
|
22985
|
+
model: this.served || this.model.name,
|
|
22986
|
+
requests: this.requests,
|
|
22987
|
+
failedRequests: this.failed,
|
|
22988
|
+
inputTokens: this.tokens,
|
|
22989
|
+
costUsd: Math.round(this.tokens * this.model.pricePerMillionInputTokens / 1e6 * 1e4) / 1e4
|
|
22731
22990
|
};
|
|
22732
|
-
}
|
|
22733
|
-
|
|
22734
|
-
|
|
22735
|
-
|
|
22736
|
-
|
|
22737
|
-
|
|
22738
|
-
|
|
22739
|
-
|
|
22740
|
-
|
|
22741
|
-
|
|
22742
|
-
|
|
22743
|
-
|
|
22744
|
-
|
|
22745
|
-
|
|
22746
|
-
|
|
22747
|
-
|
|
22748
|
-
|
|
22749
|
-
|
|
22750
|
-
|
|
22751
|
-
|
|
22752
|
-
|
|
22991
|
+
}
|
|
22992
|
+
async slot() {
|
|
22993
|
+
while (this.active >= this.limit) await new Promise((resolve) => this.waiting.push(resolve));
|
|
22994
|
+
this.active += 1;
|
|
22995
|
+
}
|
|
22996
|
+
release() {
|
|
22997
|
+
this.active -= 1;
|
|
22998
|
+
this.waiting.shift()?.();
|
|
22999
|
+
}
|
|
23000
|
+
async ask(request) {
|
|
23001
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
|
|
23002
|
+
if (this.authError) break;
|
|
23003
|
+
let delay = 0;
|
|
23004
|
+
await this.slot();
|
|
23005
|
+
try {
|
|
23006
|
+
const response = await this.model.ask(request);
|
|
23007
|
+
this.requests += 1;
|
|
23008
|
+
this.tokens += response.inputTokens;
|
|
23009
|
+
this.served = response.model;
|
|
23010
|
+
this.streak += 1;
|
|
23011
|
+
if (this.streak >= 50 && this.limit < MAX_CONCURRENCY) {
|
|
23012
|
+
this.limit += 1;
|
|
23013
|
+
this.streak = 0;
|
|
23014
|
+
this.waiting.shift()?.();
|
|
23015
|
+
}
|
|
23016
|
+
return response;
|
|
23017
|
+
} catch (error) {
|
|
23018
|
+
const failure = error instanceof DecisionModelError ? error : new DecisionModelError("unavailable", String(error));
|
|
23019
|
+
if (failure.kind === "auth") {
|
|
23020
|
+
this.authError = failure;
|
|
23021
|
+
break;
|
|
23022
|
+
}
|
|
23023
|
+
if (!failure.retryable || attempt === MAX_ATTEMPTS) break;
|
|
23024
|
+
if (failure.kind === "rate-limit") {
|
|
23025
|
+
this.limit = Math.max(MIN_CONCURRENCY, Math.floor(this.limit / 2));
|
|
23026
|
+
this.streak = 0;
|
|
23027
|
+
}
|
|
23028
|
+
delay = failure.retryAfterMs ?? 2 ** attempt * 250 + Math.floor(Math.random() * 250);
|
|
23029
|
+
} finally {
|
|
23030
|
+
this.release();
|
|
23031
|
+
}
|
|
23032
|
+
await sleep(delay);
|
|
22753
23033
|
}
|
|
22754
|
-
|
|
23034
|
+
this.failed += 1;
|
|
23035
|
+
return null;
|
|
22755
23036
|
}
|
|
22756
|
-
|
|
22757
|
-
|
|
22758
|
-
|
|
22759
|
-
|
|
22760
|
-
|
|
22761
|
-
|
|
22762
|
-
|
|
23037
|
+
};
|
|
23038
|
+
//#endregion
|
|
23039
|
+
//#region packages/core/src/analyze/internal-links-index.ts
|
|
23040
|
+
const PAGE_TERMS = 300;
|
|
23041
|
+
const VECTOR_TERMS = 100;
|
|
23042
|
+
var SiteIndex = class {
|
|
23043
|
+
ids = /* @__PURE__ */ new Map();
|
|
23044
|
+
df = [];
|
|
23045
|
+
counts = [];
|
|
23046
|
+
vectors = [];
|
|
23047
|
+
postings = [];
|
|
23048
|
+
scratch = /* @__PURE__ */ new Float64Array(0);
|
|
23049
|
+
get size() {
|
|
23050
|
+
return this.counts.length;
|
|
23051
|
+
}
|
|
23052
|
+
addPage(termCounts) {
|
|
23053
|
+
const kept = Object.entries(termCounts).sort((left, right) => right[1] - left[1] || (left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0)).slice(0, PAGE_TERMS);
|
|
23054
|
+
const ids = new Int32Array(kept.length);
|
|
23055
|
+
const counts = new Float32Array(kept.length);
|
|
23056
|
+
kept.forEach(([term, count], index) => {
|
|
23057
|
+
let id = this.ids.get(term);
|
|
23058
|
+
if (id === void 0) {
|
|
23059
|
+
id = this.df.length;
|
|
23060
|
+
this.ids.set(term, id);
|
|
23061
|
+
this.df.push(0);
|
|
23062
|
+
}
|
|
23063
|
+
this.df[id] = (this.df[id] ?? 0) + 1;
|
|
23064
|
+
ids[index] = id;
|
|
23065
|
+
counts[index] = count;
|
|
22763
23066
|
});
|
|
22764
|
-
|
|
22765
|
-
|
|
22766
|
-
|
|
22767
|
-
|
|
22768
|
-
|
|
22769
|
-
const lexicalTargets = allTargets.slice(0, LEXICAL_TARGET_LIMIT);
|
|
22770
|
-
const exactTargets = new Map(allTargets.map((target) => [normalizeInternalLinkQuery(target.query), target]));
|
|
22771
|
-
const sourceMap = /* @__PURE__ */ new Map();
|
|
22772
|
-
let sourceValidRows = 0;
|
|
22773
|
-
for (const row of input.sourceRows) {
|
|
22774
|
-
if (!validInternalLinkRow(row)) {
|
|
22775
|
-
increment("sourceInvalidRows");
|
|
22776
|
-
continue;
|
|
22777
|
-
}
|
|
22778
|
-
sourceValidRows += 1;
|
|
22779
|
-
const query = row.keys[0] ?? "";
|
|
22780
|
-
const page = normalizeInternalLinkUrl(row.keys[1] ?? "");
|
|
22781
|
-
if (!page) continue;
|
|
22782
|
-
if (aliases.has(page)) {
|
|
22783
|
-
increment("sourceTargetAliasRows");
|
|
22784
|
-
continue;
|
|
22785
|
-
}
|
|
22786
|
-
const key = `${page}\u0000${normalizeInternalLinkQuery(query)}`;
|
|
22787
|
-
const current = sourceMap.get(key) ?? {
|
|
22788
|
-
page,
|
|
22789
|
-
query,
|
|
22790
|
-
clicks: 0,
|
|
22791
|
-
impressions: 0
|
|
22792
|
-
};
|
|
22793
|
-
current.clicks += row.clicks;
|
|
22794
|
-
current.impressions += row.impressions;
|
|
22795
|
-
if (compareInternalLinkText(query, current.query) < 0) current.query = query;
|
|
22796
|
-
sourceMap.set(key, current);
|
|
23067
|
+
this.counts.push({
|
|
23068
|
+
ids,
|
|
23069
|
+
counts
|
|
23070
|
+
});
|
|
23071
|
+
return this.counts.length - 1;
|
|
22797
23072
|
}
|
|
22798
|
-
|
|
22799
|
-
|
|
22800
|
-
|
|
22801
|
-
|
|
22802
|
-
|
|
22803
|
-
|
|
22804
|
-
|
|
23073
|
+
idfOf(id) {
|
|
23074
|
+
return Math.log((this.size + 1) / ((this.df[id] ?? 1) + 1)) + 1;
|
|
23075
|
+
}
|
|
23076
|
+
idf(term) {
|
|
23077
|
+
const id = this.ids.get(term);
|
|
23078
|
+
return id === void 0 ? Math.log((this.size + 1) / 2) + 1 : this.idfOf(id);
|
|
23079
|
+
}
|
|
23080
|
+
finish() {
|
|
23081
|
+
this.vectors = this.counts.map(({ ids, counts }) => {
|
|
23082
|
+
let norm = 0;
|
|
23083
|
+
const shared = [];
|
|
23084
|
+
for (let index = 0; index < ids.length; index += 1) {
|
|
23085
|
+
const id = ids[index] ?? 0;
|
|
23086
|
+
const weight = (1 + Math.log(counts[index] ?? 1)) * this.idfOf(id);
|
|
23087
|
+
norm += weight * weight;
|
|
23088
|
+
if ((this.df[id] ?? 0) >= 2) shared.push([id, weight]);
|
|
23089
|
+
}
|
|
23090
|
+
norm = Math.sqrt(norm) || 1;
|
|
23091
|
+
shared.sort((left, right) => right[1] - left[1] || left[0] - right[0]);
|
|
23092
|
+
const top = shared.slice(0, VECTOR_TERMS);
|
|
23093
|
+
return {
|
|
23094
|
+
ids: Int32Array.from(top.map(([id]) => id)),
|
|
23095
|
+
weights: Float32Array.from(top.map(([, weight]) => weight / norm))
|
|
23096
|
+
};
|
|
22805
23097
|
});
|
|
22806
|
-
|
|
22807
|
-
|
|
22808
|
-
|
|
22809
|
-
|
|
22810
|
-
|
|
22811
|
-
|
|
22812
|
-
|
|
22813
|
-
|
|
22814
|
-
|
|
22815
|
-
|
|
22816
|
-
|
|
22817
|
-
exactTargets,
|
|
22818
|
-
lexicalTargets,
|
|
22819
|
-
scoreLexical
|
|
23098
|
+
const lists = this.df.map(() => ({
|
|
23099
|
+
pages: [],
|
|
23100
|
+
weights: []
|
|
23101
|
+
}));
|
|
23102
|
+
this.vectors.forEach((vector, page) => {
|
|
23103
|
+
vector.ids.forEach((id, index) => {
|
|
23104
|
+
const list = lists[id];
|
|
23105
|
+
if (!list) return;
|
|
23106
|
+
list.pages.push(page);
|
|
23107
|
+
list.weights.push(vector.weights[index] ?? 0);
|
|
23108
|
+
});
|
|
22820
23109
|
});
|
|
22821
|
-
|
|
22822
|
-
|
|
22823
|
-
|
|
23110
|
+
this.postings = lists.map((list) => ({
|
|
23111
|
+
pages: Int32Array.from(list.pages),
|
|
23112
|
+
weights: Float32Array.from(list.weights)
|
|
23113
|
+
}));
|
|
23114
|
+
this.scratch = new Float64Array(this.size);
|
|
23115
|
+
}
|
|
23116
|
+
similarity(left, right) {
|
|
23117
|
+
const a = this.vectors[left];
|
|
23118
|
+
const b = this.vectors[right];
|
|
23119
|
+
if (!a || !b) return 0;
|
|
23120
|
+
const weights = /* @__PURE__ */ new Map();
|
|
23121
|
+
a.ids.forEach((id, index) => {
|
|
23122
|
+
weights.set(id, a.weights[index] ?? 0);
|
|
23123
|
+
});
|
|
23124
|
+
let sum = 0;
|
|
23125
|
+
b.ids.forEach((id, index) => {
|
|
23126
|
+
sum += (weights.get(id) ?? 0) * (b.weights[index] ?? 0);
|
|
23127
|
+
});
|
|
23128
|
+
return sum;
|
|
23129
|
+
}
|
|
23130
|
+
neighbours(page, min) {
|
|
23131
|
+
const vector = this.vectors[page];
|
|
23132
|
+
if (!vector) return [];
|
|
23133
|
+
const scores = this.scratch;
|
|
23134
|
+
const touched = [];
|
|
23135
|
+
vector.ids.forEach((id, index) => {
|
|
23136
|
+
const weight = vector.weights[index] ?? 0;
|
|
23137
|
+
const list = this.postings[id];
|
|
23138
|
+
if (!list) return;
|
|
23139
|
+
for (let at = 0; at < list.pages.length; at += 1) {
|
|
23140
|
+
const other = list.pages[at] ?? 0;
|
|
23141
|
+
if (other === page) continue;
|
|
23142
|
+
if (scores[other] === 0) touched.push(other);
|
|
23143
|
+
scores[other] = (scores[other] ?? 0) + weight * (list.weights[at] ?? 0);
|
|
23144
|
+
}
|
|
23145
|
+
});
|
|
23146
|
+
const out = [];
|
|
23147
|
+
for (const other of touched) {
|
|
23148
|
+
const score = scores[other] ?? 0;
|
|
23149
|
+
if (score >= min) out.push({
|
|
23150
|
+
page: other,
|
|
23151
|
+
score
|
|
23152
|
+
});
|
|
23153
|
+
scores[other] = 0;
|
|
22824
23154
|
}
|
|
22825
|
-
|
|
22826
|
-
const matches = matchesByUrl.get(source.page) ?? [];
|
|
22827
|
-
matches.push(match);
|
|
22828
|
-
matchesByUrl.set(source.page, matches);
|
|
23155
|
+
return out.sort((left, right) => right.score - left.score || left.page - right.page);
|
|
22829
23156
|
}
|
|
22830
|
-
|
|
22831
|
-
|
|
22832
|
-
|
|
22833
|
-
targetQueries: lexicalTargets,
|
|
22834
|
-
candidates,
|
|
22835
|
-
selection: {
|
|
22836
|
-
targetSourceRows: input.targetRows.length,
|
|
22837
|
-
targetValidRows,
|
|
22838
|
-
targetInvalidRows: value("targetInvalidRows"),
|
|
22839
|
-
targetUrlMismatchRows: value("targetUrlMismatchRows"),
|
|
22840
|
-
targetLowImpressionQueries: value("targetLowImpressionQueries"),
|
|
22841
|
-
targetLowActionabilityQueries: value("targetLowActionabilityQueries"),
|
|
22842
|
-
targetBrandQueries: value("targetBrandQueries"),
|
|
22843
|
-
targetEligibleQueries: allTargets.length,
|
|
22844
|
-
selectedLexicalTargetQueries: lexicalTargets.length,
|
|
22845
|
-
sourceRows: input.sourceRows.length,
|
|
22846
|
-
sourceValidRows,
|
|
22847
|
-
sourceInvalidRows: value("sourceInvalidRows"),
|
|
22848
|
-
sourceTargetAliasRows: value("sourceTargetAliasRows"),
|
|
22849
|
-
sourceLowImpressionQueries: value("sourceLowImpressionQueries"),
|
|
22850
|
-
sourceLowActionabilityQueries: value("sourceLowActionabilityQueries"),
|
|
22851
|
-
sourceBrandQueries: value("sourceBrandQueries"),
|
|
22852
|
-
sourceUnmatchedQueries: value("sourceUnmatchedQueries"),
|
|
22853
|
-
candidateQueries,
|
|
22854
|
-
candidateUrls: candidates.length
|
|
22855
|
-
}
|
|
22856
|
-
};
|
|
22857
|
-
}
|
|
22858
|
-
const MAX_TERMS = 100;
|
|
22859
|
-
const PLACEMENT_SENTENCES = 2;
|
|
22860
|
-
const ANCHORS_PER_SENTENCE = 3;
|
|
23157
|
+
};
|
|
23158
|
+
const PAIR_SENTENCES = 4;
|
|
23159
|
+
const PAIR_ANCHORS = 8;
|
|
22861
23160
|
const MIN_SENTENCE_WORDS = 6;
|
|
22862
23161
|
const NAME_WEIGHT = 2;
|
|
22863
23162
|
const GENERIC_ANCHORS = /^(click here|here|read more|learn more|more|this|this post|this article|this page|link|website|check it out|see more|view|continue reading)$/i;
|
|
@@ -23127,65 +23426,49 @@ function pageSentences(html) {
|
|
|
23127
23426
|
flushRaw("");
|
|
23128
23427
|
return sentences;
|
|
23129
23428
|
}
|
|
23130
|
-
|
|
23131
|
-
|
|
23132
|
-
|
|
23133
|
-
|
|
23134
|
-
|
|
23135
|
-
|
|
23136
|
-
|
|
23137
|
-
|
|
23138
|
-
return Math.log((this.size + 1) / ((this.df.get(term) ?? 1) + 1)) + 1;
|
|
23139
|
-
}
|
|
23140
|
-
vector(counts) {
|
|
23141
|
-
let norm = 0;
|
|
23142
|
-
const shared = [];
|
|
23143
|
-
for (const term in counts) {
|
|
23144
|
-
const weight = (1 + Math.log(counts[term] ?? 1)) * this.idf(term);
|
|
23145
|
-
norm += weight * weight;
|
|
23146
|
-
if ((this.df.get(term) ?? 0) >= 2) shared.push([term, weight]);
|
|
23147
|
-
}
|
|
23148
|
-
norm = Math.sqrt(norm) || 1;
|
|
23149
|
-
shared.sort((left, right) => right[1] - left[1] || (left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0));
|
|
23150
|
-
return new Map(shared.slice(0, MAX_TERMS).map(([term, weight]) => [term, weight / norm]));
|
|
23151
|
-
}
|
|
23152
|
-
similarity(left, right) {
|
|
23153
|
-
let sum = 0;
|
|
23154
|
-
for (const [term, weight] of left) sum += weight * (right.get(term) ?? 0);
|
|
23155
|
-
return sum;
|
|
23429
|
+
function linkableSentences(sentences) {
|
|
23430
|
+
const seen = /* @__PURE__ */ new Set();
|
|
23431
|
+
const out = [];
|
|
23432
|
+
for (const sentence of sentences) {
|
|
23433
|
+
if (out.length >= 150) break;
|
|
23434
|
+
if (seen.has(sentence.text)) continue;
|
|
23435
|
+
seen.add(sentence.text);
|
|
23436
|
+
if (!sentence.hasLink && sentence.text.split(/\s+/).length >= MIN_SENTENCE_WORDS) out.push(sentence);
|
|
23156
23437
|
}
|
|
23157
|
-
|
|
23158
|
-
|
|
23159
|
-
|
|
23160
|
-
|
|
23161
|
-
|
|
23162
|
-
|
|
23163
|
-
|
|
23164
|
-
|
|
23165
|
-
|
|
23166
|
-
|
|
23167
|
-
|
|
23438
|
+
return out;
|
|
23439
|
+
}
|
|
23440
|
+
function sentenceMatch(sentence, key, weights) {
|
|
23441
|
+
if (!key.size) return 0;
|
|
23442
|
+
const seen = new Set(terms(sentence));
|
|
23443
|
+
let hit = 0;
|
|
23444
|
+
let total = 0;
|
|
23445
|
+
for (const [term, place] of key) {
|
|
23446
|
+
const weight = weights.idf(term) * place;
|
|
23447
|
+
total += weight;
|
|
23448
|
+
if (seen.has(term)) hit += weight;
|
|
23168
23449
|
}
|
|
23169
|
-
|
|
23170
|
-
|
|
23450
|
+
return total ? hit / total : 0;
|
|
23451
|
+
}
|
|
23452
|
+
function anchorCandidates(sentence, key, weights, limit) {
|
|
23171
23453
|
const scored = /* @__PURE__ */ new Map();
|
|
23172
23454
|
const parts = sentence.split(/\s+/).map((word) => word.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "")).filter(Boolean);
|
|
23173
|
-
for (let size =
|
|
23455
|
+
for (let size = 1; size <= 4; size += 1) for (let start = 0; start + size <= parts.length; start += 1) {
|
|
23174
23456
|
const gram = parts.slice(start, start + size);
|
|
23175
23457
|
const first = (gram[0] ?? "").toLowerCase();
|
|
23176
23458
|
const last = (gram[size - 1] ?? "").toLowerCase();
|
|
23177
23459
|
if (FUNCTION_WORDS.has(first) || FUNCTION_WORDS.has(last) || /^\d+$/.test(first) || /^\d+$/.test(last)) continue;
|
|
23178
23460
|
const stems = terms(gram.join(" "));
|
|
23179
|
-
if (stems.length < 2) continue;
|
|
23461
|
+
if (stems.length < size || size > 1 && stems.length < 2) continue;
|
|
23462
|
+
if (size === 1 && key.get(stems[0] ?? "") !== NAME_WEIGHT) continue;
|
|
23180
23463
|
const hits = stems.filter((term) => key.has(term)).length;
|
|
23181
|
-
const named = stems.reduce((sum, term) => sum + (key.get(term) ?? 0) *
|
|
23464
|
+
const named = stems.reduce((sum, term) => sum + (key.get(term) ?? 0) * weights.idf(term), 0);
|
|
23182
23465
|
if (!stems.some((term) => key.get(term) === NAME_WEIGHT)) continue;
|
|
23183
23466
|
if (Number(key.has(terms(first)[0] ?? "")) + Number(key.has(terms(last)[0] ?? "")) < 2) continue;
|
|
23184
23467
|
const text = gram.join(" ");
|
|
23185
23468
|
const lower = text.toLowerCase();
|
|
23186
23469
|
if (GENERIC_ANCHORS.test(lower)) continue;
|
|
23187
|
-
const rarity = stems.reduce((sum, term) => sum +
|
|
23188
|
-
const score = named * (1 + hits / key.size) + rarity * .05 - (size === 4 ? .2 : 0);
|
|
23470
|
+
const rarity = stems.reduce((sum, term) => sum + weights.idf(term), 0);
|
|
23471
|
+
const score = named * (1 + hits / key.size) + rarity * .05 - (size === 1 ? .5 : size === 4 ? .2 : 0);
|
|
23189
23472
|
const existing = scored.get(lower);
|
|
23190
23473
|
if (!existing || existing.score < score) scored.set(lower, {
|
|
23191
23474
|
text,
|
|
@@ -23194,27 +23477,27 @@ function anchorCandidates(sentence, key, corpus, limit) {
|
|
|
23194
23477
|
}
|
|
23195
23478
|
return [...scored.values()].sort((left, right) => right.score - left.score || (left.text < right.text ? -1 : left.text > right.text ? 1 : 0)).slice(0, limit).map((item) => item.text);
|
|
23196
23479
|
}
|
|
23197
|
-
function
|
|
23480
|
+
function pairPlacement(sentences, target, weights) {
|
|
23198
23481
|
const key = keyTerms(target);
|
|
23199
|
-
const
|
|
23200
|
-
const seen = /* @__PURE__ */ new Set();
|
|
23201
|
-
const sentences = source.sentences.filter((sentence) => {
|
|
23202
|
-
if (seen.has(sentence.text)) return false;
|
|
23203
|
-
seen.add(sentence.text);
|
|
23204
|
-
return !sentence.hasLink && sentence.text.split(/\s+/).length >= MIN_SENTENCE_WORDS;
|
|
23205
|
-
}).map((sentence, index) => ({
|
|
23482
|
+
const chosen = sentences.map((sentence, index) => ({
|
|
23206
23483
|
sentence,
|
|
23207
23484
|
index,
|
|
23208
|
-
match:
|
|
23209
|
-
})).filter((item) => item.match > 0).sort((left, right) => right.match - left.match || left.index - right.index).slice(0,
|
|
23210
|
-
|
|
23211
|
-
|
|
23212
|
-
|
|
23213
|
-
|
|
23214
|
-
|
|
23485
|
+
match: sentenceMatch(sentence.text, key, weights)
|
|
23486
|
+
})).filter((item) => item.match > 0).sort((left, right) => right.match - left.match || left.index - right.index).slice(0, PAIR_SENTENCES);
|
|
23487
|
+
const perSentence = Math.max(1, Math.floor(PAIR_ANCHORS / Math.max(1, chosen.length)));
|
|
23488
|
+
const anchors = [];
|
|
23489
|
+
for (const { sentence, index } of chosen) for (const text of anchorCandidates(sentence.text, key, weights, perSentence)) anchors.push({
|
|
23490
|
+
text,
|
|
23491
|
+
sentence: `S${index + 1}`
|
|
23492
|
+
});
|
|
23215
23493
|
return {
|
|
23216
|
-
|
|
23217
|
-
|
|
23494
|
+
sentences: chosen.map(({ sentence, index, match }) => ({
|
|
23495
|
+
id: `S${index + 1}`,
|
|
23496
|
+
text: sentence.text,
|
|
23497
|
+
section: sentence.section,
|
|
23498
|
+
match: Math.round(match * 100) / 100
|
|
23499
|
+
})),
|
|
23500
|
+
anchors: anchors.slice(0, PAIR_ANCHORS)
|
|
23218
23501
|
};
|
|
23219
23502
|
}
|
|
23220
23503
|
function placementPage(input) {
|
|
@@ -23228,542 +23511,1141 @@ function placementPage(input) {
|
|
|
23228
23511
|
};
|
|
23229
23512
|
}
|
|
23230
23513
|
//#endregion
|
|
23231
|
-
//#region packages/core/src/analyze/internal-links-
|
|
23232
|
-
const
|
|
23233
|
-
const
|
|
23234
|
-
|
|
23235
|
-
|
|
23236
|
-
|
|
23237
|
-
|
|
23238
|
-
|
|
23239
|
-
|
|
23240
|
-
|
|
23241
|
-
|
|
23242
|
-
|
|
23243
|
-
|
|
23244
|
-
|
|
23245
|
-
|
|
23246
|
-
|
|
23247
|
-
|
|
23248
|
-
|
|
23249
|
-
|
|
23250
|
-
|
|
23251
|
-
|
|
23252
|
-
|
|
23253
|
-
|
|
23254
|
-
|
|
23255
|
-
|
|
23256
|
-
|
|
23257
|
-
|
|
23258
|
-
|
|
23259
|
-
|
|
23260
|
-
|
|
23261
|
-
|
|
23514
|
+
//#region packages/core/src/analyze/internal-links-crawl.ts
|
|
23515
|
+
const DEFAULT_MAX_PAGES = 500;
|
|
23516
|
+
const MAX_MAX_PAGES = 5e3;
|
|
23517
|
+
const FETCH_CONCURRENCY = 6;
|
|
23518
|
+
const MAX_CRAWL_DELAY_SECONDS = 10;
|
|
23519
|
+
const CRAWL_DELAY_BUDGET_SECONDS = 900;
|
|
23520
|
+
const REDIRECT_CHECKS = 150;
|
|
23521
|
+
const GENERIC_ANCHOR_LIMIT = 100;
|
|
23522
|
+
const EXCERPT_CHARS = 500;
|
|
23523
|
+
const ASSET = /\.(avif|bmp|css|csv|dmg|docx?|eot|gif|gz|ico|jpe?g|js|json|map|mov|mp3|mp4|ogg|otf|pdf|png|pptx?|rss|svg|tar|ttf|txt|wav|webm|webp|woff2?|xlsx?|xml|zip)$/i;
|
|
23524
|
+
const TRACKING = /^(utm_[a-z]+|gclid|fbclid|msclkid|mc_cid|mc_eid|ref)$/i;
|
|
23525
|
+
var SentenceStore = class SentenceStore {
|
|
23526
|
+
dir;
|
|
23527
|
+
handle;
|
|
23528
|
+
size = 0;
|
|
23529
|
+
constructor(dir, handle) {
|
|
23530
|
+
this.dir = dir;
|
|
23531
|
+
this.handle = handle;
|
|
23532
|
+
}
|
|
23533
|
+
static async open() {
|
|
23534
|
+
const dir = await mkdtemp(join(tmpdir(), "seo-internal-links-"));
|
|
23535
|
+
return new SentenceStore(dir, await open$1(join(dir, "sentences"), "w+"));
|
|
23536
|
+
}
|
|
23537
|
+
async write(sentences) {
|
|
23538
|
+
const bytes = Buffer.from(JSON.stringify(sentences.map((s) => [s.text, s.section])));
|
|
23539
|
+
const offset = this.size;
|
|
23540
|
+
this.size += bytes.length;
|
|
23541
|
+
await this.handle.write(bytes, 0, bytes.length, offset);
|
|
23542
|
+
return {
|
|
23543
|
+
offset,
|
|
23544
|
+
length: bytes.length
|
|
23545
|
+
};
|
|
23262
23546
|
}
|
|
23263
|
-
|
|
23264
|
-
|
|
23265
|
-
|
|
23266
|
-
|
|
23267
|
-
|
|
23268
|
-
|
|
23269
|
-
|
|
23270
|
-
|
|
23271
|
-
|
|
23272
|
-
if (!value) return void 0;
|
|
23273
|
-
try {
|
|
23274
|
-
return assertUrlMatchesGscProperty(site, value);
|
|
23275
|
-
} catch {
|
|
23276
|
-
return;
|
|
23547
|
+
async read(ref) {
|
|
23548
|
+
if (!ref.length) return [];
|
|
23549
|
+
const bytes = Buffer.alloc(ref.length);
|
|
23550
|
+
await this.handle.read(bytes, 0, ref.length, ref.offset);
|
|
23551
|
+
return JSON.parse(bytes.toString("utf8")).map(([text, section]) => ({
|
|
23552
|
+
text,
|
|
23553
|
+
section,
|
|
23554
|
+
hasLink: false
|
|
23555
|
+
}));
|
|
23277
23556
|
}
|
|
23278
|
-
|
|
23279
|
-
|
|
23280
|
-
|
|
23281
|
-
|
|
23282
|
-
|
|
23283
|
-
}
|
|
23284
|
-
function sameObservedUrl(left, right) {
|
|
23285
|
-
try {
|
|
23286
|
-
return withoutFragment(left) === withoutFragment(right);
|
|
23287
|
-
} catch {
|
|
23288
|
-
return false;
|
|
23557
|
+
async close() {
|
|
23558
|
+
await this.handle.close().catch(() => void 0);
|
|
23559
|
+
await rm(this.dir, {
|
|
23560
|
+
recursive: true,
|
|
23561
|
+
force: true
|
|
23562
|
+
});
|
|
23289
23563
|
}
|
|
23290
|
-
}
|
|
23291
|
-
|
|
23292
|
-
|
|
23564
|
+
};
|
|
23565
|
+
const bareHost = (host) => host.toLowerCase().replace(/^www\./, "");
|
|
23566
|
+
function crawlUrl(value, base) {
|
|
23293
23567
|
try {
|
|
23294
|
-
|
|
23568
|
+
const url = base ? new URL(value, base) : new URL(value);
|
|
23569
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
23570
|
+
url.hash = "";
|
|
23571
|
+
for (const key of [...url.searchParams.keys()]) if (TRACKING.test(key)) url.searchParams.delete(key);
|
|
23572
|
+
return url.toString();
|
|
23295
23573
|
} catch {
|
|
23296
23574
|
return;
|
|
23297
23575
|
}
|
|
23298
23576
|
}
|
|
23299
|
-
|
|
23300
|
-
let
|
|
23301
|
-
|
|
23302
|
-
|
|
23303
|
-
|
|
23304
|
-
|
|
23305
|
-
|
|
23306
|
-
|
|
23577
|
+
function excerptOf(sentences) {
|
|
23578
|
+
let out = "";
|
|
23579
|
+
for (const sentence of sentences) {
|
|
23580
|
+
const next = out ? `${out} ${sentence.text}` : sentence.text;
|
|
23581
|
+
if (next.length > EXCERPT_CHARS) break;
|
|
23582
|
+
out = next;
|
|
23583
|
+
}
|
|
23584
|
+
return out || (sentences[0]?.text.slice(0, EXCERPT_CHARS) ?? "");
|
|
23585
|
+
}
|
|
23586
|
+
function readPage(fetched, finalUrl) {
|
|
23587
|
+
const $ = load(fetched.html);
|
|
23588
|
+
const meta = (name) => $(`meta[name="${name}" i]`).map((_, element) => $(element).attr("content") ?? "").get();
|
|
23589
|
+
const headings = $("h1, h2, h3").map((_, element) => ({
|
|
23590
|
+
level: Number(element.tagName.slice(1)),
|
|
23591
|
+
text: $(element).text().replace(/\s+/g, " ").trim()
|
|
23592
|
+
})).get().filter((heading) => heading.text);
|
|
23593
|
+
const links = $("a[href]").map((_, element) => {
|
|
23594
|
+
const anchor = $(element);
|
|
23307
23595
|
return {
|
|
23308
|
-
|
|
23309
|
-
|
|
23310
|
-
|
|
23311
|
-
verification: "failed",
|
|
23312
|
-
technicalSignals: [],
|
|
23313
|
-
warnings: [{
|
|
23314
|
-
stage: "target-fetch",
|
|
23315
|
-
url: input.targetUrl,
|
|
23316
|
-
code: "fetch-failed",
|
|
23317
|
-
message: errorMessage(error)
|
|
23318
|
-
}]
|
|
23596
|
+
href: anchor.attr("href") ?? "",
|
|
23597
|
+
text: anchor.text().replace(/\s+/g, " ").trim(),
|
|
23598
|
+
location: anchor.closest("nav, header").length ? "navigation" : anchor.closest("footer").length ? "footer" : anchor.closest("main, article, [role=\"main\"]").length ? "main-content" : "other"
|
|
23319
23599
|
};
|
|
23320
|
-
}
|
|
23321
|
-
|
|
23600
|
+
}).get();
|
|
23601
|
+
const robots = [...meta("robots"), ...meta("googlebot")];
|
|
23602
|
+
return {
|
|
23603
|
+
title: $("title").first().text().replace(/\s+/g, " ").trim(),
|
|
23604
|
+
metaDescription: meta("description")[0]?.trim() ?? "",
|
|
23605
|
+
canonical: $("link[rel~=\"canonical\" i]").first().attr("href"),
|
|
23606
|
+
lang: $("html").attr("lang"),
|
|
23607
|
+
headings,
|
|
23608
|
+
links,
|
|
23609
|
+
nofollow: robots.some((value) => /\bnofollow\b/i.test(value)),
|
|
23610
|
+
noindex: hasMetaRobotsDirective(robots, "noindex") || hasXRobotsDirective(fetched.headers["x-robots-tag"], "noindex"),
|
|
23611
|
+
url: finalUrl
|
|
23612
|
+
};
|
|
23613
|
+
}
|
|
23614
|
+
async function crawlForLinks(input) {
|
|
23615
|
+
const warnings = [];
|
|
23616
|
+
const robots = createRobotsSession({ refresh: input.refresh });
|
|
23617
|
+
const sentences = await SentenceStore.open();
|
|
23618
|
+
const renderer = createPageRenderer({ concurrency: 2 });
|
|
23322
23619
|
try {
|
|
23323
|
-
|
|
23620
|
+
return await crawl(input, robots, sentences, renderer, warnings);
|
|
23324
23621
|
} catch (error) {
|
|
23325
|
-
|
|
23326
|
-
|
|
23327
|
-
|
|
23328
|
-
|
|
23329
|
-
status: fetched.status,
|
|
23330
|
-
aliases: [input.targetUrl],
|
|
23331
|
-
verification: "failed",
|
|
23332
|
-
technicalSignals: [],
|
|
23333
|
-
fetchDiagnostics: fetched.diagnostics,
|
|
23334
|
-
warnings: [{
|
|
23335
|
-
stage: "target-extract",
|
|
23336
|
-
url: input.targetUrl,
|
|
23337
|
-
code: "extract-failed",
|
|
23338
|
-
message: errorMessage(error)
|
|
23339
|
-
}]
|
|
23340
|
-
};
|
|
23622
|
+
await sentences.close();
|
|
23623
|
+
throw error;
|
|
23624
|
+
} finally {
|
|
23625
|
+
await renderer.close();
|
|
23341
23626
|
}
|
|
23342
|
-
|
|
23343
|
-
|
|
23344
|
-
const
|
|
23345
|
-
|
|
23346
|
-
|
|
23347
|
-
|
|
23348
|
-
|
|
23349
|
-
|
|
23350
|
-
|
|
23351
|
-
|
|
23352
|
-
|
|
23353
|
-
|
|
23354
|
-
|
|
23355
|
-
|
|
23356
|
-
|
|
23357
|
-
|
|
23358
|
-
|
|
23359
|
-
|
|
23360
|
-
|
|
23361
|
-
|
|
23362
|
-
|
|
23363
|
-
|
|
23364
|
-
|
|
23365
|
-
|
|
23366
|
-
|
|
23367
|
-
|
|
23368
|
-
|
|
23369
|
-
|
|
23370
|
-
|
|
23371
|
-
|
|
23372
|
-
|
|
23373
|
-
|
|
23627
|
+
}
|
|
23628
|
+
async function crawl(input, robots, sentences, renderer, warnings) {
|
|
23629
|
+
const start = crawlUrl(input.startUrl) ?? input.startUrl;
|
|
23630
|
+
let origin = new URL(start).origin;
|
|
23631
|
+
const site = bareHost(new URL(start).host);
|
|
23632
|
+
const sameSite = (url) => {
|
|
23633
|
+
try {
|
|
23634
|
+
return bareHost(new URL(url).host) === site;
|
|
23635
|
+
} catch {
|
|
23636
|
+
return false;
|
|
23637
|
+
}
|
|
23638
|
+
};
|
|
23639
|
+
const urls = [];
|
|
23640
|
+
const urlIds = /* @__PURE__ */ new Map();
|
|
23641
|
+
const idOf = (url) => {
|
|
23642
|
+
let id = urlIds.get(url);
|
|
23643
|
+
if (id === void 0) {
|
|
23644
|
+
id = urls.length;
|
|
23645
|
+
urls.push(url);
|
|
23646
|
+
urlIds.set(url, id);
|
|
23647
|
+
}
|
|
23648
|
+
return id;
|
|
23649
|
+
};
|
|
23650
|
+
const aliases = /* @__PURE__ */ new Map();
|
|
23651
|
+
const pageOfUrl = /* @__PURE__ */ new Map();
|
|
23652
|
+
const pages = [];
|
|
23653
|
+
const index = new SiteIndex();
|
|
23654
|
+
const genericAnchors = [];
|
|
23655
|
+
const skipped = {};
|
|
23656
|
+
const skip = (reason) => {
|
|
23657
|
+
skipped[reason] = (skipped[reason] ?? 0) + 1;
|
|
23658
|
+
};
|
|
23659
|
+
const rootRobots = await robots.resolve(origin, start);
|
|
23660
|
+
const delay = await robots.crawlDelay(origin);
|
|
23661
|
+
const crawlDelaySeconds = delay === void 0 ? null : Math.min(delay, MAX_CRAWL_DELAY_SECONDS);
|
|
23662
|
+
const pageLimit = crawlDelaySeconds === null ? input.maxPages : Math.min(input.maxPages, Math.max(20, Math.floor(CRAWL_DELAY_BUDGET_SECONDS / crawlDelaySeconds)));
|
|
23663
|
+
if (pageLimit < input.maxPages) warnings.push(`robots.txt asks for ${crawlDelaySeconds} seconds between requests, so the crawl stopped at ${pageLimit} pages to finish in about ${CRAWL_DELAY_BUDGET_SECONDS / 60} minutes.`);
|
|
23664
|
+
const rate = crawlDelaySeconds === null ? {
|
|
23665
|
+
concurrency: FETCH_CONCURRENCY,
|
|
23666
|
+
intervalCap: FETCH_CONCURRENCY,
|
|
23667
|
+
intervalMs: 1e3
|
|
23668
|
+
} : {
|
|
23669
|
+
concurrency: 1,
|
|
23670
|
+
intervalCap: 1,
|
|
23671
|
+
intervalMs: Math.ceil(crawlDelaySeconds * 1e3)
|
|
23672
|
+
};
|
|
23673
|
+
const options = {
|
|
23674
|
+
js: "auto",
|
|
23675
|
+
refresh: input.refresh,
|
|
23676
|
+
respectRobots: true,
|
|
23677
|
+
robotsResolver: robots.resolve,
|
|
23678
|
+
renderer,
|
|
23679
|
+
rate
|
|
23680
|
+
};
|
|
23681
|
+
const stats = (extra = {}) => ({
|
|
23682
|
+
maxPages: input.maxPages,
|
|
23683
|
+
pageLimit,
|
|
23684
|
+
urlsFound: 0,
|
|
23685
|
+
pagesCrawled: pages.length,
|
|
23686
|
+
capReached: false,
|
|
23687
|
+
skipped,
|
|
23688
|
+
crawlDelaySeconds,
|
|
23689
|
+
blockedByRobots: false,
|
|
23690
|
+
redirectsChecked: 0,
|
|
23691
|
+
...extra
|
|
23692
|
+
});
|
|
23693
|
+
if (rootRobots.allowed === false) return {
|
|
23694
|
+
origin,
|
|
23695
|
+
method: "links",
|
|
23696
|
+
pages,
|
|
23697
|
+
urls,
|
|
23698
|
+
pageOfUrl,
|
|
23699
|
+
index,
|
|
23700
|
+
sentences,
|
|
23701
|
+
genericAnchors,
|
|
23702
|
+
stats: stats({ blockedByRobots: true }),
|
|
23703
|
+
warnings: [`robots.txt does not allow this crawler to read ${start}.`]
|
|
23704
|
+
};
|
|
23705
|
+
const declared = await robots.sitemapUrls(origin);
|
|
23706
|
+
const sitemap = await sitemapSeeds({
|
|
23707
|
+
url: origin,
|
|
23708
|
+
maxPages: pageLimit * 2,
|
|
23709
|
+
declaredSitemapUrls: declared,
|
|
23374
23710
|
warnings,
|
|
23375
|
-
|
|
23711
|
+
fetchSitemapUrls
|
|
23712
|
+
});
|
|
23713
|
+
const seeds = sitemap.urls.map((url) => crawlUrl(url)).filter((url) => Boolean(url) && sameSite(url) && !ASSET.test(new URL(url).pathname));
|
|
23714
|
+
const method = seeds.length ? "sitemap" : "links";
|
|
23715
|
+
const queue = [];
|
|
23716
|
+
const queued = /* @__PURE__ */ new Set();
|
|
23717
|
+
const maxQueue = pageLimit * 5;
|
|
23718
|
+
const enqueue = (url) => {
|
|
23719
|
+
if (!url || queued.has(url) || queued.size >= maxQueue) return;
|
|
23720
|
+
if (!sameSite(url) || ASSET.test(new URL(url).pathname)) return;
|
|
23721
|
+
queued.add(url);
|
|
23722
|
+
idOf(url);
|
|
23723
|
+
queue.push(url);
|
|
23724
|
+
};
|
|
23725
|
+
enqueue(input.firstUrl ? crawlUrl(input.firstUrl) : void 0);
|
|
23726
|
+
if (method === "sitemap") for (const url of seeds) enqueue(url);
|
|
23727
|
+
else enqueue(start);
|
|
23728
|
+
let overflow = 0;
|
|
23729
|
+
const crawlOne = async (url) => {
|
|
23730
|
+
let fetched;
|
|
23731
|
+
try {
|
|
23732
|
+
fetched = await fetchPage(url, options);
|
|
23733
|
+
} catch (error) {
|
|
23734
|
+
skip(error instanceof Error && error.name === "RobotsAccessError" ? "robots" : "fetch-failed");
|
|
23735
|
+
return;
|
|
23736
|
+
}
|
|
23737
|
+
const finalUrl = crawlUrl(fetched.finalUrl) ?? url;
|
|
23738
|
+
if (finalUrl !== url) aliases.set(idOf(url), idOf(finalUrl));
|
|
23739
|
+
if (!sameSite(finalUrl)) return skip("off-site");
|
|
23740
|
+
if (fetched.status < 200 || fetched.status > 299) return skip("http-error");
|
|
23741
|
+
if (!/html/i.test(fetched.headers["content-type"] ?? "text/html")) return skip("not-html");
|
|
23742
|
+
let page;
|
|
23743
|
+
try {
|
|
23744
|
+
page = readPage(fetched, finalUrl);
|
|
23745
|
+
} catch {
|
|
23746
|
+
return skip("fetch-failed");
|
|
23747
|
+
}
|
|
23748
|
+
const canonical = page.canonical ? crawlUrl(page.canonical, finalUrl) : void 0;
|
|
23749
|
+
if (canonical && sameSite(canonical) && normalizePageUrl(canonical) !== normalizePageUrl(finalUrl)) {
|
|
23750
|
+
aliases.set(idOf(finalUrl), idOf(canonical));
|
|
23751
|
+
enqueue(canonical);
|
|
23752
|
+
return skip("canonicalised");
|
|
23753
|
+
}
|
|
23754
|
+
const finalId = idOf(finalUrl);
|
|
23755
|
+
if (pageOfUrl.has(finalId)) return skip("duplicate");
|
|
23756
|
+
if (pages.length >= pageLimit) {
|
|
23757
|
+
overflow += 1;
|
|
23758
|
+
return;
|
|
23759
|
+
}
|
|
23760
|
+
const text = placementPage({
|
|
23376
23761
|
...page,
|
|
23377
|
-
url:
|
|
23762
|
+
url: finalUrl,
|
|
23378
23763
|
html: fetched.html
|
|
23379
|
-
})
|
|
23764
|
+
});
|
|
23765
|
+
const contextual = /* @__PURE__ */ new Set();
|
|
23766
|
+
const other = /* @__PURE__ */ new Set();
|
|
23767
|
+
const id = pages.length;
|
|
23768
|
+
const hasMain = /<(main|article)[\s>]|role=["']main["']/i.test(fetched.html);
|
|
23769
|
+
for (const link of page.links) {
|
|
23770
|
+
const href = crawlUrl(link.href, finalUrl);
|
|
23771
|
+
if (!href || !sameSite(href) || ASSET.test(new URL(href).pathname)) continue;
|
|
23772
|
+
const hrefId = idOf(href);
|
|
23773
|
+
if (link.location === "main-content" || !hasMain && link.location === "other") {
|
|
23774
|
+
contextual.add(hrefId);
|
|
23775
|
+
if (GENERIC_ANCHORS.test(link.text.trim()) && genericAnchors.length < GENERIC_ANCHOR_LIMIT) genericAnchors.push({
|
|
23776
|
+
source: id,
|
|
23777
|
+
href,
|
|
23778
|
+
text: link.text.trim()
|
|
23779
|
+
});
|
|
23780
|
+
} else other.add(hrefId);
|
|
23781
|
+
}
|
|
23782
|
+
const record = {
|
|
23783
|
+
url: finalUrl,
|
|
23784
|
+
title: text.title,
|
|
23785
|
+
h1: text.h1,
|
|
23786
|
+
description: text.description,
|
|
23787
|
+
headings: text.headings,
|
|
23788
|
+
excerpt: excerptOf(text.sentences),
|
|
23789
|
+
words: text.sentences.reduce((sum, sentence) => sum + sentence.text.split(/\s+/).length, 0),
|
|
23790
|
+
lang: page.lang?.toLowerCase().split(/[-_]/)[0] || null,
|
|
23791
|
+
noindex: page.noindex,
|
|
23792
|
+
contextual: [...contextual],
|
|
23793
|
+
other: [...other],
|
|
23794
|
+
text: {
|
|
23795
|
+
offset: 0,
|
|
23796
|
+
length: 0
|
|
23797
|
+
}
|
|
23798
|
+
};
|
|
23799
|
+
pageOfUrl.set(finalId, id);
|
|
23800
|
+
pages.push(record);
|
|
23801
|
+
index.addPage(termCounts(text));
|
|
23802
|
+
record.text = await sentences.write(linkableSentences(text.sentences));
|
|
23803
|
+
if (method === "links" && !page.nofollow) for (const link of page.links) enqueue(crawlUrl(link.href, finalUrl));
|
|
23804
|
+
if (pages.length % 50 === 0) input.progress?.(`Crawled ${pages.length} pages of up to ${pageLimit}.`);
|
|
23805
|
+
};
|
|
23806
|
+
const worker = async () => {
|
|
23807
|
+
while (pages.length < pageLimit) {
|
|
23808
|
+
const url = queue.shift();
|
|
23809
|
+
if (!url) {
|
|
23810
|
+
if (active === 0) return;
|
|
23811
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
23812
|
+
continue;
|
|
23813
|
+
}
|
|
23814
|
+
active += 1;
|
|
23815
|
+
try {
|
|
23816
|
+
await crawlOne(url);
|
|
23817
|
+
} finally {
|
|
23818
|
+
active -= 1;
|
|
23819
|
+
}
|
|
23820
|
+
}
|
|
23380
23821
|
};
|
|
23381
|
-
|
|
23382
|
-
|
|
23383
|
-
|
|
23384
|
-
|
|
23385
|
-
|
|
23386
|
-
|
|
23387
|
-
|
|
23388
|
-
|
|
23389
|
-
|
|
23390
|
-
|
|
23391
|
-
|
|
23392
|
-
|
|
23822
|
+
let active = 0;
|
|
23823
|
+
if (method === "links" && queue[0] === start) {
|
|
23824
|
+
queue.shift();
|
|
23825
|
+
active += 1;
|
|
23826
|
+
await crawlOne(start).finally(() => {
|
|
23827
|
+
active -= 1;
|
|
23828
|
+
});
|
|
23829
|
+
const first = pages[0];
|
|
23830
|
+
if (first) origin = new URL(first.url).origin;
|
|
23831
|
+
}
|
|
23832
|
+
await Promise.all(Array.from({ length: rate.concurrency ?? 1 }, () => worker()));
|
|
23833
|
+
const capReached = pages.length >= pageLimit && (queue.length > 0 || overflow > 0);
|
|
23834
|
+
const linkedCounts = /* @__PURE__ */ new Map();
|
|
23835
|
+
for (const page of pages) for (const id of [...page.contextual, ...page.other]) if (!pageOfUrl.has(id) && !aliases.has(id)) linkedCounts.set(id, (linkedCounts.get(id) ?? 0) + 1);
|
|
23836
|
+
const toCheck = [...linkedCounts].sort((left, right) => right[1] - left[1] || left[0] - right[0]).slice(0, REDIRECT_CHECKS);
|
|
23837
|
+
let redirectsChecked = 0;
|
|
23838
|
+
await Promise.all(Array.from({ length: rate.concurrency ?? 1 }, async () => {
|
|
23839
|
+
while (toCheck.length) {
|
|
23840
|
+
const next = toCheck.shift();
|
|
23841
|
+
if (!next) break;
|
|
23842
|
+
const url = urls[next[0]];
|
|
23843
|
+
try {
|
|
23844
|
+
const fetched = await fetchPage(url, {
|
|
23845
|
+
...options,
|
|
23846
|
+
js: "off"
|
|
23847
|
+
});
|
|
23848
|
+
redirectsChecked += 1;
|
|
23849
|
+
const finalUrl = crawlUrl(fetched.finalUrl);
|
|
23850
|
+
if (finalUrl && finalUrl !== url) aliases.set(next[0], idOf(finalUrl));
|
|
23851
|
+
} catch {}
|
|
23393
23852
|
}
|
|
23853
|
+
}));
|
|
23854
|
+
for (let id = 0; id < urls.length; id += 1) {
|
|
23855
|
+
if (pageOfUrl.has(id)) continue;
|
|
23856
|
+
const seen = /* @__PURE__ */ new Set();
|
|
23857
|
+
let current = id;
|
|
23858
|
+
while (current !== void 0 && !seen.has(current)) {
|
|
23859
|
+
seen.add(current);
|
|
23860
|
+
const page = pageOfUrl.get(current);
|
|
23861
|
+
if (page !== void 0) {
|
|
23862
|
+
pageOfUrl.set(id, page);
|
|
23863
|
+
break;
|
|
23864
|
+
}
|
|
23865
|
+
current = aliases.get(current);
|
|
23866
|
+
}
|
|
23867
|
+
}
|
|
23868
|
+
index.finish();
|
|
23869
|
+
return {
|
|
23870
|
+
origin,
|
|
23871
|
+
method,
|
|
23872
|
+
pages,
|
|
23873
|
+
urls,
|
|
23874
|
+
pageOfUrl,
|
|
23875
|
+
index,
|
|
23876
|
+
sentences,
|
|
23877
|
+
genericAnchors,
|
|
23878
|
+
stats: stats({
|
|
23879
|
+
urlsFound: Math.max(queued.size, sitemap.discovery.urlsReturned),
|
|
23880
|
+
capReached,
|
|
23881
|
+
redirectsChecked
|
|
23882
|
+
}),
|
|
23883
|
+
warnings
|
|
23394
23884
|
};
|
|
23395
23885
|
}
|
|
23396
|
-
|
|
23397
|
-
|
|
23398
|
-
|
|
23399
|
-
|
|
23400
|
-
|
|
23401
|
-
|
|
23402
|
-
|
|
23403
|
-
|
|
23404
|
-
|
|
23405
|
-
|
|
23406
|
-
|
|
23407
|
-
|
|
23886
|
+
//#endregion
|
|
23887
|
+
//#region packages/core/src/analyze/internal-links-judge.ts
|
|
23888
|
+
const PAGE_TYPES = {
|
|
23889
|
+
pillar_guide: "A long, broad guide that covers a whole topic and would naturally link out to many narrower pages",
|
|
23890
|
+
support_article: "A focused article, post, tutorial, or explainer about one narrower subject",
|
|
23891
|
+
product_or_service: "Sells or describes a product, service, plan, pricing, demo, or signup: a page that makes money",
|
|
23892
|
+
comparison_or_review: "Compares options, reviews one, or lists the best of something to help a reader choose",
|
|
23893
|
+
glossary_or_definition: "A short page that defines one term or answers one 'what is' question",
|
|
23894
|
+
tool_or_calculator: "An interactive tool, calculator, generator, or checker the reader uses on the page",
|
|
23895
|
+
news_or_update: "A dated announcement, release note, changelog entry, or news item that will go stale",
|
|
23896
|
+
category_hub: "A listing, archive, tag, category, or index page that mostly lists other pages",
|
|
23897
|
+
home: "The site's home page",
|
|
23898
|
+
about_or_company: "About, team, contact, press, careers, or other company information",
|
|
23899
|
+
legal_or_utility: "Privacy, terms, cookies, login, search, 404, thank-you, or other utility pages readers do not seek out",
|
|
23900
|
+
other: null
|
|
23901
|
+
};
|
|
23902
|
+
const SEARCH_INTENTS = {
|
|
23903
|
+
informational: "The reader wants to learn or understand something",
|
|
23904
|
+
commercial: "The reader is comparing or evaluating options before buying",
|
|
23905
|
+
transactional: "The reader wants to buy, sign up, download, or complete an action now",
|
|
23906
|
+
navigational: "The reader wants to reach a specific page or part of the site"
|
|
23907
|
+
};
|
|
23908
|
+
const LINK_ROLES = {
|
|
23909
|
+
explains_concept: "The target explains a term or idea the source mentions",
|
|
23910
|
+
deeper_detail: "The target goes deeper into a sub-topic the source only touches on",
|
|
23911
|
+
broader_guide: "The target is the broader guide or hub the source belongs under",
|
|
23912
|
+
next_step: "The target is what a reader should read or do after this part",
|
|
23913
|
+
product_or_service: "The target is the product or service that solves the problem discussed",
|
|
23914
|
+
comparison_or_alternative: "The target compares or offers an alternative to what the source discusses",
|
|
23915
|
+
weak_or_none: "No clear reason a reader would want the target here"
|
|
23916
|
+
};
|
|
23917
|
+
const THRESHOLDS = {
|
|
23918
|
+
possible: .5,
|
|
23919
|
+
strong: .75
|
|
23920
|
+
};
|
|
23921
|
+
const FLOORS = {
|
|
23922
|
+
minSentenceMatch: .1,
|
|
23923
|
+
minSimilarityAlone: .15
|
|
23924
|
+
};
|
|
23925
|
+
function targetWeight(type) {
|
|
23926
|
+
if (type === "home" || type === "legal_or_utility" || type === "about_or_company" || type === "category_hub") return 0;
|
|
23927
|
+
return type === "news_or_update" ? .6 : 1;
|
|
23928
|
+
}
|
|
23929
|
+
function canBeSource(type) {
|
|
23930
|
+
return type !== "home" && type !== "category_hub" && type !== "legal_or_utility";
|
|
23931
|
+
}
|
|
23932
|
+
function classifyRequest(pages) {
|
|
23933
|
+
const questions = {};
|
|
23934
|
+
pages.forEach((_, index) => {
|
|
23935
|
+
questions[`type_${index}`] = {
|
|
23936
|
+
type: "choice",
|
|
23937
|
+
instructions: `What kind of page is \`pages[${index}]\`?`,
|
|
23938
|
+
criteria: PAGE_TYPES
|
|
23408
23939
|
};
|
|
23409
|
-
|
|
23410
|
-
|
|
23411
|
-
|
|
23412
|
-
|
|
23413
|
-
js: "auto",
|
|
23414
|
-
refresh: input.refresh
|
|
23415
|
-
});
|
|
23416
|
-
} catch (error) {
|
|
23417
|
-
return {
|
|
23418
|
-
exclusion: "failed",
|
|
23419
|
-
warnings: [{
|
|
23420
|
-
stage: "source-fetch",
|
|
23421
|
-
url: input.candidate.sourceUrl,
|
|
23422
|
-
code: "fetch-failed",
|
|
23423
|
-
message: errorMessage(error)
|
|
23424
|
-
}]
|
|
23940
|
+
questions[`intent_${index}`] = {
|
|
23941
|
+
type: "choice",
|
|
23942
|
+
instructions: `What does a visitor arriving at \`pages[${index}]\` from a search engine most likely want?`,
|
|
23943
|
+
criteria: SEARCH_INTENTS
|
|
23425
23944
|
};
|
|
23426
|
-
}
|
|
23427
|
-
|
|
23428
|
-
|
|
23429
|
-
|
|
23430
|
-
|
|
23431
|
-
|
|
23432
|
-
|
|
23433
|
-
|
|
23434
|
-
|
|
23435
|
-
|
|
23436
|
-
|
|
23437
|
-
|
|
23438
|
-
|
|
23945
|
+
});
|
|
23946
|
+
return {
|
|
23947
|
+
state: { pages: pages.map((page) => ({
|
|
23948
|
+
url: page.url,
|
|
23949
|
+
title: page.title,
|
|
23950
|
+
description: page.description,
|
|
23951
|
+
h1: page.h1,
|
|
23952
|
+
headings: page.headings,
|
|
23953
|
+
excerpt: page.excerpt,
|
|
23954
|
+
word_count: page.words
|
|
23955
|
+
})) },
|
|
23956
|
+
questions
|
|
23957
|
+
};
|
|
23958
|
+
}
|
|
23959
|
+
function choiceOf(answer) {
|
|
23960
|
+
return answer?.type === "choice" ? answer.choice : null;
|
|
23961
|
+
}
|
|
23962
|
+
function readClassification(answers, index) {
|
|
23963
|
+
const type = choiceOf(answers[`type_${index}`]);
|
|
23964
|
+
const intent = choiceOf(answers[`intent_${index}`]);
|
|
23965
|
+
return {
|
|
23966
|
+
type: type && Object.hasOwn(PAGE_TYPES, type) ? type : null,
|
|
23967
|
+
intent: intent && Object.hasOwn(SEARCH_INTENTS, intent) ? intent : null
|
|
23968
|
+
};
|
|
23969
|
+
}
|
|
23970
|
+
function judgeRequest(source, targets) {
|
|
23971
|
+
const sentences = /* @__PURE__ */ new Map();
|
|
23972
|
+
for (const target of targets) for (const sentence of target.placement.sentences) sentences.set(sentence.id, {
|
|
23973
|
+
id: sentence.id,
|
|
23974
|
+
under_heading: sentence.section,
|
|
23975
|
+
text: sentence.text
|
|
23976
|
+
});
|
|
23977
|
+
const questions = {};
|
|
23978
|
+
targets.forEach((target, index) => {
|
|
23979
|
+
const ref = `\`targets[${index}]\``;
|
|
23980
|
+
questions[`link_${index}`] = {
|
|
23981
|
+
type: "noul",
|
|
23982
|
+
instructions: `Should the page \`source\` add a link from within its body text to the page ${ref}?`,
|
|
23983
|
+
criteria: {
|
|
23984
|
+
true: "A reader of the source would gain something by reading the target here; the two pages cover related subjects and the link would feel natural and helpful",
|
|
23985
|
+
false: "The pages are only loosely related, the target adds nothing for this reader, or the link would feel forced or spammy"
|
|
23986
|
+
}
|
|
23987
|
+
};
|
|
23988
|
+
questions[`where_${index}`] = {
|
|
23989
|
+
type: "choice",
|
|
23990
|
+
instructions: `Which sentence in \`source.sentences\` is the most natural place to add a link to ${ref}? Choose "none" if no listed sentence leads a reader towards that page.`,
|
|
23991
|
+
criteria: {
|
|
23992
|
+
...Object.fromEntries(target.placement.sentences.map((sentence) => [sentence.id, sentence.text])),
|
|
23993
|
+
none: "No listed sentence is a natural place for this link"
|
|
23994
|
+
}
|
|
23995
|
+
};
|
|
23996
|
+
if (target.placement.anchors.length) questions[`anchor_${index}`] = {
|
|
23997
|
+
type: "choice",
|
|
23998
|
+
instructions: `Which phrase should become the clickable link text for a link from \`source\` to ${ref}? Prefer a phrase that tells the reader what the target page is about and reads naturally as a link.`,
|
|
23999
|
+
criteria: {
|
|
24000
|
+
...Object.fromEntries(target.placement.anchors.map((anchor, at) => [`A${at + 1}`, {
|
|
24001
|
+
text: anchor.text,
|
|
24002
|
+
in_sentence: anchor.sentence
|
|
24003
|
+
}])),
|
|
24004
|
+
none: "None of these phrases would make a good link"
|
|
24005
|
+
}
|
|
24006
|
+
};
|
|
24007
|
+
questions[`role_${index}`] = {
|
|
24008
|
+
type: "choice",
|
|
24009
|
+
instructions: `If \`source\` linked to ${ref}, what would the link do for the reader?`,
|
|
24010
|
+
criteria: LINK_ROLES
|
|
23439
24011
|
};
|
|
23440
|
-
}
|
|
23441
|
-
const warnings = page.warnings.map((message) => ({
|
|
23442
|
-
stage: "source-extract",
|
|
23443
|
-
url: input.candidate.sourceUrl,
|
|
23444
|
-
code: "extractor-warning",
|
|
23445
|
-
message
|
|
23446
|
-
}));
|
|
23447
|
-
if (input.target.aliases.some((alias) => sameObservedUrl(page.finalUrl, alias))) return {
|
|
23448
|
-
exclusion: "self-alias",
|
|
23449
|
-
warnings
|
|
23450
|
-
};
|
|
23451
|
-
const technicalSignals = pageTechnicalSignals({
|
|
23452
|
-
url: input.candidate.sourceUrl,
|
|
23453
|
-
page,
|
|
23454
|
-
fetchDiagnostics: fetched.diagnostics,
|
|
23455
|
-
httpStatus: fetched.status
|
|
23456
24012
|
});
|
|
23457
|
-
if (technicalSignals.length) return {
|
|
23458
|
-
exclusion: "technical",
|
|
23459
|
-
warnings
|
|
23460
|
-
};
|
|
23461
|
-
const observed = linksToAliases(page, input.target.aliases).map((link) => ({
|
|
23462
|
-
href: link.href,
|
|
23463
|
-
text: link.text,
|
|
23464
|
-
rel: link.rel,
|
|
23465
|
-
location: link.location
|
|
23466
|
-
}));
|
|
23467
|
-
const contextual = observed.filter((link) => link.location === "main-content");
|
|
23468
|
-
if (contextual.some((link) => sameObservedUrl(link.href, input.target.preferredUrl))) return {
|
|
23469
|
-
exclusion: "existing-link",
|
|
23470
|
-
warnings
|
|
23471
|
-
};
|
|
23472
|
-
const aliasContextual = contextual.length > 0;
|
|
23473
|
-
const linkStatus = aliasContextual ? "alias-contextual" : observed.length ? "non-contextual-only" : "missing";
|
|
23474
|
-
const confidence = input.candidate.bestMatchKind === "exact-query" && !page.contentExtraction.fallback ? "medium" : "low";
|
|
23475
|
-
const actionType = aliasContextual ? "review-alias-link" : "review-contextual-link";
|
|
23476
|
-
const evidenceRef = aliasContextual ? `A main-content link points to a non-preferred target alias; the preferred target is ${input.target.preferredUrl}.` : `${input.candidate.sourceUrl} matched ${input.candidate.matchedQueries} retained GSC query row${input.candidate.matchedQueries === 1 ? "" : "s"}, and no main-content link to the preferred target was observed in the fetched HTML.`;
|
|
23477
|
-
const action = aliasContextual ? `Review the existing contextual link and update it to the preferred target ${input.target.preferredUrl} if the redirect or old alias is not intentional.` : `Review whether this source page and ${input.target.preferredUrl} serve complementary intent. If the link would help a reader, add a natural main-content link without forcing exact-match anchor text.`;
|
|
23478
24013
|
return {
|
|
23479
|
-
|
|
23480
|
-
|
|
23481
|
-
|
|
23482
|
-
|
|
23483
|
-
|
|
23484
|
-
|
|
23485
|
-
pageWarnings: page.warnings,
|
|
23486
|
-
actionType,
|
|
23487
|
-
linkEvidence: {
|
|
23488
|
-
status: linkStatus,
|
|
23489
|
-
observedCount: observed.length,
|
|
23490
|
-
observedLimit: 20,
|
|
23491
|
-
limitedCount: Math.max(0, observed.length - 20),
|
|
23492
|
-
observed: observed.slice(0, 20)
|
|
24014
|
+
state: {
|
|
24015
|
+
source: {
|
|
24016
|
+
url: source.url,
|
|
24017
|
+
title: source.title,
|
|
24018
|
+
page_type: source.type ?? "unknown",
|
|
24019
|
+
sentences: [...sentences.values()].sort((left, right) => Number(left.id.slice(1)) - Number(right.id.slice(1)))
|
|
23493
24020
|
},
|
|
23494
|
-
|
|
23495
|
-
|
|
23496
|
-
|
|
23497
|
-
|
|
23498
|
-
|
|
23499
|
-
|
|
23500
|
-
|
|
23501
|
-
|
|
23502
|
-
}
|
|
24021
|
+
targets: targets.map((target) => ({
|
|
24022
|
+
url: target.url,
|
|
24023
|
+
title: target.title,
|
|
24024
|
+
description: target.description,
|
|
24025
|
+
h1: target.h1,
|
|
24026
|
+
page_type: target.type ?? "unknown",
|
|
24027
|
+
search_intent: target.intent ?? "unknown",
|
|
24028
|
+
opening: target.opening
|
|
24029
|
+
}))
|
|
23503
24030
|
},
|
|
23504
|
-
|
|
23505
|
-
|
|
23506
|
-
|
|
23507
|
-
|
|
23508
|
-
|
|
23509
|
-
|
|
24031
|
+
questions
|
|
24032
|
+
};
|
|
24033
|
+
}
|
|
24034
|
+
function floor(sentence, similarity, probability) {
|
|
24035
|
+
if (sentence && sentence.match < FLOORS.minSentenceMatch) return {
|
|
24036
|
+
state: "held-back",
|
|
24037
|
+
reason: "weak-sentence",
|
|
24038
|
+
detail: `the sentence has ${Math.round(sentence.match * 100)}% of the words that name the page; a link needs ${Math.round(FLOORS.minSentenceMatch * 100)}%`,
|
|
24039
|
+
probability
|
|
24040
|
+
};
|
|
24041
|
+
if (!sentence && similarity < FLOORS.minSimilarityAlone) return {
|
|
24042
|
+
state: "held-back",
|
|
24043
|
+
reason: "no-sentence",
|
|
24044
|
+
detail: `no sentence to place it in, and the pages are ${similarity.toFixed(2)} similar; a link without a sentence needs ${FLOORS.minSimilarityAlone}`,
|
|
24045
|
+
probability
|
|
24046
|
+
};
|
|
24047
|
+
return null;
|
|
24048
|
+
}
|
|
24049
|
+
function decideWithModel(placement, similarity, answers, index) {
|
|
24050
|
+
const link = answers[`link_${index}`];
|
|
24051
|
+
const role = choiceOf(answers[`role_${index}`]);
|
|
24052
|
+
if (link?.type !== "noul" || !role) return {
|
|
24053
|
+
state: "declined",
|
|
24054
|
+
reason: "no-answer",
|
|
24055
|
+
probability: null
|
|
24056
|
+
};
|
|
24057
|
+
const probability = Math.round(link.noul * 1e3) / 1e3;
|
|
24058
|
+
if (probability < THRESHOLDS.possible) return {
|
|
24059
|
+
state: "declined",
|
|
24060
|
+
reason: "below-threshold",
|
|
24061
|
+
probability
|
|
24062
|
+
};
|
|
24063
|
+
if (role === "weak_or_none" || !Object.hasOwn(LINK_ROLES, role)) return {
|
|
24064
|
+
state: "declined",
|
|
24065
|
+
reason: "weak-role",
|
|
24066
|
+
probability
|
|
24067
|
+
};
|
|
24068
|
+
const where = choiceOf(answers[`where_${index}`]);
|
|
24069
|
+
const sentence = placement.sentences.find((option) => option.id === where) ?? null;
|
|
24070
|
+
const held = floor(sentence, similarity, probability);
|
|
24071
|
+
if (held) return held;
|
|
24072
|
+
let anchor = null;
|
|
24073
|
+
const picked = answers[`anchor_${index}`];
|
|
24074
|
+
if (sentence && picked?.type === "choice" && picked.choice !== "none") {
|
|
24075
|
+
const inSentence = placement.anchors.map((option, at) => ({
|
|
24076
|
+
...option,
|
|
24077
|
+
key: `A${at + 1}`
|
|
24078
|
+
})).filter((option) => option.sentence === sentence.id);
|
|
24079
|
+
anchor = (inSentence.find((option) => option.key === picked.choice) ?? [...inSentence].sort((left, right) => (picked.probabilities[right.key] ?? 0) - (picked.probabilities[left.key] ?? 0) || left.key.localeCompare(right.key))[0])?.text ?? null;
|
|
24080
|
+
}
|
|
24081
|
+
return {
|
|
24082
|
+
state: "suggest",
|
|
24083
|
+
tier: probability >= THRESHOLDS.strong ? "strong" : "possible",
|
|
24084
|
+
probability,
|
|
24085
|
+
role,
|
|
24086
|
+
sentence: sentence ? {
|
|
24087
|
+
text: sentence.text,
|
|
24088
|
+
section: sentence.section,
|
|
24089
|
+
match: sentence.match
|
|
24090
|
+
} : null,
|
|
24091
|
+
anchor
|
|
24092
|
+
};
|
|
24093
|
+
}
|
|
24094
|
+
function decideWithoutModel(placement, similarity) {
|
|
24095
|
+
const sentence = placement.sentences[0] ?? null;
|
|
24096
|
+
const held = floor(sentence, similarity, null);
|
|
24097
|
+
if (held) return held;
|
|
24098
|
+
return {
|
|
24099
|
+
state: "suggest",
|
|
24100
|
+
tier: "heuristic",
|
|
24101
|
+
probability: null,
|
|
24102
|
+
role: null,
|
|
24103
|
+
sentence: sentence ? {
|
|
24104
|
+
text: sentence.text,
|
|
24105
|
+
section: sentence.section,
|
|
24106
|
+
match: sentence.match
|
|
24107
|
+
} : null,
|
|
24108
|
+
anchor: placement.anchors.find((option) => option.sentence === sentence?.id)?.text ?? null
|
|
23510
24109
|
};
|
|
23511
24110
|
}
|
|
23512
24111
|
//#endregion
|
|
24112
|
+
//#region packages/core/src/analyze/internal-links-report.ts
|
|
24113
|
+
const DAY$1 = 864e5;
|
|
24114
|
+
const ISO_DAY = /^\d{4}-\d{2}-\d{2}$/;
|
|
24115
|
+
function internalLinksReportRange(input, fallbackDays, now) {
|
|
24116
|
+
if (!input.startDate && !input.endDate) return {
|
|
24117
|
+
...finalGscDateRange(fallbackDays, now),
|
|
24118
|
+
days: fallbackDays
|
|
24119
|
+
};
|
|
24120
|
+
const { startDate, endDate } = input;
|
|
24121
|
+
if (!startDate || !endDate || !ISO_DAY.test(startDate) || !ISO_DAY.test(endDate) || startDate > endDate) throw new SeoError("INVALID_INPUT", "Send a start date and an end date together, in order, as YYYY-MM-DD.");
|
|
24122
|
+
return {
|
|
24123
|
+
startDate,
|
|
24124
|
+
endDate,
|
|
24125
|
+
days: Math.round((Date.parse(`${endDate}T00:00:00Z`) - Date.parse(`${startDate}T00:00:00Z`)) / DAY$1) + 1
|
|
24126
|
+
};
|
|
24127
|
+
}
|
|
24128
|
+
const plural$3 = (count, word) => `${count} ${word}${count === 1 ? "" : "s"}`;
|
|
24129
|
+
function internalLinksVerdict(report) {
|
|
24130
|
+
const { counts } = report;
|
|
24131
|
+
if (report.dataStatus === "blocked") return "robots.txt does not allow this crawler to read the site, so nothing was checked.";
|
|
24132
|
+
if (report.dataStatus === "empty") return "No HTML page could be crawled, so there is nothing to link.";
|
|
24133
|
+
if (report.dataStatus === "target-not-crawled") return "The target page could not be crawled as its own page (it failed, left the site, or names another page as canonical). Check it with seo page first.";
|
|
24134
|
+
if (report.dataStatus === "target-not-linkable") return `The target is ${report.target?.noindex ? "noindex" : `a ${report.target?.type?.replaceAll("_", " ")} page`}, so no links to it are suggested.`;
|
|
24135
|
+
const judged = report.judge.status === "used" || report.judge.status === "partial" ? `${counts.strong} strong and ${counts.possible} possible link${counts.strong + counts.possible === 1 ? "" : "s"} judged by ${report.judge.model}${counts.heuristic ? `, plus ${plural$3(counts.heuristic, "heuristic link")} where a request failed` : ""}` : `${plural$3(counts.heuristic, "heuristic link")} (no decision model)`;
|
|
24136
|
+
const scope = report.mode === "url" ? `to ${report.target?.url}` : `across ${plural$3(counts.pagesCrawled, "page")}`;
|
|
24137
|
+
const capped = report.crawl.capReached ? ` The crawl stopped at its limit of ${report.crawl.pageLimit} pages, so pages beyond it were not checked.` : "";
|
|
24138
|
+
return `${judged} ${scope}${counts.heldBack ? `; ${counts.heldBack} held back by the evidence floors` : ""}.${capped}`;
|
|
24139
|
+
}
|
|
24140
|
+
//#endregion
|
|
23513
24141
|
//#region packages/core/src/analyze/internal-links.ts
|
|
24142
|
+
const GSC_DAYS = 90;
|
|
23514
24143
|
const MAX_GSC_ROWS = 1e5;
|
|
23515
|
-
const
|
|
23516
|
-
const
|
|
23517
|
-
const
|
|
23518
|
-
const
|
|
23519
|
-
const
|
|
23520
|
-
|
|
23521
|
-
|
|
23522
|
-
|
|
23523
|
-
|
|
23524
|
-
|
|
23525
|
-
|
|
23526
|
-
|
|
23527
|
-
|
|
23528
|
-
const
|
|
23529
|
-
|
|
23530
|
-
|
|
23531
|
-
|
|
23532
|
-
|
|
23533
|
-
const
|
|
23534
|
-
|
|
23535
|
-
|
|
23536
|
-
|
|
23537
|
-
|
|
23538
|
-
|
|
24144
|
+
const MAX_PAGES_PER_QUERY = 25;
|
|
24145
|
+
const MAX_GSC_PAIRS = 5e5;
|
|
24146
|
+
const MIN_SIMILARITY = .08;
|
|
24147
|
+
const TARGETS_PER_SOURCE = 8;
|
|
24148
|
+
const EXTRA_GSC_TARGETS = 2;
|
|
24149
|
+
const URL_SOURCES_PLACED = 60;
|
|
24150
|
+
const URL_SOURCES_JUDGED = 24;
|
|
24151
|
+
const SOURCE_WINDOW = 64;
|
|
24152
|
+
const WEAK_INBOUND = 3;
|
|
24153
|
+
const EXAMPLE_LIMIT = 20;
|
|
24154
|
+
const LIST_LIMIT = 100;
|
|
24155
|
+
const OPENING_CHARS = 300;
|
|
24156
|
+
function siteStartUrl(site) {
|
|
24157
|
+
const value = site.trim();
|
|
24158
|
+
if (value.startsWith("sc-domain:")) return `https://${value.slice(10)}/`;
|
|
24159
|
+
return `${new URL(/^https?:\/\//i.test(value) ? value : `https://${value}`).origin}/`;
|
|
24160
|
+
}
|
|
24161
|
+
function round$1(value, places = 3) {
|
|
24162
|
+
const factor = 10 ** places;
|
|
24163
|
+
return Math.round(value * factor) / factor;
|
|
24164
|
+
}
|
|
24165
|
+
async function internalLinksReport(input, dependencies) {
|
|
24166
|
+
const deps = {
|
|
24167
|
+
crawl: crawlForLinks,
|
|
24168
|
+
now: () => /* @__PURE__ */ new Date(),
|
|
24169
|
+
...dependencies
|
|
24170
|
+
};
|
|
24171
|
+
const mode = input.targetUrl ? "url" : "site";
|
|
24172
|
+
if (mode === "url" && input.maxPages !== void 0) throw new SeoError("INVALID_INPUT", `--max-pages is for a site audit only. A page run crawls up to 500 pages.`);
|
|
24173
|
+
const maxPages = input.maxPages ?? 500;
|
|
24174
|
+
if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > 5e3) throw new SeoError("INVALID_INPUT", `--max-pages is a whole number from 1 to ${MAX_MAX_PAGES.toLocaleString("en-US")}.`);
|
|
24175
|
+
const targetUrl = input.targetUrl ? crawlUrl(input.targetUrl) : void 0;
|
|
24176
|
+
if (input.targetUrl && !targetUrl) throw new SeoError("INVALID_INPUT", "The target is an http or https URL.");
|
|
24177
|
+
const now = deps.now();
|
|
24178
|
+
const startUrl = targetUrl ? `${new URL(targetUrl).origin}/` : siteStartUrl(input.site);
|
|
24179
|
+
deps.progress?.(`Crawling ${startUrl} (up to ${maxPages} pages).`);
|
|
24180
|
+
const crawl = await deps.crawl({
|
|
24181
|
+
startUrl,
|
|
24182
|
+
maxPages,
|
|
24183
|
+
firstUrl: targetUrl,
|
|
23539
24184
|
refresh: input.refresh,
|
|
23540
|
-
|
|
23541
|
-
});
|
|
23542
|
-
const targetRequests = await Promise.all(target.aliases.map(async (alias) => {
|
|
23543
|
-
const result = await dependencies.searchAnalytics(input.site, {
|
|
23544
|
-
...range,
|
|
23545
|
-
dimensions: ["query", "page"],
|
|
23546
|
-
type: "web",
|
|
23547
|
-
dataState: "final",
|
|
23548
|
-
maxRows: MAX_GSC_ROWS,
|
|
23549
|
-
dimensionFilterGroups: [{
|
|
23550
|
-
groupType: "and",
|
|
23551
|
-
filters: [{
|
|
23552
|
-
dimension: "page",
|
|
23553
|
-
operator: "equals",
|
|
23554
|
-
expression: alias
|
|
23555
|
-
}]
|
|
23556
|
-
}]
|
|
23557
|
-
}, { refresh: input.refresh });
|
|
23558
|
-
ledger.addGsc(result.calls, result.rowsFetched);
|
|
23559
|
-
return {
|
|
23560
|
-
alias,
|
|
23561
|
-
result
|
|
23562
|
-
};
|
|
23563
|
-
}));
|
|
23564
|
-
const targetRows = targetRequests.flatMap(({ result }) => result.rows);
|
|
23565
|
-
const preliminary = analyzeInternalLinksFromRows({
|
|
23566
|
-
targetRows,
|
|
23567
|
-
sourceRows: [],
|
|
23568
|
-
site: input.site,
|
|
23569
|
-
targetAliases: target.aliases,
|
|
23570
|
-
minImpressions,
|
|
23571
|
-
brandTerms
|
|
24185
|
+
progress: deps.progress
|
|
23572
24186
|
});
|
|
23573
|
-
|
|
23574
|
-
|
|
23575
|
-
|
|
23576
|
-
|
|
23577
|
-
|
|
23578
|
-
|
|
23579
|
-
|
|
23580
|
-
}
|
|
23581
|
-
|
|
23582
|
-
|
|
23583
|
-
|
|
24187
|
+
try {
|
|
24188
|
+
return await analyse(input, deps, crawl, {
|
|
24189
|
+
mode,
|
|
24190
|
+
targetUrl,
|
|
24191
|
+
startUrl,
|
|
24192
|
+
now
|
|
24193
|
+
});
|
|
24194
|
+
} finally {
|
|
24195
|
+
await crawl.sentences.close();
|
|
24196
|
+
}
|
|
24197
|
+
}
|
|
24198
|
+
async function analyse(input, deps, crawl, context) {
|
|
24199
|
+
const { pages, index } = crawl;
|
|
24200
|
+
const count = pages.length;
|
|
24201
|
+
const warnings = [...crawl.warnings];
|
|
24202
|
+
const lookup = /* @__PURE__ */ new Map();
|
|
24203
|
+
crawl.urls.forEach((url, id) => {
|
|
24204
|
+
const page = crawl.pageOfUrl.get(id);
|
|
24205
|
+
if (page === void 0) return;
|
|
24206
|
+
lookup.set(url, page);
|
|
24207
|
+
if (!lookup.has(normalizePageUrl(url))) lookup.set(normalizePageUrl(url), page);
|
|
24208
|
+
});
|
|
24209
|
+
const pageAt = (url) => {
|
|
24210
|
+
const clean = crawlUrl(url);
|
|
24211
|
+
if (!clean) return void 0;
|
|
24212
|
+
return lookup.get(clean) ?? lookup.get(normalizePageUrl(clean));
|
|
24213
|
+
};
|
|
24214
|
+
const target = context.targetUrl !== void 0 ? pageAt(context.targetUrl) : void 0;
|
|
24215
|
+
const contextual = pages.map(() => /* @__PURE__ */ new Set());
|
|
24216
|
+
const anywhere = pages.map(() => /* @__PURE__ */ new Set());
|
|
24217
|
+
const redirectedLinks = [];
|
|
24218
|
+
pages.forEach((page, source) => {
|
|
24219
|
+
for (const id of page.contextual) {
|
|
24220
|
+
const to = crawl.pageOfUrl.get(id);
|
|
24221
|
+
if (to === void 0 || to === source) continue;
|
|
24222
|
+
contextual[source]?.add(to);
|
|
24223
|
+
anywhere[source]?.add(to);
|
|
24224
|
+
const via = crawl.urls[id];
|
|
24225
|
+
const final = pages[to]?.url;
|
|
24226
|
+
if (normalizePageUrl(via) !== normalizePageUrl(final) && redirectedLinks.length < LIST_LIMIT) redirectedLinks.push({
|
|
24227
|
+
source: page.url,
|
|
24228
|
+
via,
|
|
24229
|
+
target: final
|
|
24230
|
+
});
|
|
24231
|
+
}
|
|
24232
|
+
for (const id of page.other) {
|
|
24233
|
+
const to = crawl.pageOfUrl.get(id);
|
|
24234
|
+
if (to !== void 0 && to !== source) anywhere[source]?.add(to);
|
|
24235
|
+
}
|
|
24236
|
+
});
|
|
24237
|
+
const inbound = new Array(count).fill(0);
|
|
24238
|
+
const inboundAnywhere = new Array(count).fill(0);
|
|
24239
|
+
contextual.forEach((set) => {
|
|
24240
|
+
for (const to of set) inbound[to] = (inbound[to] ?? 0) + 1;
|
|
24241
|
+
});
|
|
24242
|
+
anywhere.forEach((set) => {
|
|
24243
|
+
for (const to of set) inboundAnywhere[to] = (inboundAnywhere[to] ?? 0) + 1;
|
|
24244
|
+
});
|
|
24245
|
+
const existingContextualLinks = contextual.reduce((sum, set) => sum + set.size, 0);
|
|
24246
|
+
const range = internalLinksReportRange(input, GSC_DAYS, context.now);
|
|
24247
|
+
const sharedQueries = /* @__PURE__ */ new Map();
|
|
24248
|
+
const pairKey = (source, target) => source * count + target;
|
|
24249
|
+
let searchConsole = {
|
|
24250
|
+
status: "not-available",
|
|
24251
|
+
property: null,
|
|
24252
|
+
range: null,
|
|
24253
|
+
rows: 0,
|
|
24254
|
+
possiblyTruncated: false,
|
|
24255
|
+
pairsWithSharedQueries: 0,
|
|
24256
|
+
note: "unavailable" in deps.searchConsole ? deps.searchConsole.unavailable : null
|
|
24257
|
+
};
|
|
24258
|
+
if ("property" in deps.searchConsole && count > 1) {
|
|
24259
|
+
const { startDate, endDate } = range;
|
|
24260
|
+
try {
|
|
24261
|
+
deps.progress?.("Reading Search Console queries.");
|
|
24262
|
+
const result = await deps.searchConsole.query({
|
|
24263
|
+
startDate,
|
|
24264
|
+
endDate,
|
|
24265
|
+
dimensions: ["query", "page"],
|
|
24266
|
+
type: "web",
|
|
24267
|
+
dataState: "final",
|
|
24268
|
+
maxRows: MAX_GSC_ROWS
|
|
24269
|
+
});
|
|
24270
|
+
const brandTerms = deriveBrandTerms({ siteUrl: crawl.origin });
|
|
24271
|
+
const byQuery = /* @__PURE__ */ new Map();
|
|
24272
|
+
for (const row of result.rows) {
|
|
24273
|
+
const [query, url] = row.keys;
|
|
24274
|
+
if (!query || !url || row.impressions < 1) continue;
|
|
24275
|
+
if (isBrandQuery(query, brandTerms)) continue;
|
|
24276
|
+
const page = pageAt(url);
|
|
24277
|
+
if (page === void 0) continue;
|
|
24278
|
+
let set = byQuery.get(query);
|
|
24279
|
+
if (!set) {
|
|
24280
|
+
set = /* @__PURE__ */ new Set();
|
|
24281
|
+
byQuery.set(query, set);
|
|
24282
|
+
}
|
|
24283
|
+
set.add(page);
|
|
24284
|
+
}
|
|
24285
|
+
let capped = false;
|
|
24286
|
+
for (const set of byQuery.values()) {
|
|
24287
|
+
if (set.size < 2 || set.size > MAX_PAGES_PER_QUERY) continue;
|
|
24288
|
+
const list = [...set];
|
|
24289
|
+
for (const source of list) for (const to of list) {
|
|
24290
|
+
if (source === to) continue;
|
|
24291
|
+
if (target !== void 0 && to !== target) continue;
|
|
24292
|
+
const key = pairKey(source, to);
|
|
24293
|
+
if (!sharedQueries.has(key) && sharedQueries.size >= MAX_GSC_PAIRS) {
|
|
24294
|
+
capped = true;
|
|
24295
|
+
continue;
|
|
24296
|
+
}
|
|
24297
|
+
sharedQueries.set(key, (sharedQueries.get(key) ?? 0) + 1);
|
|
24298
|
+
}
|
|
24299
|
+
}
|
|
24300
|
+
searchConsole = {
|
|
24301
|
+
status: "used",
|
|
24302
|
+
property: deps.searchConsole.property,
|
|
24303
|
+
range: {
|
|
24304
|
+
startDate,
|
|
24305
|
+
endDate
|
|
24306
|
+
},
|
|
24307
|
+
rows: result.rowsFetched,
|
|
24308
|
+
possiblyTruncated: result.rowsFetched >= MAX_GSC_ROWS,
|
|
24309
|
+
pairsWithSharedQueries: sharedQueries.size,
|
|
24310
|
+
note: capped ? `Shared queries were counted for the first ${MAX_GSC_PAIRS.toLocaleString("en-US")} page pairs.` : null
|
|
24311
|
+
};
|
|
24312
|
+
} catch (error) {
|
|
24313
|
+
searchConsole = {
|
|
24314
|
+
...searchConsole,
|
|
24315
|
+
status: "failed",
|
|
24316
|
+
property: deps.searchConsole.property,
|
|
24317
|
+
note: error instanceof Error ? error.message : String(error)
|
|
24318
|
+
};
|
|
24319
|
+
}
|
|
24320
|
+
}
|
|
24321
|
+
const runner = "model" in deps.decisionModel ? new DecisionRunner(deps.decisionModel.model) : null;
|
|
24322
|
+
const types = pages.map((page) => new URL(page.url).pathname === "/" ? "home" : null);
|
|
24323
|
+
const intents = pages.map(() => null);
|
|
24324
|
+
let pagesClassified = 0;
|
|
24325
|
+
const classify = async (ids) => {
|
|
24326
|
+
if (!runner) return;
|
|
24327
|
+
const batches = [];
|
|
24328
|
+
for (let at = 0; at < ids.length; at += 5) batches.push(ids.slice(at, at + 5));
|
|
24329
|
+
deps.progress?.(`Classifying ${ids.length} pages in ${batches.length} requests.`);
|
|
24330
|
+
await Promise.all(batches.map(async (batch) => {
|
|
24331
|
+
const response = await runner.ask(classifyRequest(batch.map((id) => pages[id])));
|
|
24332
|
+
if (!response) return;
|
|
24333
|
+
batch.forEach((id, at) => {
|
|
24334
|
+
const read = readClassification(response.answers, at);
|
|
24335
|
+
if (read.type) types[id] = read.type;
|
|
24336
|
+
intents[id] = read.intent;
|
|
24337
|
+
pagesClassified += 1;
|
|
24338
|
+
});
|
|
24339
|
+
}));
|
|
23584
24340
|
};
|
|
23585
|
-
|
|
23586
|
-
|
|
23587
|
-
|
|
23588
|
-
|
|
23589
|
-
|
|
23590
|
-
|
|
23591
|
-
|
|
23592
|
-
|
|
23593
|
-
}
|
|
23594
|
-
const
|
|
23595
|
-
|
|
23596
|
-
|
|
23597
|
-
|
|
23598
|
-
|
|
23599
|
-
|
|
23600
|
-
|
|
23601
|
-
|
|
23602
|
-
|
|
23603
|
-
|
|
23604
|
-
|
|
23605
|
-
|
|
23606
|
-
|
|
23607
|
-
|
|
24341
|
+
const linkable = (source, to) => {
|
|
24342
|
+
const from = pages[source];
|
|
24343
|
+
const page = pages[to];
|
|
24344
|
+
if (!from || !page || source === to) return false;
|
|
24345
|
+
if (page.noindex || targetWeight(types[to] ?? null) === 0) return false;
|
|
24346
|
+
if (contextual[source]?.has(to)) return false;
|
|
24347
|
+
if (from.lang && page.lang && from.lang !== page.lang) return false;
|
|
24348
|
+
return canBeSource(types[source] ?? null);
|
|
24349
|
+
};
|
|
24350
|
+
const adjusted = (to, raw) => raw * ((inbound[to] ?? 0) < WEAK_INBOUND ? 1.25 : 1) * targetWeight(types[to] ?? null);
|
|
24351
|
+
let status = "complete";
|
|
24352
|
+
const candidatesBySource = /* @__PURE__ */ new Map();
|
|
24353
|
+
if (crawl.stats.blockedByRobots) status = "blocked";
|
|
24354
|
+
else if (count === 0) status = "empty";
|
|
24355
|
+
else if (context.mode === "url" && target === void 0) status = "target-not-crawled";
|
|
24356
|
+
if (status === "complete" && context.mode === "site") {
|
|
24357
|
+
await classify(pages.map((_, id) => id));
|
|
24358
|
+
const extra = /* @__PURE__ */ new Map();
|
|
24359
|
+
for (const key of sharedQueries.keys()) {
|
|
24360
|
+
const source = Math.floor(key / count);
|
|
24361
|
+
const list = extra.get(source) ?? [];
|
|
24362
|
+
list.push(key % count);
|
|
24363
|
+
extra.set(source, list);
|
|
24364
|
+
}
|
|
24365
|
+
for (let source = 0; source < count; source += 1) {
|
|
24366
|
+
if (!canBeSource(types[source] ?? null)) continue;
|
|
24367
|
+
const chosen = index.neighbours(source, MIN_SIMILARITY).filter(({ page }) => linkable(source, page)).map(({ page, score }) => ({
|
|
24368
|
+
source,
|
|
24369
|
+
target: page,
|
|
24370
|
+
raw: score,
|
|
24371
|
+
similarity: adjusted(page, score),
|
|
24372
|
+
sharedQueries: sharedQueries.get(pairKey(source, page)) ?? 0
|
|
24373
|
+
})).sort((left, right) => right.similarity - left.similarity || left.target - right.target).slice(0, TARGETS_PER_SOURCE);
|
|
24374
|
+
const taken = new Set(chosen.map((candidate) => candidate.target));
|
|
24375
|
+
const fromQueries = (extra.get(source) ?? []).filter((to) => !taken.has(to) && linkable(source, to)).map((to) => {
|
|
24376
|
+
const raw = index.similarity(source, to);
|
|
24377
|
+
return {
|
|
24378
|
+
source,
|
|
24379
|
+
target: to,
|
|
24380
|
+
raw,
|
|
24381
|
+
similarity: adjusted(to, raw),
|
|
24382
|
+
sharedQueries: sharedQueries.get(pairKey(source, to)) ?? 0
|
|
24383
|
+
};
|
|
24384
|
+
}).sort((left, right) => right.sharedQueries - left.sharedQueries || right.similarity - left.similarity || left.target - right.target).slice(0, EXTRA_GSC_TARGETS);
|
|
24385
|
+
const all = [...chosen, ...fromQueries];
|
|
24386
|
+
if (all.length) candidatesBySource.set(source, all);
|
|
24387
|
+
}
|
|
24388
|
+
}
|
|
24389
|
+
if (status === "complete" && context.mode === "url" && target !== void 0) {
|
|
24390
|
+
const near = /* @__PURE__ */ new Map();
|
|
24391
|
+
for (const { page, score } of index.neighbours(target, MIN_SIMILARITY)) near.set(page, score);
|
|
24392
|
+
for (const key of sharedQueries.keys()) {
|
|
24393
|
+
const source = Math.floor(key / count);
|
|
24394
|
+
if (!near.has(source)) near.set(source, index.similarity(source, target));
|
|
24395
|
+
}
|
|
24396
|
+
const ranked = [...near].map(([source, raw]) => ({
|
|
24397
|
+
source,
|
|
23608
24398
|
target,
|
|
23609
|
-
|
|
23610
|
-
|
|
23611
|
-
|
|
23612
|
-
|
|
23613
|
-
|
|
23614
|
-
|
|
23615
|
-
|
|
23616
|
-
|
|
23617
|
-
|
|
23618
|
-
|
|
23619
|
-
|
|
23620
|
-
|
|
23621
|
-
|
|
23622
|
-
|
|
23623
|
-
|
|
23624
|
-
const
|
|
23625
|
-
|
|
23626
|
-
|
|
23627
|
-
|
|
23628
|
-
const
|
|
23629
|
-
|
|
23630
|
-
|
|
23631
|
-
|
|
23632
|
-
|
|
23633
|
-
|
|
23634
|
-
|
|
23635
|
-
|
|
23636
|
-
|
|
23637
|
-
|
|
23638
|
-
|
|
23639
|
-
|
|
23640
|
-
|
|
23641
|
-
|
|
23642
|
-
|
|
23643
|
-
|
|
23644
|
-
|
|
23645
|
-
|
|
23646
|
-
|
|
23647
|
-
|
|
23648
|
-
|
|
23649
|
-
|
|
24399
|
+
raw,
|
|
24400
|
+
similarity: adjusted(target, raw),
|
|
24401
|
+
sharedQueries: sharedQueries.get(pairKey(source, target)) ?? 0
|
|
24402
|
+
})).sort((left, right) => Number(right.sharedQueries > 0) - Number(left.sharedQueries > 0) || right.similarity - left.similarity || left.source - right.source);
|
|
24403
|
+
await classify([target, ...ranked.slice(0, URL_SOURCES_PLACED).map((c) => c.source)]);
|
|
24404
|
+
if (targetWeight(types[target] ?? null) === 0 || pages[target]?.noindex) status = "target-not-linkable";
|
|
24405
|
+
else for (const candidate of ranked) {
|
|
24406
|
+
if (candidatesBySource.size >= URL_SOURCES_PLACED) break;
|
|
24407
|
+
if (!linkable(candidate.source, target)) continue;
|
|
24408
|
+
candidatesBySource.set(candidate.source, [{
|
|
24409
|
+
...candidate,
|
|
24410
|
+
similarity: adjusted(target, candidate.raw)
|
|
24411
|
+
}]);
|
|
24412
|
+
}
|
|
24413
|
+
}
|
|
24414
|
+
const judged = [];
|
|
24415
|
+
let pairsConsidered = 0;
|
|
24416
|
+
let pairsJudged = 0;
|
|
24417
|
+
let fellBack = 0;
|
|
24418
|
+
const sources = [...candidatesBySource.keys()].sort((left, right) => left - right);
|
|
24419
|
+
let sourcesWithPlacement = 0;
|
|
24420
|
+
for (let at = 0; at < sources.length; at += SOURCE_WINDOW) {
|
|
24421
|
+
if (context.mode === "url" && sourcesWithPlacement >= URL_SOURCES_JUDGED) break;
|
|
24422
|
+
const window = sources.slice(at, at + SOURCE_WINDOW);
|
|
24423
|
+
deps.progress?.(`Placing and judging links from pages ${at + 1} to ${at + window.length} of ${sources.length}.`);
|
|
24424
|
+
let pairs = (await Promise.all(window.map(async (source) => {
|
|
24425
|
+
const page = pages[source];
|
|
24426
|
+
if (!page) return [];
|
|
24427
|
+
const sentences = await crawl.sentences.read(page.text);
|
|
24428
|
+
const placed = [];
|
|
24429
|
+
for (const candidate of candidatesBySource.get(source) ?? []) {
|
|
24430
|
+
pairsConsidered += 1;
|
|
24431
|
+
const to = pages[candidate.target];
|
|
24432
|
+
if (!to) continue;
|
|
24433
|
+
const placement = pairPlacement(sentences, to, index);
|
|
24434
|
+
if (!placement.sentences.length) continue;
|
|
24435
|
+
placed.push({
|
|
24436
|
+
candidate,
|
|
24437
|
+
placement
|
|
24438
|
+
});
|
|
24439
|
+
}
|
|
24440
|
+
return placed;
|
|
24441
|
+
}))).flat();
|
|
24442
|
+
if (context.mode === "url") {
|
|
24443
|
+
pairs = pairs.sort((left, right) => Number(right.candidate.sharedQueries > 0) - Number(left.candidate.sharedQueries > 0) || right.candidate.similarity - left.candidate.similarity || left.candidate.source - right.candidate.source).slice(0, URL_SOURCES_JUDGED - sourcesWithPlacement);
|
|
24444
|
+
sourcesWithPlacement += pairs.length;
|
|
24445
|
+
}
|
|
24446
|
+
const bySource = /* @__PURE__ */ new Map();
|
|
24447
|
+
for (const pair of pairs) {
|
|
24448
|
+
const list = bySource.get(pair.candidate.source) ?? [];
|
|
24449
|
+
list.push(pair);
|
|
24450
|
+
bySource.set(pair.candidate.source, list);
|
|
24451
|
+
}
|
|
24452
|
+
const groups = [];
|
|
24453
|
+
for (const list of bySource.values()) for (let start = 0; start < list.length; start += 2) groups.push(list.slice(start, start + 2));
|
|
24454
|
+
await Promise.all(groups.map(async (group) => {
|
|
24455
|
+
const first = group[0];
|
|
24456
|
+
if (!first) return;
|
|
24457
|
+
const source = pages[first.candidate.source];
|
|
24458
|
+
if (!source) return;
|
|
24459
|
+
const response = runner ? await runner.ask(judgeRequest({
|
|
24460
|
+
url: source.url,
|
|
24461
|
+
title: source.title,
|
|
24462
|
+
type: types[first.candidate.source] ?? null
|
|
24463
|
+
}, group.map(({ candidate, placement }) => {
|
|
24464
|
+
const to = pages[candidate.target];
|
|
24465
|
+
return {
|
|
24466
|
+
url: to.url,
|
|
24467
|
+
title: to.title,
|
|
24468
|
+
description: to.description,
|
|
24469
|
+
h1: to.h1,
|
|
24470
|
+
type: types[candidate.target] ?? null,
|
|
24471
|
+
intent: intents[candidate.target] ?? null,
|
|
24472
|
+
opening: to.excerpt.slice(0, OPENING_CHARS),
|
|
24473
|
+
placement
|
|
24474
|
+
};
|
|
24475
|
+
}))) : null;
|
|
24476
|
+
group.forEach(({ candidate, placement }, position) => {
|
|
24477
|
+
let decision;
|
|
24478
|
+
if (response) {
|
|
24479
|
+
pairsJudged += 1;
|
|
24480
|
+
decision = decideWithModel(placement, candidate.similarity, response.answers, position);
|
|
24481
|
+
} else {
|
|
24482
|
+
if (runner) fellBack += 1;
|
|
24483
|
+
decision = decideWithoutModel(placement, candidate.similarity);
|
|
24484
|
+
}
|
|
24485
|
+
judged.push({
|
|
24486
|
+
candidate,
|
|
24487
|
+
placement,
|
|
24488
|
+
decision
|
|
24489
|
+
});
|
|
24490
|
+
});
|
|
24491
|
+
}));
|
|
24492
|
+
}
|
|
24493
|
+
const heuristicScore = (item) => item.candidate.similarity * (.5 + (item.decision.state === "suggest" ? item.decision.sentence?.match ?? 0 : 0));
|
|
24494
|
+
const suggested = judged.filter((item) => item.decision.state === "suggest").sort((left, right) => (right.decision.probability ?? -1) - (left.decision.probability ?? -1) || heuristicScore(right) - heuristicScore(left) || (pages[left.candidate.source]?.url ?? "").localeCompare(pages[right.candidate.source]?.url ?? "") || (pages[left.candidate.target]?.url ?? "").localeCompare(pages[right.candidate.target]?.url ?? ""));
|
|
24495
|
+
const perTarget = /* @__PURE__ */ new Map();
|
|
24496
|
+
const usedSentences = /* @__PURE__ */ new Set();
|
|
24497
|
+
const kept = suggested.filter((item) => {
|
|
24498
|
+
if (item.decision.tier === "heuristic" && item.decision.sentence) {
|
|
24499
|
+
const key = `${item.candidate.source}\u0000${item.decision.sentence.text}`;
|
|
24500
|
+
if (usedSentences.has(key)) return false;
|
|
24501
|
+
usedSentences.add(key);
|
|
24502
|
+
}
|
|
24503
|
+
const seen = (perTarget.get(item.candidate.target) ?? 0) + 1;
|
|
24504
|
+
perTarget.set(item.candidate.target, seen);
|
|
24505
|
+
return seen <= 8;
|
|
24506
|
+
});
|
|
24507
|
+
const items = kept.map(({ candidate, decision }) => ({
|
|
24508
|
+
source: pages[candidate.source]?.url ?? "",
|
|
24509
|
+
target: pages[candidate.target]?.url ?? "",
|
|
24510
|
+
tier: decision.tier,
|
|
24511
|
+
probability: decision.probability,
|
|
24512
|
+
role: decision.role,
|
|
24513
|
+
anchor: decision.anchor,
|
|
24514
|
+
sentence: decision.sentence?.text ?? null,
|
|
24515
|
+
section: decision.sentence?.section || null,
|
|
24516
|
+
sentenceMatch: decision.sentence?.match ?? null,
|
|
24517
|
+
similarity: round$1(candidate.similarity),
|
|
24518
|
+
sharedQueries: candidate.sharedQueries,
|
|
24519
|
+
targetInboundLinks: inbound[candidate.target] ?? 0,
|
|
24520
|
+
sourceType: types[candidate.source] ?? null,
|
|
24521
|
+
targetType: types[candidate.target] ?? null
|
|
24522
|
+
}));
|
|
24523
|
+
const suggestedInbound = new Array(count).fill(0);
|
|
24524
|
+
const suggestedOutbound = new Array(count).fill(0);
|
|
24525
|
+
for (const { candidate } of kept) {
|
|
24526
|
+
suggestedInbound[candidate.target] = (suggestedInbound[candidate.target] ?? 0) + 1;
|
|
24527
|
+
suggestedOutbound[candidate.source] = (suggestedOutbound[candidate.source] ?? 0) + 1;
|
|
24528
|
+
}
|
|
24529
|
+
const held = judged.filter((item) => item.decision.state === "held-back");
|
|
24530
|
+
const reasons = {};
|
|
24531
|
+
for (const item of held) reasons[item.decision.reason] = (reasons[item.decision.reason] ?? 0) + 1;
|
|
24532
|
+
const declined = judged.filter((item) => item.decision.state === "declined").length;
|
|
24533
|
+
const usage = runner ? runner.usage() : null;
|
|
24534
|
+
const judgeStatus = !runner ? "not-used" : runner.authError || usage && usage.requests === 0 && usage.failedRequests > 0 ? "failed" : (usage?.failedRequests ?? 0) > 0 ? "partial" : "used";
|
|
24535
|
+
const modelLabel = usage ? `${usage.model}` : null;
|
|
24536
|
+
const judgeNote = !runner ? `Jev was not used: ${"unavailable" in deps.decisionModel ? deps.decisionModel.unavailable : "no TypeSafe key"}. Links come from TF-IDF similarity and sentence matching (heuristics). Add a TypeSafe API key at https://seoskill.dev/account#keys and run seo login, or set TYPESAFE_API_KEY on this machine, to have Jev classify pages and judge each link.` : runner.authError ? `${runner.authError.message} ${fellBack} link${fellBack === 1 ? "" : "s"} fell back to heuristics.` : judgeStatus === "partial" ? `${usage?.failedRequests} request${usage?.failedRequests === 1 ? "" : "s"} to ${modelLabel} failed after retries; ${fellBack} link${fellBack === 1 ? "" : "s"} fell back to heuristics.` : `${modelLabel} classified ${pagesClassified} page${pagesClassified === 1 ? "" : "s"} and judged ${pairsJudged} link${pairsJudged === 1 ? "" : "s"} in ${usage?.requests ?? 0} requests, for $${(usage?.costUsd ?? 0).toFixed(4)}.`;
|
|
24537
|
+
if (status === "complete" && (crawl.stats.capReached || judgeStatus === "partial" || judgeStatus === "failed")) status = "partial";
|
|
24538
|
+
const counts = {
|
|
24539
|
+
verdict: "",
|
|
24540
|
+
pagesCrawled: count,
|
|
24541
|
+
existingContextualLinks,
|
|
24542
|
+
orphanPages: inboundAnywhere.filter((value, id) => value === 0 && types[id] !== "home").length,
|
|
24543
|
+
weaklyLinkedPages: inbound.filter((value, id) => value < WEAK_INBOUND && types[id] !== "home").length,
|
|
24544
|
+
pairsConsidered,
|
|
24545
|
+
suggestions: items.length,
|
|
24546
|
+
strong: items.filter((item) => item.tier === "strong").length,
|
|
24547
|
+
possible: items.filter((item) => item.tier === "possible").length,
|
|
24548
|
+
heuristic: items.filter((item) => item.tier === "heuristic").length,
|
|
24549
|
+
heldBack: held.length,
|
|
24550
|
+
declined
|
|
24551
|
+
};
|
|
24552
|
+
const targetPage = target !== void 0 ? pages[target] : void 0;
|
|
24553
|
+
const report = {
|
|
23650
24554
|
site: input.site,
|
|
23651
|
-
|
|
23652
|
-
|
|
23653
|
-
|
|
23654
|
-
|
|
23655
|
-
|
|
23656
|
-
|
|
23657
|
-
|
|
23658
|
-
|
|
23659
|
-
|
|
23660
|
-
|
|
23661
|
-
|
|
23662
|
-
|
|
23663
|
-
|
|
23664
|
-
|
|
23665
|
-
|
|
23666
|
-
calls: result.calls,
|
|
23667
|
-
possiblyTruncated: result.rowsFetched >= MAX_GSC_ROWS
|
|
23668
|
-
})),
|
|
23669
|
-
rowsFetched: targetRequests.reduce((sum, { result }) => sum + result.rowsFetched, 0),
|
|
23670
|
-
calls: targetRequests.reduce((sum, { result }) => sum + result.calls, 0),
|
|
23671
|
-
maxRowsPerRequest: MAX_GSC_ROWS,
|
|
23672
|
-
possiblyTruncated: targetTruncated
|
|
23673
|
-
},
|
|
23674
|
-
candidates: {
|
|
23675
|
-
queried: queryCandidates,
|
|
23676
|
-
rowsFetched: sourceResult.rowsFetched,
|
|
23677
|
-
calls: sourceResult.calls,
|
|
23678
|
-
maxRows: MAX_GSC_ROWS,
|
|
23679
|
-
possiblyTruncated: sourceTruncated
|
|
23680
|
-
},
|
|
23681
|
-
completeness: !queryCandidates ? "not-queried" : targetTruncated || sourceTruncated ? "possibly-truncated" : "retained-rows-only"
|
|
23682
|
-
},
|
|
23683
|
-
methodology: {
|
|
23684
|
-
id: "gsc_internal_link_candidates",
|
|
23685
|
-
version: 4,
|
|
23686
|
-
lexicalTargetLimit: 50,
|
|
23687
|
-
matchedQueryEvidenceLimit: 10,
|
|
23688
|
-
observedLinkEvidenceLimit: 20,
|
|
23689
|
-
matching: "pairwise_exact_then_precision_lexical",
|
|
23690
|
-
ranking: "exact_matches_then_matched_query_impressions_then_relevance",
|
|
23691
|
-
contextualPlacementVerified: true,
|
|
23692
|
-
placement: "tfidf_sentence_match_then_phrase_anchors"
|
|
23693
|
-
},
|
|
23694
|
-
filters: {
|
|
23695
|
-
minImpressions,
|
|
23696
|
-
resultLimit: limit,
|
|
23697
|
-
checkLimit,
|
|
23698
|
-
maxGscRowsPerRequest: MAX_GSC_ROWS
|
|
23699
|
-
},
|
|
23700
|
-
target: {
|
|
23701
|
-
requestedUrl: target.requestedUrl,
|
|
23702
|
-
preferredUrl: target.preferredUrl,
|
|
23703
|
-
finalUrl: target.finalUrl,
|
|
23704
|
-
canonical: target.canonical,
|
|
23705
|
-
status: target.status,
|
|
23706
|
-
aliases: target.aliases,
|
|
23707
|
-
verification: target.verification,
|
|
23708
|
-
technicalSignals: target.technicalSignals,
|
|
23709
|
-
fetchDiagnostics: target.fetchDiagnostics,
|
|
23710
|
-
queries: analysis.targetQueries
|
|
24555
|
+
mode: context.mode,
|
|
24556
|
+
targetUrl: context.targetUrl ?? null,
|
|
24557
|
+
generatedAt: context.now.toISOString(),
|
|
24558
|
+
dataStatus: status,
|
|
24559
|
+
crawl: {
|
|
24560
|
+
startUrl: context.startUrl,
|
|
24561
|
+
method: crawl.method,
|
|
24562
|
+
maxPages: crawl.stats.maxPages,
|
|
24563
|
+
pageLimit: crawl.stats.pageLimit,
|
|
24564
|
+
urlsFound: crawl.stats.urlsFound,
|
|
24565
|
+
pagesCrawled: count,
|
|
24566
|
+
capReached: crawl.stats.capReached,
|
|
24567
|
+
crawlDelaySeconds: crawl.stats.crawlDelaySeconds,
|
|
24568
|
+
skipped: crawl.stats.skipped,
|
|
24569
|
+
redirectsChecked: crawl.stats.redirectsChecked
|
|
23711
24570
|
},
|
|
23712
|
-
|
|
23713
|
-
|
|
23714
|
-
|
|
23715
|
-
|
|
23716
|
-
|
|
23717
|
-
|
|
23718
|
-
|
|
23719
|
-
|
|
23720
|
-
|
|
23721
|
-
|
|
23722
|
-
|
|
23723
|
-
|
|
23724
|
-
|
|
23725
|
-
|
|
24571
|
+
searchConsole,
|
|
24572
|
+
judge: {
|
|
24573
|
+
status: judgeStatus,
|
|
24574
|
+
note: judgeNote,
|
|
24575
|
+
keySource: "model" in deps.decisionModel ? deps.decisionModel.keySource : null,
|
|
24576
|
+
provider: usage?.provider ?? null,
|
|
24577
|
+
model: usage?.model ?? null,
|
|
24578
|
+
requests: usage?.requests ?? 0,
|
|
24579
|
+
failedRequests: usage?.failedRequests ?? 0,
|
|
24580
|
+
inputTokens: usage?.inputTokens ?? 0,
|
|
24581
|
+
costUsd: usage?.costUsd ?? 0,
|
|
24582
|
+
pagesClassified,
|
|
24583
|
+
pairsJudged,
|
|
24584
|
+
thresholds: { ...THRESHOLDS },
|
|
24585
|
+
floors: { ...FLOORS }
|
|
23726
24586
|
},
|
|
24587
|
+
target: target !== void 0 && targetPage ? {
|
|
24588
|
+
url: targetPage.url,
|
|
24589
|
+
title: targetPage.title,
|
|
24590
|
+
type: types[target] ?? null,
|
|
24591
|
+
intent: intents[target] ?? null,
|
|
24592
|
+
inboundLinks: inbound[target] ?? 0,
|
|
24593
|
+
inboundLinksAnywhere: inboundAnywhere[target] ?? 0,
|
|
24594
|
+
noindex: targetPage.noindex
|
|
24595
|
+
} : null,
|
|
24596
|
+
summary: counts,
|
|
23727
24597
|
items,
|
|
23728
|
-
|
|
23729
|
-
|
|
23730
|
-
|
|
23731
|
-
|
|
23732
|
-
|
|
23733
|
-
|
|
23734
|
-
|
|
23735
|
-
|
|
23736
|
-
|
|
23737
|
-
|
|
23738
|
-
|
|
23739
|
-
|
|
23740
|
-
|
|
23741
|
-
|
|
23742
|
-
|
|
24598
|
+
heldBack: {
|
|
24599
|
+
count: held.length,
|
|
24600
|
+
reasons,
|
|
24601
|
+
examples: held.slice(0, EXAMPLE_LIMIT).map((item) => ({
|
|
24602
|
+
source: pages[item.candidate.source]?.url ?? "",
|
|
24603
|
+
target: pages[item.candidate.target]?.url ?? "",
|
|
24604
|
+
reason: item.decision.reason,
|
|
24605
|
+
detail: item.decision.detail
|
|
24606
|
+
}))
|
|
24607
|
+
},
|
|
24608
|
+
pageInventory: context.mode === "site" ? pages.map((page, id) => ({
|
|
24609
|
+
url: page.url,
|
|
24610
|
+
title: page.title,
|
|
24611
|
+
type: types[id] ?? null,
|
|
24612
|
+
intent: intents[id] ?? null,
|
|
24613
|
+
words: page.words,
|
|
24614
|
+
inboundLinks: inbound[id] ?? 0,
|
|
24615
|
+
inboundLinksAnywhere: inboundAnywhere[id] ?? 0,
|
|
24616
|
+
outboundLinks: contextual[id]?.size ?? 0,
|
|
24617
|
+
suggestedInbound: suggestedInbound[id] ?? 0,
|
|
24618
|
+
suggestedOutbound: suggestedOutbound[id] ?? 0,
|
|
24619
|
+
noindex: page.noindex
|
|
24620
|
+
})).sort((left, right) => left.inboundLinks - right.inboundLinks || left.url.localeCompare(right.url)) : [],
|
|
24621
|
+
redirectedLinks,
|
|
24622
|
+
genericAnchors: crawl.genericAnchors.map((anchor) => ({
|
|
24623
|
+
source: pages[anchor.source]?.url ?? "",
|
|
24624
|
+
href: anchor.href,
|
|
24625
|
+
text: anchor.text
|
|
24626
|
+
})),
|
|
24627
|
+
warnings,
|
|
24628
|
+
caveats: [],
|
|
24629
|
+
next: null
|
|
23743
24630
|
};
|
|
23744
|
-
|
|
23745
|
-
|
|
23746
|
-
|
|
23747
|
-
const placed = found.map(({ item, text }) => {
|
|
23748
|
-
const placement = target && text ? linkPlacement(text, target, corpus) : {
|
|
23749
|
-
contentSimilarity: 0,
|
|
23750
|
-
sentences: []
|
|
23751
|
-
};
|
|
23752
|
-
const best = placement.sentences[0];
|
|
23753
|
-
const recommendation = best ? {
|
|
23754
|
-
...item.recommendation,
|
|
23755
|
-
action: `${item.recommendation.action} It fits in: "${best.text}" Anchor text could be ${best.anchors.map((anchor) => `"${anchor}"`).join(" or ")}.`
|
|
23756
|
-
} : {
|
|
23757
|
-
...item.recommendation,
|
|
23758
|
-
action: `${item.recommendation.action} No sentence on this page names the target, so a link here needs new text.`
|
|
23759
|
-
};
|
|
23760
|
-
return {
|
|
23761
|
-
...item,
|
|
23762
|
-
placement,
|
|
23763
|
-
recommendation
|
|
23764
|
-
};
|
|
24631
|
+
report.summary.verdict = internalLinksVerdict({
|
|
24632
|
+
...report,
|
|
24633
|
+
counts
|
|
23765
24634
|
});
|
|
23766
|
-
|
|
24635
|
+
report.caveats = caveats$1(report);
|
|
24636
|
+
const top = items[0];
|
|
24637
|
+
report.next = context.mode === "site" ? top ? `seo internal-links --url ${top.target}` : null : `seo page --url ${report.target?.url ?? context.targetUrl}`;
|
|
24638
|
+
return report;
|
|
24639
|
+
}
|
|
24640
|
+
function caveats$1(report) {
|
|
24641
|
+
const crawl = report.crawl;
|
|
24642
|
+
return [
|
|
24643
|
+
`Crawl: ${crawl.pagesCrawled} page${crawl.pagesCrawled === 1 ? "" : "s"} from ${crawl.method === "sitemap" ? "the sitemap" : "links from the start page"}, limit ${crawl.pageLimit}${crawl.capReached ? " (reached, so pages beyond it were not checked and orphan and weak-link counts cover crawled pages only)" : " (not reached)"}.${crawl.crawlDelaySeconds ? ` robots.txt crawl-delay of ${crawl.crawlDelaySeconds}s was honoured.` : ""}`,
|
|
24644
|
+
report.judge.note,
|
|
24645
|
+
report.searchConsole.status === "used" ? `Search Console: ${report.searchConsole.rows.toLocaleString("en-US")} query rows from ${report.searchConsole.range?.startDate} to ${report.searchConsole.range?.endDate}, brand queries left out; ${report.searchConsole.pairsWithSharedQueries} page pairs share a query.${report.searchConsole.possiblyTruncated ? " The 100,000-row cap was reached." : ""}` : `Search Console was not used${report.searchConsole.note ? `: ${report.searchConsole.note}` : ""}. Candidates come from page text only.`,
|
|
24646
|
+
`Rules in code: a link needs a model probability of ${THRESHOLDS.possible} (${THRESHOLDS.strong} is strong), a sentence with at least ${FLOORS.minSentenceMatch * 100}% of the words that name the target or pages at least ${FLOORS.minSimilarityAlone} similar, and an anchor from the chosen sentence. At most 8 links per target.`,
|
|
24647
|
+
"Similarity, sentence match and anchors are text heuristics, and a model probability is a judgement of reader value. None of them predicts rankings or traffic."
|
|
24648
|
+
];
|
|
23767
24649
|
}
|
|
23768
24650
|
//#endregion
|
|
23769
24651
|
//#region packages/core/src/analyze/url-templates.ts
|
|
@@ -25152,7 +26034,7 @@ function reportId(input) {
|
|
|
25152
26034
|
})).digest("hex").slice(0, 20)}`;
|
|
25153
26035
|
}
|
|
25154
26036
|
function cacheGet(id, now) {
|
|
25155
|
-
const row =
|
|
26037
|
+
const row = cacheDb().prepare("SELECT report_json FROM performance_reports WHERE id = ? AND expires_at > ?").get(id, now);
|
|
25156
26038
|
if (!row?.report_json) return void 0;
|
|
25157
26039
|
try {
|
|
25158
26040
|
const report = JSON.parse(row.report_json);
|
|
@@ -25167,10 +26049,12 @@ function cacheSet(report, now) {
|
|
|
25167
26049
|
raw: void 0
|
|
25168
26050
|
};
|
|
25169
26051
|
const reportJson = JSON.stringify(stored);
|
|
25170
|
-
|
|
25171
|
-
|
|
25172
|
-
|
|
25173
|
-
|
|
26052
|
+
writeToCache((database) => {
|
|
26053
|
+
database.prepare(`INSERT OR REPLACE INTO performance_reports
|
|
26054
|
+
(id, url, strategy, report_json, created_at, expires_at)
|
|
26055
|
+
VALUES (?, ?, ?, ?, ?, ?)`).run(report.id, report.url, report.strategy, reportJson, Date.parse(report.generatedAt), now + CACHE_TTL_MS);
|
|
26056
|
+
noteCacheWrite(Buffer.byteLength(reportJson), database);
|
|
26057
|
+
});
|
|
25174
26058
|
}
|
|
25175
26059
|
async function fallbackReport(input) {
|
|
25176
26060
|
const fetched = await fetchPage(input.url, {
|
|
@@ -27658,8 +28542,8 @@ const CLICKY_KEY_PREFIX = "x-clicky-";
|
|
|
27658
28542
|
//#endregion
|
|
27659
28543
|
//#region packages/core/src/commands/page-facts.ts
|
|
27660
28544
|
const MAX_PAGES = 20;
|
|
27661
|
-
const TIMEOUT_MS = 6e3;
|
|
27662
|
-
const MAX_BYTES = 1e6;
|
|
28545
|
+
const TIMEOUT_MS$1 = 6e3;
|
|
28546
|
+
const MAX_BYTES$1 = 1e6;
|
|
27663
28547
|
const clean = (value) => value?.trim().replace(/\s+/g, " ") || null;
|
|
27664
28548
|
async function boundedText(response) {
|
|
27665
28549
|
if (!response.body) return "";
|
|
@@ -27667,7 +28551,7 @@ async function boundedText(response) {
|
|
|
27667
28551
|
const decoder = new TextDecoder();
|
|
27668
28552
|
let text = "";
|
|
27669
28553
|
let total = 0;
|
|
27670
|
-
while (total < MAX_BYTES) {
|
|
28554
|
+
while (total < MAX_BYTES$1) {
|
|
27671
28555
|
const { done, value } = await reader.read();
|
|
27672
28556
|
if (done) return text + decoder.decode();
|
|
27673
28557
|
total += value.byteLength;
|
|
@@ -27747,7 +28631,7 @@ async function facts(url) {
|
|
|
27747
28631
|
"user-agent": SEO_CRAWLER_USER_AGENT,
|
|
27748
28632
|
accept: "text/html"
|
|
27749
28633
|
},
|
|
27750
|
-
signal: AbortSignal.timeout(TIMEOUT_MS)
|
|
28634
|
+
signal: AbortSignal.timeout(TIMEOUT_MS$1)
|
|
27751
28635
|
});
|
|
27752
28636
|
out.status = response.status;
|
|
27753
28637
|
const location = response.headers.get("location");
|
|
@@ -28116,6 +29000,25 @@ async function localResearchKeys() {
|
|
|
28116
29000
|
if (ahrefs?.apiKey) keys.set("x-ahrefs-key", ahrefs.apiKey);
|
|
28117
29001
|
return keys;
|
|
28118
29002
|
}
|
|
29003
|
+
const TYPESAFE_API_KEY_ENV = "TYPESAFE_API_KEY";
|
|
29004
|
+
async function localInternalLinksDependencies(options) {
|
|
29005
|
+
const searchConsole = await resolveLocalSite(options.site, { refresh: options.refresh }).then((property) => ({
|
|
29006
|
+
property,
|
|
29007
|
+
query: (request) => querySearchAnalytics(property, request, { refresh: options.refresh })
|
|
29008
|
+
}), (error) => ({ unavailable: `no Search Console property was found for this site (${error instanceof Error ? error.message : String(error)})` }));
|
|
29009
|
+
const local = process.env[TYPESAFE_API_KEY_ENV]?.trim();
|
|
29010
|
+
const saved = local ? void 0 : (await localResearchKeys().catch(() => new Headers())).get("x-typesafe-key");
|
|
29011
|
+
return {
|
|
29012
|
+
searchConsole,
|
|
29013
|
+
decisionModel: local ? {
|
|
29014
|
+
model: typesafeDecisionModel({ apiKey: local }),
|
|
29015
|
+
keySource: "environment"
|
|
29016
|
+
} : saved ? {
|
|
29017
|
+
model: typesafeDecisionModel({ apiKey: saved }),
|
|
29018
|
+
keySource: "account"
|
|
29019
|
+
} : { unavailable: "no TypeSafe key is set" }
|
|
29020
|
+
};
|
|
29021
|
+
}
|
|
28119
29022
|
async function localRefreshClicky() {
|
|
28120
29023
|
if (!cloudLinked()) return null;
|
|
28121
29024
|
await cloudCredentials({ refreshClicky: true });
|
|
@@ -29705,6 +30608,94 @@ async function resetSeoData() {
|
|
|
29705
30608
|
});
|
|
29706
30609
|
}
|
|
29707
30610
|
//#endregion
|
|
30611
|
+
//#region packages/core/src/update-notice.ts
|
|
30612
|
+
const REGISTRY_URL = "https://registry.npmjs.org/seo/latest";
|
|
30613
|
+
const CHECK_EVERY_MS = 864e5;
|
|
30614
|
+
const TIMEOUT_MS = 2e3;
|
|
30615
|
+
const MAX_BYTES = 65536;
|
|
30616
|
+
let checking = false;
|
|
30617
|
+
function runByNpx(env, cliPath) {
|
|
30618
|
+
if (env.npm_command === "exec" || /npx/i.test(env.npm_execpath ?? "")) return true;
|
|
30619
|
+
if (!cliPath) return false;
|
|
30620
|
+
let path = cliPath;
|
|
30621
|
+
try {
|
|
30622
|
+
path = realpathSync(cliPath);
|
|
30623
|
+
} catch {}
|
|
30624
|
+
return /[\\/](?:_npx|dlx)[\\/]/.test(path);
|
|
30625
|
+
}
|
|
30626
|
+
function isNewerVersion(current, latest) {
|
|
30627
|
+
const parse = (value) => value.replace(/^v/, "").split("-")[0]?.split(".").map((part) => Number.parseInt(part, 10)) ?? [];
|
|
30628
|
+
const currentParts = parse(current);
|
|
30629
|
+
const latestParts = parse(latest);
|
|
30630
|
+
for (let index = 0; index < 3; index += 1) {
|
|
30631
|
+
const currentPart = currentParts[index] ?? 0;
|
|
30632
|
+
const latestPart = latestParts[index] ?? 0;
|
|
30633
|
+
if (!Number.isFinite(currentPart) || !Number.isFinite(latestPart)) return false;
|
|
30634
|
+
if (latestPart > currentPart) return true;
|
|
30635
|
+
if (latestPart < currentPart) return false;
|
|
30636
|
+
}
|
|
30637
|
+
return false;
|
|
30638
|
+
}
|
|
30639
|
+
function isLatestCache(value) {
|
|
30640
|
+
if (!value || typeof value !== "object") return false;
|
|
30641
|
+
const cache = value;
|
|
30642
|
+
return typeof cache.checkedAt === "number" && typeof cache.latest === "string" && /^\d+\.\d+\.\d+$/.test(cache.latest);
|
|
30643
|
+
}
|
|
30644
|
+
function checkRegistry(cacheFile, now) {
|
|
30645
|
+
if (checking) return;
|
|
30646
|
+
checking = true;
|
|
30647
|
+
const done = () => {
|
|
30648
|
+
checking = false;
|
|
30649
|
+
};
|
|
30650
|
+
const req = request(REGISTRY_URL, { headers: { accept: "application/json" } });
|
|
30651
|
+
const timer = setTimeout(() => req.destroy(), TIMEOUT_MS);
|
|
30652
|
+
timer.unref();
|
|
30653
|
+
req.on("socket", (socket) => socket.unref());
|
|
30654
|
+
req.on("error", () => void 0);
|
|
30655
|
+
req.on("close", () => {
|
|
30656
|
+
clearTimeout(timer);
|
|
30657
|
+
done();
|
|
30658
|
+
});
|
|
30659
|
+
req.on("response", (response) => {
|
|
30660
|
+
const chunks = [];
|
|
30661
|
+
let size = 0;
|
|
30662
|
+
response.on("error", () => void 0);
|
|
30663
|
+
response.on("data", (chunk) => {
|
|
30664
|
+
size += chunk.length;
|
|
30665
|
+
if (size > MAX_BYTES) req.destroy();
|
|
30666
|
+
else chunks.push(chunk);
|
|
30667
|
+
});
|
|
30668
|
+
response.on("end", () => {
|
|
30669
|
+
if (response.statusCode !== 200 || size > MAX_BYTES) return;
|
|
30670
|
+
readJson(new Response(Buffer.concat(chunks)), "The npm registry").then((body) => {
|
|
30671
|
+
const cache = {
|
|
30672
|
+
checkedAt: now,
|
|
30673
|
+
latest: body.version
|
|
30674
|
+
};
|
|
30675
|
+
if (isLatestCache(cache)) writeJsonAtomic(cacheFile, cache, 384);
|
|
30676
|
+
}).catch(() => void 0);
|
|
30677
|
+
});
|
|
30678
|
+
});
|
|
30679
|
+
req.end();
|
|
30680
|
+
}
|
|
30681
|
+
function updateNotice(options = {}) {
|
|
30682
|
+
try {
|
|
30683
|
+
const env = options.env ?? process.env;
|
|
30684
|
+
const argv = options.argv ?? process.argv;
|
|
30685
|
+
if (env.NO_UPDATE_NOTIFIER || argv.includes("--no-update-notifier") || isTelemetryCi(env)) return;
|
|
30686
|
+
const cacheFile = options.cacheFile ?? join(getSeoCliPaths().cacheDir, "latest-version.json");
|
|
30687
|
+
const now = (options.now ?? /* @__PURE__ */ new Date()).getTime();
|
|
30688
|
+
const stored = readJsonFile(cacheFile);
|
|
30689
|
+
const cache = isLatestCache(stored) ? stored : void 0;
|
|
30690
|
+
if (!cache || now - cache.checkedAt >= CHECK_EVERY_MS) checkRegistry(cacheFile, now);
|
|
30691
|
+
const current = options.version ?? SEO_VERSION;
|
|
30692
|
+
if (!cache || !isNewerVersion(current, cache.latest)) return void 0;
|
|
30693
|
+
return runByNpx(env, options.cliPath ?? argv[1]) ? `seo ${cache.latest} is out (you have ${current}). Run npx -y seo@latest; in an MCP config use seo@latest, then restart the client.` : `seo ${cache.latest} is out (you have ${current}). Update with: npm i -g seo`;
|
|
30694
|
+
} catch {
|
|
30695
|
+
return;
|
|
30696
|
+
}
|
|
30697
|
+
}
|
|
30698
|
+
//#endregion
|
|
29708
30699
|
//#region packages/core/src/utils/time.ts
|
|
29709
30700
|
function parseOlderThan(input) {
|
|
29710
30701
|
if (!input) return;
|
|
@@ -29739,6 +30730,6 @@ function isoDate(daysAgo = 0) {
|
|
|
29739
30730
|
return date.toISOString().slice(0, 10);
|
|
29740
30731
|
}
|
|
29741
30732
|
//#endregion
|
|
29742
|
-
export { AHREFS_API_KEY_ENV, AHREFS_API_KEY_SECRET, AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, ANALYTICS_PROVIDER_IDS, AUDIT_PAGE_RULE_IDS, AhrefsClient, BING_API_KEY_ENV, BING_API_KEY_SECRET, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CLICKY_SITEKEY_ENV, CLICKY_SITE_KEYS_SECRET, CLOUD_SIGN_IN, CLOUD_URL, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DATAFORSEO_CREDENTIAL_SECRET, DATAFORSEO_LOGIN_ENV, DATAFORSEO_PASSWORD_ENV, DEFAULT_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, DEFAULT_UNREACHED_PAGE_LIMIT, DataForSeoClient, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, GoogleApiError, INCIDENTS_URL,
|
|
30733
|
+
export { AHREFS_API_KEY_ENV, AHREFS_API_KEY_SECRET, AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, ANALYTICS_PROVIDER_IDS, AUDIT_PAGE_RULE_IDS, AhrefsClient, BING_API_KEY_ENV, BING_API_KEY_SECRET, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CLICKY_SITEKEY_ENV, CLICKY_SITE_KEYS_SECRET, CLOUD_SIGN_IN, CLOUD_URL, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DATAFORSEO_CREDENTIAL_SECRET, DATAFORSEO_LOGIN_ENV, DATAFORSEO_PASSWORD_ENV, DEFAULT_MAX_PAGES, DEFAULT_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, DEFAULT_UNREACHED_PAGE_LIMIT, DataForSeoClient, DecisionModelError, DecisionRunner, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, GoogleApiError, INCIDENTS_URL, JsonFileStorageAdapter, LINK_ROLES, LOCAL_BING_CONNECT, LOCAL_SIGN_IN, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MANAGED_PROVIDER_SECRET_NAMES, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, MAX_MAX_PAGES, MAX_SEARCH_CONSOLE_EXPORT_FILES, MAX_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, PAGE_TYPES, PROVIDER_SECRET_NAMES, ProviderError, ProviderExtensionRegistry, RESEARCH_IMPORT_COLUMN_KEYS, ResponseSizeLimitError, SEARCH_INTENTS, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_PROVIDER_API_VERSION, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, STEP_CHANGE_RULES, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_FAILURE_CONTEXTS, TELEMETRY_FAILURE_REASONS, TELEMETRY_NOTICE, TELEMETRY_OPERATIONS, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, TYPESAFE_API_KEY_ENV, agentActionsView, aiReferralSourceForValue, aiReferralsReport, analyticsConnectionLabel, analyticsConnectionProviderId, analyticsConnectionSchema, analyticsProviderDetails, analyticsProviderSchema, analyzeAiReferralRows, analyzeCannibalImportRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, auditCrawlPages, auditCrawlRequests, auditPage, authStatus, bearerFetch, boundedSitemapInventory, buildAgentFindings, buildQueryClusterReportFromRows, buildTelemetryPayload, cacheDb, cannibalImportReport, cannibalReport, cannibalReportWith, changeWindow, checkDatabaseReadiness, classifyTelemetryFailure, clearCache, cloudAccountFor, cloudBing, cloudCredentials, cloudGoogleEmails, cloudGoogleToken, cloudKeys, cloudLinked, cloudLogout, clusterQueryRows, combineRobotsValues, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, ctrUnderperformersReportWith, decayClusterDrafts, decayComparisonRange, decayingReport, decayingReportWith, deleteAhrefsApiKey, deleteBingApiKey, deleteClickySiteKey, deleteDataForSeoCredentials, deleteManagedProviderSecrets, deleteProviderExtensionAccount, deleteProviderExtensionCredentials, deleteProviderSecret, deleteTokens, deriveBrandTerms, describeInstalledProviderExtension, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensurePrivateDatabaseFiles, ensureSeoCliDirs, estimateSerpTitleWidth, expectedCtrForPosition, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fileSize, finalGscDateRange, findOverlappingSearchUpdates, finishCloudLogin, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, getAuthModeStatus, getCacheStats, getClientConfig, getDb, getPrivacySnapshot, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleJson, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoverageSignals, inferTemplateMovement, initializeTelemetry, inspectProviderPackage, inspectSearchConsoleUrl, inspectUrl, installProviderPackage, installedProviderPackageSchema, installedProviderPackagesSchema, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isNewerVersion, isSkippableReportError, isTelemetryCi, isTelemetryOperation, isTelemetryReport, isoDate, isoWeek, latestFinalGscDate, linkRecover, linkRecoverRecommendation, listAnalyticsAccounts, listAnalyticsDataStreams, listGa4AccountSummaries, listGa4DataStreams, listGoogleAccounts, listInstalledProviderExtensions, listRules, listSearchConsoleSites, listSearchUpdates, listSites, loadInstalledProviderExtensions, loadProviderExtensions, loadProviderPackage, loadSearchConsoleExport, localAccounts, localBing, localCredentialsProvider, localError, localGoogleFetch, localInternalLinksDependencies, localRefreshClicky, localResearchKeys, loginWithLoopback, makeRunId, matchGa4WebStreams, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, nodeCommandDeps, normalizeBrandText, normalizeClickySiteId, normalizeClickySiteKey, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, normalizeProviderExtensionAccount, normalizedResearchColumnName, notSignedIn, noteCacheWrite, openCloudLink, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, pendingCloudLogin, performanceAudit, performanceReportIsCacheable, plural, providerCapabilitySchema, providerConnectionFieldSchema, providerExactVersionSchema, providerExtensionIdSchema, providerIdSchema, providerIntegritySchema, providerKindSchema, providerPackageManifestSchema, providerPackageNameSchema, providerPreferenceSchema, providerRegistrationSchema, providerRequestJson, providerRequestText, pruneLogs, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, quickWinsReportWith, readAhrefsApiKey, readAllTokens, readBingApiKey, readBoundedResponseText, readClickySiteKey, readClickySiteKeys, readConfig, readDataForSeoCredentials, readInstalledProviderPackages, readJson, readJsonOrUndefined, readOauthClient, readProviderExtensionAccount, readProviderExtensionCredentials, readProviderSecret, readTokens, recommendationForRule, reconcileExportPagesWithCrawl, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, registrableDomain, removeInstalledProviderPackage, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderWorkflowMarkdown, reportNarrative, requestHeaders, requirePageVerification, researchImportColumns, resetSeoData, resolveLocalSite, resolveSegmentRanges, resolveSeoCliPaths, reviewObservations, roundedPosition, runAnalyticsProviderLandingPages, runAnalyticsReport, runDoctor, runGa4Report, runProviderExtensionAction, runtimeForProvider, safeProviderAdapterError, saveInstalledProviderPackage, searchAnalyticsRows, searchUpdatesFromIncidents, secondPage, secondPageWith, segmentComparisonRange, segmentImpact, segmentRangeDays, seoErrorEnvelope, sessionSourceInListFilter, setActiveGoogleAccount, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteStartUrl, startCloudLogin, strikingDistance, strikingDistanceWith, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, telemetryErrorCategory, toSeoError, tokenSchema, tokenStoreSchema, topFixes, trackTelemetryCommandFailed, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, trafficAnomalyWith, typesafeDecisionModel, unavailableSegmentImpactReport, unicodeTokens, uninstallProviderExtension, updateCorrelation, updateNotice, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateResearchImportColumnSources, verifyAnalyticsProvider, verifyProviderExtension, verifyQueryContent, verifyQueryContentWith, withAgentReportContract, withRankingHistory, workflowPresentation, writeAhrefsApiKey, writeBingApiKey, writeClickySiteKey, writeConfig, writeDataForSeoCredentials, writeInstalledProviderPackages, writeOauthClient, writeProviderExtensionAccount, writeProviderExtensionCredentials, writeProviderSecret, writeToCache, writeTokens, xRobotsDirectives };
|
|
29743
30734
|
|
|
29744
30735
|
//# sourceMappingURL=index.js.map
|