seo 0.8.3 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +16 -3
- package/README.md +1 -1
- package/dist/cli.js +80 -80
- package/dist/cli.js.map +1 -1
- package/dist/{dist-Bv1nbxq_.js → dist-BvbFtizP.js} +1 -1
- package/dist/{dist-B8MTNp_F.js → dist-CsWo-wJG.js} +101 -95
- package/dist/dist-CsWo-wJG.js.map +1 -0
- package/dist/{esm-BZhSLbcQ.js → esm-CM7xTOjT.js} +2 -2
- package/dist/{esm-BZhSLbcQ.js.map → esm-CM7xTOjT.js.map} +1 -1
- package/dist/index.d.ts +361 -217
- package/dist/index.js +1793 -932
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +2574 -1343
- package/dist/mcp.js.map +1 -1
- package/package.json +3 -1
- package/skills/seo/SKILL.md +2 -2
- package/dist/dist-B8MTNp_F.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { n as version, t as load } from "./esm-
|
|
2
|
+
import { n as version, t as load } from "./esm-CM7xTOjT.js";
|
|
3
3
|
import { SEO_PROVIDER_API_VERSION } from "./provider-sdk.js";
|
|
4
4
|
import crypto, { createHash, randomUUID, sign } from "node:crypto";
|
|
5
5
|
import { z } from "zod";
|
|
6
|
-
import { arch, homedir, hostname, platform, totalmem } from "node:os";
|
|
6
|
+
import { arch, homedir, hostname, platform, tmpdir, totalmem } from "node:os";
|
|
7
7
|
import { basename, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
8
8
|
import open from "open";
|
|
9
9
|
import { chmodSync, createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
@@ -22,7 +22,7 @@ import { chromium } from "playwright-core";
|
|
|
22
22
|
import { gunzipSync } from "node:zlib";
|
|
23
23
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
24
24
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
25
|
-
import { mkdir, open as open$1, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
25
|
+
import { mkdir, mkdtemp, open as open$1, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
26
26
|
import { TextDecoder as TextDecoder$1, promisify } from "node:util";
|
|
27
27
|
import { execFile } from "node:child_process";
|
|
28
28
|
import pino from "pino";
|
|
@@ -1458,12 +1458,12 @@ var SeoError = class extends Error {
|
|
|
1458
1458
|
this.retryable = errorMetadata[code].retryable;
|
|
1459
1459
|
}
|
|
1460
1460
|
};
|
|
1461
|
-
function errorMessage$
|
|
1461
|
+
function errorMessage$1(error) {
|
|
1462
1462
|
return error instanceof Error ? error.message : String(error);
|
|
1463
1463
|
}
|
|
1464
1464
|
function toSeoError(error) {
|
|
1465
1465
|
if (error instanceof SeoError) return error;
|
|
1466
|
-
const message = errorMessage$
|
|
1466
|
+
const message = errorMessage$1(error);
|
|
1467
1467
|
if (/not logged in/i.test(message)) return new SeoError("AUTH_REQUIRED", message);
|
|
1468
1468
|
if (/oauth client config.*missing|shared seo google app/i.test(message)) return new SeoError("AUTH_CONFIG_REQUIRED", message);
|
|
1469
1469
|
if (/invalid_grant|refresh token.*no longer valid/i.test(message)) return new SeoError("AUTH_EXPIRED", message);
|
|
@@ -1844,6 +1844,262 @@ function decayComparisonRange(current, comparison = "previous-period") {
|
|
|
1844
1844
|
};
|
|
1845
1845
|
}
|
|
1846
1846
|
//#endregion
|
|
1847
|
+
//#region packages/core/src/updates/ranking-history.ts
|
|
1848
|
+
const RANKING_HISTORY = [
|
|
1849
|
+
{
|
|
1850
|
+
id: "XhUDXP7A67iHCD2kmbVu",
|
|
1851
|
+
begin: "2026-09-24T16:15:00+00:00",
|
|
1852
|
+
end: "2026-10-08T08:00:00+00:00",
|
|
1853
|
+
external_desc: "September 2026 spam update"
|
|
1854
|
+
},
|
|
1855
|
+
{
|
|
1856
|
+
id: "LEubPCm2octf2uMqCFKE",
|
|
1857
|
+
begin: "2026-08-18T16:27:00+00:00",
|
|
1858
|
+
end: "2026-08-21T08:49:00+00:00",
|
|
1859
|
+
external_desc: "August 2026 spam update"
|
|
1860
|
+
},
|
|
1861
|
+
{
|
|
1862
|
+
id: "YUX1peHev5a4fkxLDiUQ",
|
|
1863
|
+
begin: "2026-06-24T16:00:00+00:00",
|
|
1864
|
+
end: "2026-06-26T17:00:00+00:00",
|
|
1865
|
+
external_desc: "June 2026 spam update"
|
|
1866
|
+
},
|
|
1867
|
+
{
|
|
1868
|
+
id: "wdAXJk6LRRihEjpzEeWE",
|
|
1869
|
+
begin: "2026-05-21T15:40:00+00:00",
|
|
1870
|
+
end: "2026-06-02T12:40:00+00:00",
|
|
1871
|
+
external_desc: "May 2026 core update"
|
|
1872
|
+
},
|
|
1873
|
+
{
|
|
1874
|
+
id: "7eTbAa2jWdToLkraZj5y",
|
|
1875
|
+
begin: "2026-03-27T09:00:00+00:00",
|
|
1876
|
+
end: "2026-04-08T13:00:00+00:00",
|
|
1877
|
+
external_desc: "March 2026 core update"
|
|
1878
|
+
},
|
|
1879
|
+
{
|
|
1880
|
+
id: "VbnSXAH4SmEcxPtx4YSD",
|
|
1881
|
+
begin: "2026-03-24T19:00:00+00:00",
|
|
1882
|
+
end: "2026-03-25T14:30:00+00:00",
|
|
1883
|
+
external_desc: "March 2026 spam update"
|
|
1884
|
+
},
|
|
1885
|
+
{
|
|
1886
|
+
id: "mYbNTqV1ytDc2fA8hUz4",
|
|
1887
|
+
begin: "2026-02-05T17:00:00+00:00",
|
|
1888
|
+
end: "2026-02-27T10:00:00+00:00",
|
|
1889
|
+
external_desc: "February 2026 Discover update"
|
|
1890
|
+
},
|
|
1891
|
+
{
|
|
1892
|
+
id: "DsirqJ1gpPRgVQeccPRv",
|
|
1893
|
+
begin: "2025-12-11T17:25:00+00:00",
|
|
1894
|
+
end: "2025-12-29T19:00:00+00:00",
|
|
1895
|
+
external_desc: "December 2025 core update"
|
|
1896
|
+
},
|
|
1897
|
+
{
|
|
1898
|
+
id: "a7Aainy6E9rZsmfq82xt",
|
|
1899
|
+
begin: "2025-08-26T16:00:00+00:00",
|
|
1900
|
+
end: "2025-09-22T07:00:00+00:00",
|
|
1901
|
+
external_desc: "August 2025 spam update"
|
|
1902
|
+
},
|
|
1903
|
+
{
|
|
1904
|
+
id: "riq1AuqETW46NfBCe5NT",
|
|
1905
|
+
begin: "2025-06-30T14:30:00+00:00",
|
|
1906
|
+
end: "2025-07-17T08:00:00+00:00",
|
|
1907
|
+
external_desc: "June 2025 core update"
|
|
1908
|
+
},
|
|
1909
|
+
{
|
|
1910
|
+
id: "zpmwuSwifjDjfrVdaZUx",
|
|
1911
|
+
begin: "2025-03-13T16:00:00+00:00",
|
|
1912
|
+
end: "2025-03-27T12:35:00+00:00",
|
|
1913
|
+
external_desc: "March 2025 core update"
|
|
1914
|
+
},
|
|
1915
|
+
{
|
|
1916
|
+
id: "UUq2WSouY7PhSm8zvtD1",
|
|
1917
|
+
begin: "2024-12-19T17:00:00+00:00",
|
|
1918
|
+
end: "2024-12-26T19:00:00+00:00",
|
|
1919
|
+
external_desc: "December 2024 spam update"
|
|
1920
|
+
},
|
|
1921
|
+
{
|
|
1922
|
+
id: "V9nDKuo6nWKh2ThBALgA",
|
|
1923
|
+
begin: "2024-12-12T15:30:00+00:00",
|
|
1924
|
+
end: "2024-12-18T19:00:00+00:00",
|
|
1925
|
+
external_desc: "December 2024 core update"
|
|
1926
|
+
},
|
|
1927
|
+
{
|
|
1928
|
+
id: "G7rp11wqTaTGn6JjiPF3",
|
|
1929
|
+
begin: "2024-11-11T19:00:00+00:00",
|
|
1930
|
+
end: "2024-12-05T08:00:00+00:00",
|
|
1931
|
+
external_desc: "November 2024 core update"
|
|
1932
|
+
},
|
|
1933
|
+
{
|
|
1934
|
+
id: "42XCLGtpuCZkzqe4dc6S",
|
|
1935
|
+
begin: "2024-08-16T00:15:00+00:00",
|
|
1936
|
+
end: "2024-08-20T11:30:00+00:00",
|
|
1937
|
+
external_desc: "Ranking is experiencing an ongoing issue."
|
|
1938
|
+
},
|
|
1939
|
+
{
|
|
1940
|
+
id: "gVx6b2o78zke7GrMidGy",
|
|
1941
|
+
begin: "2024-08-15T14:58:00+00:00",
|
|
1942
|
+
end: "2024-09-03T19:00:00+00:00",
|
|
1943
|
+
external_desc: "August 2024 core update"
|
|
1944
|
+
},
|
|
1945
|
+
{
|
|
1946
|
+
id: "QdUeCQx3LRVbzQ3E7FiD",
|
|
1947
|
+
begin: "2024-06-20T15:12:00+00:00",
|
|
1948
|
+
end: "2024-06-27T16:11:00+00:00",
|
|
1949
|
+
external_desc: "June 2024 spam update"
|
|
1950
|
+
},
|
|
1951
|
+
{
|
|
1952
|
+
id: "iXz2PJfodvyjaVUeqxZE",
|
|
1953
|
+
begin: "2024-03-05T17:03:00+00:00",
|
|
1954
|
+
end: "2024-03-20T14:09:00+00:00",
|
|
1955
|
+
external_desc: "March 2024 spam update"
|
|
1956
|
+
},
|
|
1957
|
+
{
|
|
1958
|
+
id: "1jW2F89qC2NxJBWGiKxE",
|
|
1959
|
+
begin: "2024-03-05T17:02:00+00:00",
|
|
1960
|
+
end: "2024-04-19T17:00:00+00:00",
|
|
1961
|
+
external_desc: "March 2024 core update"
|
|
1962
|
+
},
|
|
1963
|
+
{
|
|
1964
|
+
id: "VqRTcmyQCwfpuYeaVNfn",
|
|
1965
|
+
begin: "2023-11-08T16:58:00+00:00",
|
|
1966
|
+
end: "2023-12-07T17:00:00+00:00",
|
|
1967
|
+
external_desc: "November 2023 reviews update"
|
|
1968
|
+
},
|
|
1969
|
+
{
|
|
1970
|
+
id: "WtY1CQgfqrr2cjxKbyUk",
|
|
1971
|
+
begin: "2023-11-02T19:09:00+00:00",
|
|
1972
|
+
end: "2023-11-28T16:30:00+00:00",
|
|
1973
|
+
external_desc: "November 2023 core update"
|
|
1974
|
+
},
|
|
1975
|
+
{
|
|
1976
|
+
id: "VKyoS53ULWzSp8BoDxqk",
|
|
1977
|
+
begin: "2023-10-05T16:00:00+00:00",
|
|
1978
|
+
end: "2023-10-19T15:00:00+00:00",
|
|
1979
|
+
external_desc: "October 2023 core update"
|
|
1980
|
+
},
|
|
1981
|
+
{
|
|
1982
|
+
id: "fCUAy6TvbDMkkLAGcwkj",
|
|
1983
|
+
begin: "2023-10-05T07:00:00+00:00",
|
|
1984
|
+
end: "2023-10-31T07:00:00+00:00",
|
|
1985
|
+
external_desc: "Ranking is experiencing an ongoing issue"
|
|
1986
|
+
},
|
|
1987
|
+
{
|
|
1988
|
+
id: "NzcEhGMDhbQEdXCS35xL",
|
|
1989
|
+
begin: "2023-10-04T16:51:00+00:00",
|
|
1990
|
+
end: "2023-10-20T05:15:00+00:00",
|
|
1991
|
+
external_desc: "October 2023 spam update"
|
|
1992
|
+
},
|
|
1993
|
+
{
|
|
1994
|
+
id: "53diuQvcEsgzqXTPBb8p",
|
|
1995
|
+
begin: "2023-09-14T20:19:00+00:00",
|
|
1996
|
+
end: "2023-09-28T07:07:00+00:00",
|
|
1997
|
+
external_desc: "September 2023 helpful content update"
|
|
1998
|
+
},
|
|
1999
|
+
{
|
|
2000
|
+
id: "nBtYtBeex4GYBbdDS2LX",
|
|
2001
|
+
begin: "2023-08-22T17:30:00+00:00",
|
|
2002
|
+
end: "2023-09-07T20:03:00+00:00",
|
|
2003
|
+
external_desc: "August 2023 core update"
|
|
2004
|
+
},
|
|
2005
|
+
{
|
|
2006
|
+
id: "5XRfC46rorevFt8yN8iR",
|
|
2007
|
+
begin: "2023-04-12T16:00:00+00:00",
|
|
2008
|
+
end: "2023-04-25T18:24:00+00:00",
|
|
2009
|
+
external_desc: "April 2023 reviews update"
|
|
2010
|
+
},
|
|
2011
|
+
{
|
|
2012
|
+
id: "Cou8Tr74r7EXNthuEsaG",
|
|
2013
|
+
begin: "2023-03-15T07:00:00+00:00",
|
|
2014
|
+
end: "2023-03-28T14:26:00+00:00",
|
|
2015
|
+
external_desc: "March 2023 core update"
|
|
2016
|
+
},
|
|
2017
|
+
{
|
|
2018
|
+
id: "NDjZ11MEUJb5i62hy5aN",
|
|
2019
|
+
begin: "2023-02-21T08:00:00+00:00",
|
|
2020
|
+
end: "2023-03-07T08:00:00+00:00",
|
|
2021
|
+
external_desc: "February 2023 product reviews update"
|
|
2022
|
+
},
|
|
2023
|
+
{
|
|
2024
|
+
id: "CxXUEK28vCKqkNocPhKR",
|
|
2025
|
+
begin: "2022-12-14T08:00:00+00:00",
|
|
2026
|
+
end: "2023-01-12T08:00:00+00:00",
|
|
2027
|
+
external_desc: "December 2022 link spam update"
|
|
2028
|
+
},
|
|
2029
|
+
{
|
|
2030
|
+
id: "XMdX5PmbMvKW4fv3CRvn",
|
|
2031
|
+
begin: "2022-12-05T08:00:00+00:00",
|
|
2032
|
+
end: "2023-01-12T08:00:00+00:00",
|
|
2033
|
+
external_desc: "December 2022 helpful content update"
|
|
2034
|
+
},
|
|
2035
|
+
{
|
|
2036
|
+
id: "SAeYuxVSo39HHotQNXHr",
|
|
2037
|
+
begin: "2022-10-19T07:00:00+00:00",
|
|
2038
|
+
end: "2022-10-21T07:00:00+00:00",
|
|
2039
|
+
external_desc: "October 2022 spam update"
|
|
2040
|
+
},
|
|
2041
|
+
{
|
|
2042
|
+
id: "ZYwuBXDhkwaoop2V5epM",
|
|
2043
|
+
begin: "2022-09-20T07:00:00+00:00",
|
|
2044
|
+
end: "2022-09-26T07:00:00+00:00",
|
|
2045
|
+
external_desc: "September 2022 product reviews update"
|
|
2046
|
+
},
|
|
2047
|
+
{
|
|
2048
|
+
id: "Y2Cg5k2xLJvfNqR58kEt",
|
|
2049
|
+
begin: "2022-09-12T07:00:00+00:00",
|
|
2050
|
+
end: "2022-09-26T07:00:00+00:00",
|
|
2051
|
+
external_desc: "September 2022 core update"
|
|
2052
|
+
},
|
|
2053
|
+
{
|
|
2054
|
+
id: "sbKc4YSptrSxcUZFDKwU",
|
|
2055
|
+
begin: "2022-08-25T07:00:00+00:00",
|
|
2056
|
+
end: "2022-09-09T07:00:00+00:00",
|
|
2057
|
+
external_desc: "August 2022 helpful content update"
|
|
2058
|
+
},
|
|
2059
|
+
{
|
|
2060
|
+
id: "Xiid524od7fpCak86sNS",
|
|
2061
|
+
begin: "2022-07-27T07:00:00+00:00",
|
|
2062
|
+
end: "2022-08-02T07:00:00+00:00",
|
|
2063
|
+
external_desc: "July 2022 product reviews update"
|
|
2064
|
+
},
|
|
2065
|
+
{
|
|
2066
|
+
id: "GHwLLtDS64a7nDng8xhD",
|
|
2067
|
+
begin: "2022-05-25T07:00:00+00:00",
|
|
2068
|
+
end: "2022-06-09T07:00:00+00:00",
|
|
2069
|
+
external_desc: "May 2022 core update"
|
|
2070
|
+
},
|
|
2071
|
+
{
|
|
2072
|
+
id: "iYWoNu4QXRsQUrzxrbLv",
|
|
2073
|
+
begin: "2022-03-23T07:00:00+00:00",
|
|
2074
|
+
end: "2022-04-06T07:00:00+00:00",
|
|
2075
|
+
external_desc: "March 2022 product reviews update"
|
|
2076
|
+
},
|
|
2077
|
+
{
|
|
2078
|
+
id: "zDCQR5mxLU3ZrQeQvwqN",
|
|
2079
|
+
begin: "2022-02-22T08:00:00+00:00",
|
|
2080
|
+
end: "2022-03-03T08:00:00+00:00",
|
|
2081
|
+
external_desc: "Page experience update for desktop"
|
|
2082
|
+
},
|
|
2083
|
+
{
|
|
2084
|
+
id: "hgwP7ei5H1uKbYSeZrBZ",
|
|
2085
|
+
begin: "2021-12-01T08:00:00+00:00",
|
|
2086
|
+
end: "2021-12-21T08:00:00+00:00",
|
|
2087
|
+
external_desc: "December 2021 product reviews update"
|
|
2088
|
+
},
|
|
2089
|
+
{
|
|
2090
|
+
id: "FncPfR7zRSZ3diemah4o",
|
|
2091
|
+
begin: "2021-11-17T08:00:00+00:00",
|
|
2092
|
+
end: "2021-11-30T08:00:00+00:00",
|
|
2093
|
+
external_desc: "November 2021 core update"
|
|
2094
|
+
},
|
|
2095
|
+
{
|
|
2096
|
+
id: "AFdXceQVsh1GEfKFvHyD",
|
|
2097
|
+
begin: "2021-11-03T07:00:00+00:00",
|
|
2098
|
+
end: "2021-11-11T08:00:00+00:00",
|
|
2099
|
+
external_desc: "November 2021 spam update"
|
|
2100
|
+
}
|
|
2101
|
+
];
|
|
2102
|
+
//#endregion
|
|
1847
2103
|
//#region packages/core/src/updates/search-updates.ts
|
|
1848
2104
|
function classifyUpdate(name, product) {
|
|
1849
2105
|
const lower = `${name} ${product}`.toLowerCase();
|
|
@@ -1871,6 +2127,18 @@ function searchUpdatesFromIncidents(incidents, input = {}) {
|
|
|
1871
2127
|
};
|
|
1872
2128
|
}).slice(0, input.limit ?? 25);
|
|
1873
2129
|
}
|
|
2130
|
+
function withRankingHistory(live) {
|
|
2131
|
+
const byId = /* @__PURE__ */ new Map();
|
|
2132
|
+
for (const saved of RANKING_HISTORY) byId.set(saved.id, {
|
|
2133
|
+
...saved,
|
|
2134
|
+
service_name: "Ranking",
|
|
2135
|
+
status_impact: "SERVICE_INFORMATION",
|
|
2136
|
+
severity: "low",
|
|
2137
|
+
uri: `incidents/${saved.id}`
|
|
2138
|
+
});
|
|
2139
|
+
for (const incident of live) byId.set(incident.id, incident);
|
|
2140
|
+
return [...byId.values()].sort((left, right) => Date.parse(right.begin) - Date.parse(left.begin) || (left.id < right.id ? -1 : 1));
|
|
2141
|
+
}
|
|
1874
2142
|
function findOverlappingSearchUpdates(input) {
|
|
1875
2143
|
const paddingMs = (input.paddingDays ?? 3) * 864e5;
|
|
1876
2144
|
const windowStart = new Date(input.startDate).getTime() - paddingMs;
|
|
@@ -6984,6 +7252,12 @@ function createRobotsSession(input = {}) {
|
|
|
6984
7252
|
};
|
|
6985
7253
|
return {
|
|
6986
7254
|
resolve: async (origin, targetUrl) => resultFromSource(await sourceFor(origin), targetUrl),
|
|
7255
|
+
crawlDelay: async (origin) => {
|
|
7256
|
+
const source = await sourceFor(origin);
|
|
7257
|
+
if (source.status === void 0 || source.status >= 300 || !source.text) return;
|
|
7258
|
+
const delay = parseRobots$1(source.robotsUrl, source.text).getCrawlDelay(SEO_CRAWLER_USER_AGENT);
|
|
7259
|
+
return delay !== void 0 && Number.isFinite(delay) && delay > 0 ? delay : void 0;
|
|
7260
|
+
},
|
|
6987
7261
|
sitemapUrls: async (origin) => {
|
|
6988
7262
|
const source = await sourceFor(origin);
|
|
6989
7263
|
if (source.status === void 0 || source.status >= 300 || !source.text) return [];
|
|
@@ -7944,7 +8218,7 @@ function finalizeUrlInspectionQuota(input) {
|
|
|
7944
8218
|
function credentialKey(input) {
|
|
7945
8219
|
return createHash("sha256").update(`${input.clientId}\n${input.accountEmail.toLowerCase()}`).digest("base64url");
|
|
7946
8220
|
}
|
|
7947
|
-
function retryAfter(response, now) {
|
|
8221
|
+
function retryAfter$1(response, now) {
|
|
7948
8222
|
const value = response.headers.get("retry-after");
|
|
7949
8223
|
if (!value) return void 0;
|
|
7950
8224
|
const seconds = Number(value);
|
|
@@ -7987,7 +8261,7 @@ async function inspectUrl(input) {
|
|
|
7987
8261
|
}
|
|
7988
8262
|
if (response.status === 429) {
|
|
7989
8263
|
const now = /* @__PURE__ */ new Date();
|
|
7990
|
-
const blockedUntil = retryAfter(response, now);
|
|
8264
|
+
const blockedUntil = retryAfter$1(response, now);
|
|
7991
8265
|
finalizeUrlInspectionQuota({
|
|
7992
8266
|
reservation,
|
|
7993
8267
|
outcome: "exhausted",
|
|
@@ -8372,7 +8646,7 @@ const AUDIT_PAGE_RULE_IDS = [
|
|
|
8372
8646
|
"canonical_streamed_in_body",
|
|
8373
8647
|
"canonical_mismatch"
|
|
8374
8648
|
];
|
|
8375
|
-
const defaultDependencies$
|
|
8649
|
+
const defaultDependencies$12 = {
|
|
8376
8650
|
fetchPage,
|
|
8377
8651
|
queryPageMetrics,
|
|
8378
8652
|
now: () => /* @__PURE__ */ new Date()
|
|
@@ -8403,7 +8677,7 @@ function buildRecommendations(report) {
|
|
|
8403
8677
|
});
|
|
8404
8678
|
return recommendations;
|
|
8405
8679
|
}
|
|
8406
|
-
async function auditPage(input, dependencies = defaultDependencies$
|
|
8680
|
+
async function auditPage(input, dependencies = defaultDependencies$12) {
|
|
8407
8681
|
const fetched = await dependencies.fetchPage(input.url, {
|
|
8408
8682
|
js: input.js,
|
|
8409
8683
|
refresh: input.refresh
|
|
@@ -8522,7 +8796,7 @@ async function auditPage(input, dependencies = defaultDependencies$13) {
|
|
|
8522
8796
|
}
|
|
8523
8797
|
//#endregion
|
|
8524
8798
|
//#region packages/core/src/analyze/content-coverage-analysis.ts
|
|
8525
|
-
const STOPWORDS
|
|
8799
|
+
const STOPWORDS = /* @__PURE__ */ new Set([
|
|
8526
8800
|
"a",
|
|
8527
8801
|
"an",
|
|
8528
8802
|
"and",
|
|
@@ -8551,7 +8825,7 @@ function hasNonAscii(value) {
|
|
|
8551
8825
|
}
|
|
8552
8826
|
function queryTerms$1(query) {
|
|
8553
8827
|
const tokens = normalizeForCoverage(query).split(" ").filter((token) => token.length > 1 || hasNonAscii(token));
|
|
8554
|
-
const meaningful = tokens.filter((token) => (token.length > 2 || hasNonAscii(token)) && !STOPWORDS
|
|
8828
|
+
const meaningful = tokens.filter((token) => (token.length > 2 || hasNonAscii(token)) && !STOPWORDS.has(token));
|
|
8555
8829
|
return [...new Set(meaningful.length ? meaningful : tokens)];
|
|
8556
8830
|
}
|
|
8557
8831
|
function countPhrase$1(query, text) {
|
|
@@ -9858,7 +10132,7 @@ const actionPriority = new Map([
|
|
|
9858
10132
|
"unverified",
|
|
9859
10133
|
"covered"
|
|
9860
10134
|
].map((type, index) => [type, index]));
|
|
9861
|
-
function plural$
|
|
10135
|
+
function plural$6(count, singular, pluralLabel) {
|
|
9862
10136
|
return count === 1 ? singular : pluralLabel;
|
|
9863
10137
|
}
|
|
9864
10138
|
function actionable(item) {
|
|
@@ -9892,7 +10166,7 @@ function pageOpportunityVerdict(input) {
|
|
|
9892
10166
|
if (focus === "unverified") return "GSC visibility exists, but the page was not verified and no specific change is justified yet.";
|
|
9893
10167
|
if (focus === "covered") return "The verified page covers the sampled queries and CTR is not materially below its directional benchmark.";
|
|
9894
10168
|
const count = analysis.summary.opportunities;
|
|
9895
|
-
const prefix = `${count} ${plural$
|
|
10169
|
+
const prefix = `${count} ${plural$6(count, "actionable observation", "actionable observations")} found.`;
|
|
9896
10170
|
if (focus === "content-gap") return `${prefix} Verified body coverage is the main issue; confirm intent before expanding the page.`;
|
|
9897
10171
|
if (focus === "serp-framing") return `${prefix} The body is relevant, but its title, meta description, or H1 framing is weak.`;
|
|
9898
10172
|
if (focus === "ranking") return `${prefix} Most upside requires stronger relevance or authority; no CTR-only lift is claimed for page-two queries.`;
|
|
@@ -9907,7 +10181,7 @@ function pageOpportunityRecommendations(analysis) {
|
|
|
9907
10181
|
}
|
|
9908
10182
|
const recommendations = [`Start with "${top.query}". ${top.recommendation}`];
|
|
9909
10183
|
const technical = actions.filter((item) => item.opportunityType === "technical-check").length;
|
|
9910
|
-
if (technical) recommendations.push(`Resolve the technical evidence affecting ${technical} ${plural$
|
|
10184
|
+
if (technical) recommendations.push(`Resolve the technical evidence affecting ${technical} ${plural$6(technical, "row", "rows")} before content work.`);
|
|
9911
10185
|
const gaps = actions.filter((item) => item.opportunityType === "content-gap").length;
|
|
9912
10186
|
if (gaps >= 3) recommendations.push(`Group the ${gaps} verified content gaps into useful sections rather than adding one thin paragraph per query.`);
|
|
9913
10187
|
return recommendations;
|
|
@@ -9915,13 +10189,13 @@ function pageOpportunityRecommendations(analysis) {
|
|
|
9915
10189
|
//#endregion
|
|
9916
10190
|
//#region packages/core/src/analyze/page-opportunities.ts
|
|
9917
10191
|
const MAX_BENCHMARK_ROWS = 1e5;
|
|
9918
|
-
const defaultDependencies$
|
|
10192
|
+
const defaultDependencies$11 = {
|
|
9919
10193
|
searchAnalytics: querySearchAnalytics,
|
|
9920
10194
|
fetch: fetchPage,
|
|
9921
10195
|
extract: extractPage,
|
|
9922
10196
|
now: () => /* @__PURE__ */ new Date()
|
|
9923
10197
|
};
|
|
9924
|
-
function errorMessage
|
|
10198
|
+
function errorMessage(error) {
|
|
9925
10199
|
return error instanceof Error ? error.message : String(error);
|
|
9926
10200
|
}
|
|
9927
10201
|
function unique$3(values) {
|
|
@@ -9942,7 +10216,7 @@ async function verifyPage(input) {
|
|
|
9942
10216
|
});
|
|
9943
10217
|
warnings.push(...fetched.warnings);
|
|
9944
10218
|
} catch (error) {
|
|
9945
|
-
const reason = `Page fetch failed: ${errorMessage
|
|
10219
|
+
const reason = `Page fetch failed: ${errorMessage(error)}`;
|
|
9946
10220
|
return {
|
|
9947
10221
|
verification: {
|
|
9948
10222
|
status: "failed",
|
|
@@ -9964,7 +10238,7 @@ async function verifyPage(input) {
|
|
|
9964
10238
|
warnings: unique$3(warnings)
|
|
9965
10239
|
};
|
|
9966
10240
|
} catch (error) {
|
|
9967
|
-
const reason = `Page extraction failed: ${errorMessage
|
|
10241
|
+
const reason = `Page extraction failed: ${errorMessage(error)}`;
|
|
9968
10242
|
return {
|
|
9969
10243
|
fetched,
|
|
9970
10244
|
verification: {
|
|
@@ -9987,7 +10261,7 @@ function reportCaveats$1(input) {
|
|
|
9987
10261
|
input.benchmarkTruncated ? `The site-wide benchmark reached its ${MAX_BENCHMARK_ROWS.toLocaleString("en-US")}-row cap, so provenance is complete but the peer sample is truncated.` : ""
|
|
9988
10262
|
].filter(Boolean);
|
|
9989
10263
|
}
|
|
9990
|
-
async function pageOpportunitiesReport(input, dependencies = defaultDependencies$
|
|
10264
|
+
async function pageOpportunitiesReport(input, dependencies = defaultDependencies$11) {
|
|
9991
10265
|
const url = assertUrlMatchesGscProperty(input.site, input.url);
|
|
9992
10266
|
const days = reportDays$1(input.days);
|
|
9993
10267
|
const range = defaultDateRange$1(days);
|
|
@@ -16050,7 +16324,7 @@ async function loadInstalledProviderExtensions(options = {}) {
|
|
|
16050
16324
|
//#endregion
|
|
16051
16325
|
//#region packages/core/src/provider-extensions/runtime.ts
|
|
16052
16326
|
const DEFAULT_MAX_REQUESTS = 8;
|
|
16053
|
-
const MAX_RESPONSE_BYTES = 10 * 1024 * 1024;
|
|
16327
|
+
const MAX_RESPONSE_BYTES$1 = 10 * 1024 * 1024;
|
|
16054
16328
|
const MAX_REQUEST_BODY_BYTES = 64 * 1024;
|
|
16055
16329
|
const TIMEOUT_MS$1 = 15e3;
|
|
16056
16330
|
function normalizeProviderExtensionAccount(provider, account) {
|
|
@@ -16098,7 +16372,7 @@ function runtimeForProvider(provider, options = {}) {
|
|
|
16098
16372
|
...input.body ? { body: input.body } : {}
|
|
16099
16373
|
},
|
|
16100
16374
|
fetch: providerFetch,
|
|
16101
|
-
maxResponseBytes: MAX_RESPONSE_BYTES,
|
|
16375
|
+
maxResponseBytes: MAX_RESPONSE_BYTES$1,
|
|
16102
16376
|
timeoutMs: TIMEOUT_MS$1,
|
|
16103
16377
|
retry: (input.method ?? "GET") === "GET" ? "safe" : "never",
|
|
16104
16378
|
schema: z.unknown()
|
|
@@ -16203,7 +16477,7 @@ async function runAnalyticsProviderLandingPages(input) {
|
|
|
16203
16477
|
}
|
|
16204
16478
|
//#endregion
|
|
16205
16479
|
//#region packages/core/src/analyze/workflows/analytics-value.ts
|
|
16206
|
-
const defaultDependencies$
|
|
16480
|
+
const defaultDependencies$10 = {
|
|
16207
16481
|
runGa4Report,
|
|
16208
16482
|
analyticsLandingPages: runAnalyticsProviderLandingPages
|
|
16209
16483
|
};
|
|
@@ -16274,7 +16548,7 @@ function landingPageRankingPolicy(input) {
|
|
|
16274
16548
|
warnings
|
|
16275
16549
|
};
|
|
16276
16550
|
}
|
|
16277
|
-
async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
16551
|
+
async function fetchLandingPageValues(input, dependencies = defaultDependencies$10) {
|
|
16278
16552
|
const retainedRowLimit = input.limit ?? 5e3;
|
|
16279
16553
|
const connection = input.connection ?? (input.propertyId ? {
|
|
16280
16554
|
provider: "google",
|
|
@@ -16293,7 +16567,7 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
16293
16567
|
}
|
|
16294
16568
|
};
|
|
16295
16569
|
if (connection.provider === "extension") try {
|
|
16296
|
-
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$
|
|
16570
|
+
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$10.analyticsLandingPages)({
|
|
16297
16571
|
providerId: connection.providerId,
|
|
16298
16572
|
account: connection.account,
|
|
16299
16573
|
startDate: input.startDate,
|
|
@@ -16333,7 +16607,7 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
16333
16607
|
clicky: async (selected) => {
|
|
16334
16608
|
if (selected.provider !== "clicky") throw new Error("Clicky analytics connection mismatch.");
|
|
16335
16609
|
try {
|
|
16336
|
-
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$
|
|
16610
|
+
const result = await (dependencies.analyticsLandingPages ?? defaultDependencies$10.analyticsLandingPages)({
|
|
16337
16611
|
providerId: "clicky",
|
|
16338
16612
|
account: { siteId: selected.siteId },
|
|
16339
16613
|
startDate: input.startDate,
|
|
@@ -16372,7 +16646,7 @@ async function fetchLandingPageValues(input, dependencies = defaultDependencies$
|
|
|
16372
16646
|
google: async (selected) => {
|
|
16373
16647
|
if (selected.provider !== "google") throw new Error("Google Analytics connection mismatch.");
|
|
16374
16648
|
try {
|
|
16375
|
-
const result = await (dependencies.runGa4Report ?? defaultDependencies$
|
|
16649
|
+
const result = await (dependencies.runGa4Report ?? defaultDependencies$10.runGa4Report)(selected.propertyId, {
|
|
16376
16650
|
dateRanges: [{
|
|
16377
16651
|
startDate: input.startDate,
|
|
16378
16652
|
endDate: input.endDate
|
|
@@ -17153,7 +17427,7 @@ const NAVIGATIONAL_STEM_THRESHOLDS = {
|
|
|
17153
17427
|
function compact(value) {
|
|
17154
17428
|
return normalizeBrandText(value).replace(/\s+/g, "");
|
|
17155
17429
|
}
|
|
17156
|
-
function round$
|
|
17430
|
+
function round$2(value, digits) {
|
|
17157
17431
|
return Number(value.toFixed(digits));
|
|
17158
17432
|
}
|
|
17159
17433
|
function stemQueryEvidence(rows, stem) {
|
|
@@ -17180,8 +17454,8 @@ function stemQueryEvidence(rows, stem) {
|
|
|
17180
17454
|
query,
|
|
17181
17455
|
clicks,
|
|
17182
17456
|
impressions,
|
|
17183
|
-
ctr: round$
|
|
17184
|
-
position: round$
|
|
17457
|
+
ctr: round$2(clicks / impressions, 4),
|
|
17458
|
+
position: round$2(weightedPosition / impressions, 2),
|
|
17185
17459
|
sourceRows
|
|
17186
17460
|
};
|
|
17187
17461
|
}
|
|
@@ -17678,9 +17952,9 @@ function analyzeCtrUnderperformersFromRows(input) {
|
|
|
17678
17952
|
//#endregion
|
|
17679
17953
|
//#region packages/core/src/analyze/ctr-underperformers-report.ts
|
|
17680
17954
|
const MAX_GSC_ROWS$3 = 1e5;
|
|
17681
|
-
const DEFAULT_DAYS$
|
|
17955
|
+
const DEFAULT_DAYS$3 = 28;
|
|
17682
17956
|
const MAX_DAYS$2 = 548;
|
|
17683
|
-
function plural$
|
|
17957
|
+
function plural$5(count, singular, pluralLabel = `${singular}s`) {
|
|
17684
17958
|
return count === 1 ? singular : pluralLabel;
|
|
17685
17959
|
}
|
|
17686
17960
|
async function ctrUnderperformersReportWith(input, dependencies) {
|
|
@@ -17701,7 +17975,7 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
17701
17975
|
const explicitRange = explicitDateRange(input, MAX_DAYS$2);
|
|
17702
17976
|
const days = explicitRange?.days ?? integerOption$1({
|
|
17703
17977
|
value: input.days,
|
|
17704
|
-
fallback: DEFAULT_DAYS$
|
|
17978
|
+
fallback: DEFAULT_DAYS$3,
|
|
17705
17979
|
minimum: 1,
|
|
17706
17980
|
maximum: MAX_DAYS$2,
|
|
17707
17981
|
label: "Days"
|
|
@@ -17783,11 +18057,11 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
17783
18057
|
minImpressions,
|
|
17784
18058
|
limit,
|
|
17785
18059
|
brandFiltering: input.includeBrand ? "included" : "excluded",
|
|
17786
|
-
verdict: top ? `${selection.eligibleUnderperformers} CTR ${plural$
|
|
18060
|
+
verdict: top ? `${selection.eligibleUnderperformers} CTR ${plural$5(selection.eligibleUnderperformers, "underperformer")} found in retained rows, with a calculated heuristic shortfall of ${totalClickShortfall.toFixed(0)} clicks for this window. Start review with "${top.query}" because it has the largest gap.` : dataStatus === "partial" ? "No material CTR underperformers remained in the validated retained rows, but partial evidence prevents an all-clear." : `No high-impression rows at average position below 4 in the retained rows are materially below the heuristic CTR benchmark.`
|
|
17787
18061
|
},
|
|
17788
18062
|
items,
|
|
17789
18063
|
caveats: [
|
|
17790
|
-
`Date window: ${range.startDate} to ${range.endDate} (${days} ${plural$
|
|
18064
|
+
`Date window: ${range.startDate} to ${range.endDate} (${days} ${plural$5(days, "day")}), using final GSC data where available.`,
|
|
17791
18065
|
`Brand queries: ${input.includeBrand ? "included" : "excluded"}. Brand terms used: ${brand.terms.join(", ") || "none"} (${brand.source === "provided" ? "provided" : "host forms, plus the bare domain stem only when Search Console shows the site is its navigational answer"}).`,
|
|
17792
18066
|
`Only rows at average position below 4 with at least ${minImpressions} impressions were checked. Rows at position 4 to 10 belong to quick-wins, which uses the same site CTR curve.`,
|
|
17793
18067
|
minImpressionsCaveat({
|
|
@@ -17795,13 +18069,13 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
17795
18069
|
belowMinimumRows: selection.belowMinimumRows
|
|
17796
18070
|
}),
|
|
17797
18071
|
"Expected CTR uses the site CTR curve from all page-one rows (75th percentile of qualified URL CTRs per position) when enough peer data exists, otherwise the fallback position curve.",
|
|
17798
|
-
...selection.fragmentRows ? [`${selection.fragmentRows} source ${plural$
|
|
17799
|
-
...selection.flaggedRows ? [`${selection.flaggedRows} eligible ${plural$
|
|
18072
|
+
...selection.fragmentRows ? [`${selection.fragmentRows} source ${plural$5(selection.fragmentRows, "row")} with a #fragment URL ${selection.fragmentRows === 1 ? "was" : "were"} merged into the base URL.`] : [],
|
|
18073
|
+
...selection.flaggedRows ? [`${selection.flaggedRows} eligible ${plural$5(selection.flaggedRows, "row")} carry heuristic query flags (likely-automated or navigational-other) and rank after unflagged rows.`] : [],
|
|
17800
18074
|
"CTR benchmarks and calculated click shortfalls are directional heuristics, not forecasts or proof that clicks are available. Validate the search intent and live SERP before editing.",
|
|
17801
|
-
...selection.invalidRows ? [`${selection.invalidRows} invalid provider ${plural$
|
|
17802
|
-
...selection.duplicateRows ? [`${selection.duplicateRows} repeated query/page ${plural$
|
|
18075
|
+
...selection.invalidRows ? [`${selection.invalidRows} invalid provider ${plural$5(selection.invalidRows, "row")} ${selection.invalidRows === 1 ? "was" : "were"} excluded, so this report is partial.`] : [],
|
|
18076
|
+
...selection.duplicateRows ? [`${selection.duplicateRows} repeated query/page ${plural$5(selection.duplicateRows, "row")} ${selection.duplicateRows === 1 ? "was" : "were"} aggregated before CTR and average position were calculated.`] : [],
|
|
17803
18077
|
...possiblyTruncated ? [`The retained-row limit of ${MAX_GSC_ROWS$3} was reached, so lower-ranked source rows may be absent.`] : [],
|
|
17804
|
-
...selection.limitedUnderperformers ? [`${selection.limitedUnderperformers} eligible ${plural$
|
|
18078
|
+
...selection.limitedUnderperformers ? [`${selection.limitedUnderperformers} eligible ${plural$5(selection.limitedUnderperformers, "underperformer")} ${selection.limitedUnderperformers === 1 ? "was" : "were"} omitted by the output limit.`] : []
|
|
17805
18079
|
],
|
|
17806
18080
|
recommendations: top ? [
|
|
17807
18081
|
`Review the live SERP and displayed snippet for "${top.query}" first. Test the title or meta description only if the observed framing does not match the query intent.`,
|
|
@@ -17812,11 +18086,11 @@ async function ctrUnderperformersReportWith(input, dependencies) {
|
|
|
17812
18086
|
}
|
|
17813
18087
|
//#endregion
|
|
17814
18088
|
//#region packages/core/src/analyze/ctr-underperformers.ts
|
|
17815
|
-
const defaultDependencies$
|
|
18089
|
+
const defaultDependencies$9 = {
|
|
17816
18090
|
searchAnalytics: querySearchAnalytics,
|
|
17817
18091
|
now: () => /* @__PURE__ */ new Date()
|
|
17818
18092
|
};
|
|
17819
|
-
function ctrUnderperformersReport(input, dependencies = defaultDependencies$
|
|
18093
|
+
function ctrUnderperformersReport(input, dependencies = defaultDependencies$9) {
|
|
17820
18094
|
return ctrUnderperformersReportWith(input, dependencies);
|
|
17821
18095
|
}
|
|
17822
18096
|
//#endregion
|
|
@@ -18284,7 +18558,7 @@ function unavailableSegmentImpactReport(input) {
|
|
|
18284
18558
|
//#endregion
|
|
18285
18559
|
//#region packages/core/src/analyze/segment-impact.ts
|
|
18286
18560
|
const DEFAULT_MAX_ROWS = 1e5;
|
|
18287
|
-
const defaultDependencies$
|
|
18561
|
+
const defaultDependencies$8 = {
|
|
18288
18562
|
searchAnalytics: querySearchAnalytics,
|
|
18289
18563
|
now: () => /* @__PURE__ */ new Date()
|
|
18290
18564
|
};
|
|
@@ -18324,7 +18598,7 @@ function compareSegmentRows(input) {
|
|
|
18324
18598
|
generatedAt: input.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString()
|
|
18325
18599
|
});
|
|
18326
18600
|
}
|
|
18327
|
-
async function segmentImpact(input, dependencies = defaultDependencies$
|
|
18601
|
+
async function segmentImpact(input, dependencies = defaultDependencies$8) {
|
|
18328
18602
|
validateSegmentSite(input.site);
|
|
18329
18603
|
const now = dependencies.now();
|
|
18330
18604
|
const ranges = resolveSegmentRanges({
|
|
@@ -20182,11 +20456,11 @@ async function cannibalReportWith(input, dependencies) {
|
|
|
20182
20456
|
}
|
|
20183
20457
|
//#endregion
|
|
20184
20458
|
//#region packages/core/src/analyze/site-diagnostics/cannibal.ts
|
|
20185
|
-
const defaultDependencies$
|
|
20459
|
+
const defaultDependencies$7 = {
|
|
20186
20460
|
searchAnalytics: querySearchAnalytics,
|
|
20187
20461
|
now: () => /* @__PURE__ */ new Date()
|
|
20188
20462
|
};
|
|
20189
|
-
async function cannibalReport(input, dependencies = defaultDependencies$
|
|
20463
|
+
async function cannibalReport(input, dependencies = defaultDependencies$7) {
|
|
20190
20464
|
if (input.researchFiles) {
|
|
20191
20465
|
const conflicts = [
|
|
20192
20466
|
["days", input.days],
|
|
@@ -20644,11 +20918,11 @@ async function decayingReportWith(input, dependencies) {
|
|
|
20644
20918
|
}
|
|
20645
20919
|
//#endregion
|
|
20646
20920
|
//#region packages/core/src/analyze/site-diagnostics/decaying.ts
|
|
20647
|
-
const defaultDependencies$
|
|
20921
|
+
const defaultDependencies$6 = {
|
|
20648
20922
|
searchAnalytics: querySearchAnalytics,
|
|
20649
20923
|
now: () => /* @__PURE__ */ new Date()
|
|
20650
20924
|
};
|
|
20651
|
-
function decayingReport(input, dependencies = defaultDependencies$
|
|
20925
|
+
function decayingReport(input, dependencies = defaultDependencies$6) {
|
|
20652
20926
|
return decayingReportWith(input, dependencies);
|
|
20653
20927
|
}
|
|
20654
20928
|
//#endregion
|
|
@@ -20958,7 +21232,7 @@ function quickWinTemplateSummaries(items) {
|
|
|
20958
21232
|
}
|
|
20959
21233
|
//#endregion
|
|
20960
21234
|
//#region packages/core/src/analyze/site-diagnostics/quick-wins-report.ts
|
|
20961
|
-
const DEFAULT_DAYS$
|
|
21235
|
+
const DEFAULT_DAYS$2 = 28;
|
|
20962
21236
|
const MAX_DAYS$1 = 548;
|
|
20963
21237
|
const DEFAULT_VERIFY_LIMIT$1 = 5;
|
|
20964
21238
|
const MAX_VERIFY_LIMIT = 100;
|
|
@@ -20980,7 +21254,7 @@ async function quickWinsReportWith(input, dependencies) {
|
|
|
20980
21254
|
const explicitRange = explicitDateRange(input, MAX_DAYS$1);
|
|
20981
21255
|
const days = explicitRange?.days ?? integerOption$1({
|
|
20982
21256
|
value: input.days,
|
|
20983
|
-
fallback: DEFAULT_DAYS$
|
|
21257
|
+
fallback: DEFAULT_DAYS$2,
|
|
20984
21258
|
minimum: 1,
|
|
20985
21259
|
maximum: MAX_DAYS$1,
|
|
20986
21260
|
label: "Days"
|
|
@@ -21145,13 +21419,13 @@ async function quickWinsReportWith(input, dependencies) {
|
|
|
21145
21419
|
}
|
|
21146
21420
|
//#endregion
|
|
21147
21421
|
//#region packages/core/src/analyze/site-diagnostics/quick-wins.ts
|
|
21148
|
-
const defaultDependencies$
|
|
21422
|
+
const defaultDependencies$5 = {
|
|
21149
21423
|
searchAnalytics: querySearchAnalytics,
|
|
21150
21424
|
fetch: fetchPage,
|
|
21151
21425
|
extract: extractPage,
|
|
21152
21426
|
now: () => /* @__PURE__ */ new Date()
|
|
21153
21427
|
};
|
|
21154
|
-
function quickWinsReport(input, dependencies = defaultDependencies$
|
|
21428
|
+
function quickWinsReport(input, dependencies = defaultDependencies$5) {
|
|
21155
21429
|
return quickWinsReportWith(input, dependencies);
|
|
21156
21430
|
}
|
|
21157
21431
|
//#endregion
|
|
@@ -21404,7 +21678,7 @@ function analyzeStrikingDistanceRows(input) {
|
|
|
21404
21678
|
}
|
|
21405
21679
|
//#endregion
|
|
21406
21680
|
//#region packages/core/src/analyze/striking-distance-report.ts
|
|
21407
|
-
const DEFAULT_DAYS$
|
|
21681
|
+
const DEFAULT_DAYS$1 = 28;
|
|
21408
21682
|
const MAX_DAYS = 548;
|
|
21409
21683
|
const DEFAULT_VERIFY_LIMIT = 5;
|
|
21410
21684
|
const MAX_SOURCE_ROWS$1 = 1e5;
|
|
@@ -21413,7 +21687,7 @@ function boundedInteger$2(value, fallback, minimum, maximum) {
|
|
|
21413
21687
|
return Math.min(maximum, Math.max(minimum, Math.floor(value)));
|
|
21414
21688
|
}
|
|
21415
21689
|
function reportDays(value) {
|
|
21416
|
-
if (value === void 0) return DEFAULT_DAYS$
|
|
21690
|
+
if (value === void 0) return DEFAULT_DAYS$1;
|
|
21417
21691
|
if (!Number.isInteger(value) || value < 1 || value > MAX_DAYS) throw new SeoError("INVALID_INPUT", `Days must be a whole number between 1 and ${MAX_DAYS}.`);
|
|
21418
21692
|
return value;
|
|
21419
21693
|
}
|
|
@@ -21560,12 +21834,12 @@ async function strikingDistanceWith(input, dependencies) {
|
|
|
21560
21834
|
}
|
|
21561
21835
|
//#endregion
|
|
21562
21836
|
//#region packages/core/src/analyze/striking-distance.ts
|
|
21563
|
-
const defaultDependencies$
|
|
21837
|
+
const defaultDependencies$4 = {
|
|
21564
21838
|
searchAnalytics: querySearchAnalytics,
|
|
21565
21839
|
verifyContent: verifyQueryContent,
|
|
21566
21840
|
now: () => /* @__PURE__ */ new Date()
|
|
21567
21841
|
};
|
|
21568
|
-
function strikingDistance(input, dependencies = defaultDependencies$
|
|
21842
|
+
function strikingDistance(input, dependencies = defaultDependencies$4) {
|
|
21569
21843
|
return strikingDistanceWith(input, dependencies);
|
|
21570
21844
|
}
|
|
21571
21845
|
//#endregion
|
|
@@ -21577,7 +21851,7 @@ async function fetchSearchStatusIncidents() {
|
|
|
21577
21851
|
return readJson(response, "Google Search Status", { maxBytes: 10 * 1024 * 1024 });
|
|
21578
21852
|
}
|
|
21579
21853
|
async function listSearchUpdates(input = {}) {
|
|
21580
|
-
return searchUpdatesFromIncidents(await fetchSearchStatusIncidents(), input);
|
|
21854
|
+
return searchUpdatesFromIncidents(withRankingHistory(await fetchSearchStatusIncidents()), input);
|
|
21581
21855
|
}
|
|
21582
21856
|
//#endregion
|
|
21583
21857
|
//#region packages/core/src/analyze/traffic-anomaly-analysis.ts
|
|
@@ -21636,7 +21910,7 @@ function anomalyEvidence(anomaly) {
|
|
|
21636
21910
|
if (anomaly.direction === "normal") return `${formatMetric(anomaly.metric)} changed ${changePhrase} (${anomaly.baselineMean.toLocaleString("en-GB")} -> ${anomaly.comparisonMean.toLocaleString("en-GB")} per observed day${anomaly.zScore === null ? "; the baseline had no observed variance" : `, z ${anomaly.zScore}`}).`;
|
|
21637
21911
|
return `${formatMetric(anomaly.metric)} ${direction} ${changePhrase} (${anomaly.baselineMean.toLocaleString("en-GB")} -> ${anomaly.comparisonMean.toLocaleString("en-GB")} per observed day${anomaly.zScore === null ? "; outside the flat observed baseline" : `, z ${anomaly.zScore}`}).`;
|
|
21638
21912
|
}
|
|
21639
|
-
function plural$
|
|
21913
|
+
function plural$4(count, singular, pluralLabel = `${singular}s`) {
|
|
21640
21914
|
return count === 1 ? singular : pluralLabel;
|
|
21641
21915
|
}
|
|
21642
21916
|
function strongestSignal(anomalies) {
|
|
@@ -21692,13 +21966,13 @@ function interpretUpdateCorrelation(input) {
|
|
|
21692
21966
|
summary,
|
|
21693
21967
|
evidence: [
|
|
21694
21968
|
...input.anomalies.map(anomalyEvidence),
|
|
21695
|
-
hasUpdate ? `${input.overlappingUpdates.length} official Google Ranking update ${plural$
|
|
21969
|
+
hasUpdate ? `${input.overlappingUpdates.length} official Google Ranking update ${plural$4(input.overlappingUpdates.length, "window")} overlapped the change date or comparison window: ${updateNames.join(", ")}.` : "No official Google Ranking update windows overlapped the change date or comparison window.",
|
|
21696
21970
|
largeMovement ? "Both clicks and impressions moved sharply in the same direction." : "Clicks and impressions did not both meet the sharp-movement evidence threshold.",
|
|
21697
21971
|
hasConfounders ? `Known overlapping site changes: ${confounders.map((confounder) => confounder.title).join("; ")}.` : "No overlapping site changes were supplied."
|
|
21698
21972
|
],
|
|
21699
21973
|
caveats: [
|
|
21700
21974
|
`GSC window: ${input.days} days; recent comparison window: ${input.recentDays} days.`,
|
|
21701
|
-
`Update overlap padding: ${input.paddingDays} ${plural$
|
|
21975
|
+
`Update overlap padding: ${input.paddingDays} ${plural$4(input.paddingDays, "day")} either side of the step-change date, or of the recent comparison window when no step change was found.`,
|
|
21702
21976
|
input.refresh ? "Data freshness: cache bypassed for GSC data." : "Data freshness: local GSC cache may have been used.",
|
|
21703
21977
|
"Source: official Google Search Status Dashboard Ranking incidents only; third-party volatility and unconfirmed chatter are not included.",
|
|
21704
21978
|
"An update-window overlap cannot establish causation, even when movement is large and no known site change was recorded.",
|
|
@@ -21866,7 +22140,7 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
21866
22140
|
if (step && changeDate) {
|
|
21867
22141
|
const previousDate = aggregated.rows[step.index - 1]?.keys[0];
|
|
21868
22142
|
const gap = previousDate ? calendarDays(previousDate, changeDate) - 2 : 0;
|
|
21869
|
-
stepCaveats.push(`${formatMetric(metric)} changed level on ${changeDate}, so its baseline is every observed day before that date and its comparison is every observed day from it.`, ...gap > 0 ? [`The ${metric} change date follows ${gap} missing ${plural$
|
|
22143
|
+
stepCaveats.push(`${formatMetric(metric)} changed level on ${changeDate}, so its baseline is every observed day before that date and its comparison is every observed day from it.`, ...gap > 0 ? [`The ${metric} change date follows ${gap} missing ${plural$4(gap, "day")}; the change may have started on one of them.`] : []);
|
|
21870
22144
|
return compareSegments({
|
|
21871
22145
|
site: input.site,
|
|
21872
22146
|
metric,
|
|
@@ -21897,9 +22171,9 @@ function analyzeTrafficAnomalyRows(input) {
|
|
|
21897
22171
|
const comparisonMissingDays = input.recentDays - comparisonRows.length;
|
|
21898
22172
|
const missingDays = expectedDays - aggregated.rows.length;
|
|
21899
22173
|
const caveats = [
|
|
21900
|
-
...missingDays > 0 ? [`${missingDays} requested calendar ${plural$
|
|
21901
|
-
...aggregated.invalidRows > 0 ? [`${aggregated.invalidRows} invalid provider ${plural$
|
|
21902
|
-
...aggregated.duplicateRows > 0 ? [`${aggregated.duplicateRows} duplicate date ${plural$
|
|
22174
|
+
...missingDays > 0 ? [`${missingDays} requested calendar ${plural$4(missingDays, "day")} had no returned date aggregate and ${missingDays === 1 ? "was" : "were"} not filled with zeros.`] : [],
|
|
22175
|
+
...aggregated.invalidRows > 0 ? [`${aggregated.invalidRows} invalid provider ${plural$4(aggregated.invalidRows, "row")} ${aggregated.invalidRows === 1 ? "was" : "were"} excluded.`] : [],
|
|
22176
|
+
...aggregated.duplicateRows > 0 ? [`${aggregated.duplicateRows} duplicate date ${plural$4(aggregated.duplicateRows, "row")} ${aggregated.duplicateRows === 1 ? "was" : "were"} aggregated before analysis.`] : [],
|
|
21903
22177
|
...stepCaveats
|
|
21904
22178
|
];
|
|
21905
22179
|
return {
|
|
@@ -21982,11 +22256,11 @@ async function trafficAnomalyWith(input, dependencies) {
|
|
|
21982
22256
|
}
|
|
21983
22257
|
//#endregion
|
|
21984
22258
|
//#region packages/core/src/analyze/traffic-anomaly.ts
|
|
21985
|
-
const defaultDependencies$
|
|
22259
|
+
const defaultDependencies$3 = {
|
|
21986
22260
|
searchAnalytics: querySearchAnalytics,
|
|
21987
22261
|
now: () => /* @__PURE__ */ new Date()
|
|
21988
22262
|
};
|
|
21989
|
-
function trafficAnomaly(input, dependencies = defaultDependencies$
|
|
22263
|
+
function trafficAnomaly(input, dependencies = defaultDependencies$3) {
|
|
21990
22264
|
return trafficAnomalyWith(input, dependencies);
|
|
21991
22265
|
}
|
|
21992
22266
|
async function updateCorrelation(input) {
|
|
@@ -21999,7 +22273,7 @@ async function updateCorrelation(input) {
|
|
|
21999
22273
|
const comparisonEnd = window?.end;
|
|
22000
22274
|
const updates = await listSearchUpdates({
|
|
22001
22275
|
product: "Ranking",
|
|
22002
|
-
limit:
|
|
22276
|
+
limit: Number.POSITIVE_INFINITY
|
|
22003
22277
|
});
|
|
22004
22278
|
const overlappingUpdates = comparisonStart && comparisonEnd ? findOverlappingSearchUpdates({
|
|
22005
22279
|
updates,
|
|
@@ -22344,7 +22618,7 @@ function emptyQuickWins(input) {
|
|
|
22344
22618
|
warnings: []
|
|
22345
22619
|
};
|
|
22346
22620
|
}
|
|
22347
|
-
const defaultDependencies$
|
|
22621
|
+
const defaultDependencies$2 = {
|
|
22348
22622
|
trafficAnomaly,
|
|
22349
22623
|
updateCorrelation,
|
|
22350
22624
|
segmentImpact,
|
|
@@ -22355,7 +22629,7 @@ const defaultDependencies$3 = {
|
|
|
22355
22629
|
};
|
|
22356
22630
|
async function diagnoseProperty(input, dependencies = {}) {
|
|
22357
22631
|
const providers = {
|
|
22358
|
-
...defaultDependencies$
|
|
22632
|
+
...defaultDependencies$2,
|
|
22359
22633
|
...dependencies
|
|
22360
22634
|
};
|
|
22361
22635
|
const limit = input.limit ?? 10;
|
|
@@ -22566,300 +22840,283 @@ async function diagnoseProperty(input, dependencies = {}) {
|
|
|
22566
22840
|
};
|
|
22567
22841
|
}
|
|
22568
22842
|
//#endregion
|
|
22569
|
-
//#region packages/core/src/
|
|
22570
|
-
|
|
22571
|
-
|
|
22572
|
-
|
|
22573
|
-
|
|
22574
|
-
|
|
22575
|
-
|
|
22576
|
-
|
|
22577
|
-
|
|
22578
|
-
"for",
|
|
22579
|
-
"free",
|
|
22580
|
-
"from",
|
|
22581
|
-
"guide",
|
|
22582
|
-
"how",
|
|
22583
|
-
"in",
|
|
22584
|
-
"is",
|
|
22585
|
-
"of",
|
|
22586
|
-
"on",
|
|
22587
|
-
"online",
|
|
22588
|
-
"or",
|
|
22589
|
-
"page",
|
|
22590
|
-
"seo",
|
|
22591
|
-
"site",
|
|
22592
|
-
"the",
|
|
22593
|
-
"to",
|
|
22594
|
-
"tool",
|
|
22595
|
-
"use",
|
|
22596
|
-
"website",
|
|
22597
|
-
"what",
|
|
22598
|
-
"when",
|
|
22599
|
-
"where",
|
|
22600
|
-
"which",
|
|
22601
|
-
"with"
|
|
22602
|
-
]);
|
|
22603
|
-
function compareInternalLinkText(left, right) {
|
|
22604
|
-
return left < right ? -1 : left > right ? 1 : 0;
|
|
22605
|
-
}
|
|
22606
|
-
function normalizeInternalLinkQuery(value) {
|
|
22607
|
-
return value.normalize("NFKC").replace(/[’']/g, "'").toLowerCase().replace(/[^\p{L}\p{N}']+/gu, " ").replace(/\s+/g, " ").trim();
|
|
22608
|
-
}
|
|
22609
|
-
function normalizeInternalLinkUrl(value) {
|
|
22610
|
-
try {
|
|
22611
|
-
const url = new URL(value);
|
|
22612
|
-
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
22613
|
-
url.hash = "";
|
|
22614
|
-
return url.toString();
|
|
22615
|
-
} catch {
|
|
22616
|
-
return;
|
|
22843
|
+
//#region packages/core/src/decision-model.ts
|
|
22844
|
+
var DecisionModelError = class extends Error {
|
|
22845
|
+
kind;
|
|
22846
|
+
retryAfterMs;
|
|
22847
|
+
constructor(kind, message, retryAfterMs) {
|
|
22848
|
+
super(message);
|
|
22849
|
+
this.kind = kind;
|
|
22850
|
+
this.retryAfterMs = retryAfterMs;
|
|
22851
|
+
this.name = "DecisionModelError";
|
|
22617
22852
|
}
|
|
22618
|
-
|
|
22619
|
-
|
|
22620
|
-
|
|
22621
|
-
|
|
22622
|
-
|
|
22623
|
-
|
|
22624
|
-
|
|
22625
|
-
|
|
22626
|
-
|
|
22627
|
-
|
|
22628
|
-
|
|
22629
|
-
|
|
22630
|
-
|
|
22631
|
-
|
|
22632
|
-
|
|
22633
|
-
|
|
22634
|
-
|
|
22635
|
-
|
|
22636
|
-
|
|
22637
|
-
|
|
22638
|
-
|
|
22639
|
-
|
|
22640
|
-
|
|
22641
|
-
|
|
22642
|
-
|
|
22643
|
-
|
|
22644
|
-
|
|
22645
|
-
|
|
22646
|
-
|
|
22647
|
-
|
|
22648
|
-
|
|
22649
|
-
|
|
22650
|
-
|
|
22651
|
-
|
|
22652
|
-
|
|
22653
|
-
|
|
22654
|
-
|
|
22655
|
-
|
|
22656
|
-
|
|
22657
|
-
|
|
22658
|
-
|
|
22659
|
-
|
|
22660
|
-
|
|
22661
|
-
|
|
22662
|
-
|
|
22663
|
-
|
|
22664
|
-
|
|
22665
|
-
|
|
22666
|
-
|
|
22667
|
-
|
|
22668
|
-
|
|
22853
|
+
get retryable() {
|
|
22854
|
+
return this.kind === "rate-limit" || this.kind === "unavailable";
|
|
22855
|
+
}
|
|
22856
|
+
};
|
|
22857
|
+
const TYPESAFE_URL = "https://api.typesafe.ai/v1/systemone";
|
|
22858
|
+
const JEV_PRICE_PER_MILLION = .042;
|
|
22859
|
+
const REQUEST_TIMEOUT_MS = 6e4;
|
|
22860
|
+
const MAX_RESPONSE_BYTES = 1024 * 1024;
|
|
22861
|
+
function retryAfter(response) {
|
|
22862
|
+
const value = Number(response.headers.get("retry-after"));
|
|
22863
|
+
return Number.isFinite(value) && value > 0 ? Math.min(value, 30) * 1e3 : void 0;
|
|
22864
|
+
}
|
|
22865
|
+
function typesafeDecisionModel(input) {
|
|
22866
|
+
const call = input.fetch ?? fetch;
|
|
22867
|
+
const model = input.model ?? "jev-latest";
|
|
22868
|
+
return {
|
|
22869
|
+
provider: "typesafe",
|
|
22870
|
+
name: model,
|
|
22871
|
+
pricePerMillionInputTokens: JEV_PRICE_PER_MILLION,
|
|
22872
|
+
async ask(request, signal) {
|
|
22873
|
+
const controller = new AbortController();
|
|
22874
|
+
const abort = () => controller.abort();
|
|
22875
|
+
const timer = setTimeout(abort, REQUEST_TIMEOUT_MS);
|
|
22876
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
22877
|
+
let response;
|
|
22878
|
+
try {
|
|
22879
|
+
response = await call(TYPESAFE_URL, {
|
|
22880
|
+
method: "POST",
|
|
22881
|
+
headers: {
|
|
22882
|
+
authorization: `Bearer ${input.apiKey}`,
|
|
22883
|
+
"content-type": "application/json"
|
|
22884
|
+
},
|
|
22885
|
+
body: JSON.stringify({
|
|
22886
|
+
model,
|
|
22887
|
+
...request
|
|
22888
|
+
}),
|
|
22889
|
+
signal: controller.signal
|
|
22890
|
+
});
|
|
22891
|
+
} catch (error) {
|
|
22892
|
+
throw new DecisionModelError("unavailable", `TypeSafe could not be reached: ${error instanceof Error ? error.message : String(error)}`);
|
|
22893
|
+
} finally {
|
|
22894
|
+
clearTimeout(timer);
|
|
22895
|
+
signal?.removeEventListener("abort", abort);
|
|
22896
|
+
}
|
|
22897
|
+
if (response.status === 401 || response.status === 403) {
|
|
22898
|
+
await response.body?.cancel();
|
|
22899
|
+
throw new DecisionModelError("auth", "TypeSafe rejected the API key. Replace it at https://seoskill.dev/account#keys or in TYPESAFE_API_KEY.");
|
|
22900
|
+
}
|
|
22901
|
+
if (response.status === 429 || response.status === 529) {
|
|
22902
|
+
await response.body?.cancel();
|
|
22903
|
+
throw new DecisionModelError("rate-limit", `TypeSafe returned HTTP ${response.status}.`, retryAfter(response));
|
|
22904
|
+
}
|
|
22905
|
+
if (response.status >= 500) {
|
|
22906
|
+
await response.body?.cancel();
|
|
22907
|
+
throw new DecisionModelError("unavailable", `TypeSafe returned HTTP ${response.status}.`);
|
|
22908
|
+
}
|
|
22909
|
+
if (!response.ok) {
|
|
22910
|
+
await response.body?.cancel();
|
|
22911
|
+
throw new DecisionModelError("invalid", `TypeSafe did not accept the request (HTTP ${response.status}).`);
|
|
22912
|
+
}
|
|
22913
|
+
const body = await readJson(response, "TypeSafe", { maxBytes: MAX_RESPONSE_BYTES });
|
|
22914
|
+
return {
|
|
22915
|
+
model: body.model ?? model,
|
|
22916
|
+
answers: body.answers ?? {},
|
|
22917
|
+
inputTokens: body.usage?.input_tokens ?? 0
|
|
22918
|
+
};
|
|
22919
|
+
}
|
|
22669
22920
|
};
|
|
22670
22921
|
}
|
|
22671
|
-
|
|
22672
|
-
|
|
22673
|
-
const
|
|
22674
|
-
const
|
|
22675
|
-
|
|
22676
|
-
|
|
22677
|
-
|
|
22678
|
-
|
|
22679
|
-
|
|
22680
|
-
|
|
22681
|
-
|
|
22682
|
-
|
|
22683
|
-
|
|
22684
|
-
|
|
22685
|
-
|
|
22686
|
-
|
|
22687
|
-
|
|
22688
|
-
|
|
22689
|
-
|
|
22690
|
-
impressions: 0
|
|
22691
|
-
};
|
|
22692
|
-
current.clicks += row.clicks;
|
|
22693
|
-
current.impressions += row.impressions;
|
|
22694
|
-
if (compareInternalLinkText(query, current.query) < 0) current.query = query;
|
|
22695
|
-
map.set(key, current);
|
|
22696
|
-
}
|
|
22697
|
-
function bestMatch(input) {
|
|
22698
|
-
const sourceKey = normalizeInternalLinkQuery(input.source.query);
|
|
22699
|
-
const exact = input.exactTargets.get(sourceKey);
|
|
22700
|
-
if (exact) return {
|
|
22701
|
-
sourceQuery: input.source.query,
|
|
22702
|
-
targetQuery: exact.query,
|
|
22703
|
-
kind: "exact-query",
|
|
22704
|
-
relevanceScore: 1,
|
|
22705
|
-
impressions: input.source.impressions,
|
|
22706
|
-
sharedTerms: internalLinkTokens(input.source.query)
|
|
22707
|
-
};
|
|
22708
|
-
return input.lexicalTargets.flatMap((target) => {
|
|
22709
|
-
const score = input.scoreLexical(input.source.query, target.query);
|
|
22710
|
-
return score ? [{
|
|
22711
|
-
sourceQuery: input.source.query,
|
|
22712
|
-
targetQuery: target.query,
|
|
22713
|
-
kind: "lexical-review",
|
|
22714
|
-
relevanceScore: score.score,
|
|
22715
|
-
impressions: input.source.impressions,
|
|
22716
|
-
sharedTerms: score.sharedTerms
|
|
22717
|
-
}] : [];
|
|
22718
|
-
}).sort((left, right) => right.relevanceScore - left.relevanceScore || compareInternalLinkText(left.targetQuery, right.targetQuery))[0];
|
|
22719
|
-
}
|
|
22720
|
-
function candidatesFromMatches(matchesByUrl) {
|
|
22721
|
-
return [...matchesByUrl.entries()].map(([sourceUrl, matches]) => {
|
|
22722
|
-
const ordered = [...matches].sort((left, right) => (left.kind === right.kind ? 0 : left.kind === "exact-query" ? -1 : 1) || right.impressions - left.impressions || right.relevanceScore - left.relevanceScore || compareInternalLinkText(left.sourceQuery, right.sourceQuery));
|
|
22922
|
+
const MAX_CONCURRENCY = 32;
|
|
22923
|
+
const MIN_CONCURRENCY = 2;
|
|
22924
|
+
const MAX_ATTEMPTS = 5;
|
|
22925
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
22926
|
+
var DecisionRunner = class {
|
|
22927
|
+
model;
|
|
22928
|
+
limit = MAX_CONCURRENCY;
|
|
22929
|
+
active = 0;
|
|
22930
|
+
streak = 0;
|
|
22931
|
+
waiting = [];
|
|
22932
|
+
served = "";
|
|
22933
|
+
requests = 0;
|
|
22934
|
+
failed = 0;
|
|
22935
|
+
tokens = 0;
|
|
22936
|
+
authError = null;
|
|
22937
|
+
constructor(model) {
|
|
22938
|
+
this.model = model;
|
|
22939
|
+
}
|
|
22940
|
+
usage() {
|
|
22723
22941
|
return {
|
|
22724
|
-
|
|
22725
|
-
|
|
22726
|
-
|
|
22727
|
-
|
|
22728
|
-
|
|
22729
|
-
|
|
22730
|
-
matches: ordered.slice(0, 10)
|
|
22942
|
+
provider: this.model.provider,
|
|
22943
|
+
model: this.served || this.model.name,
|
|
22944
|
+
requests: this.requests,
|
|
22945
|
+
failedRequests: this.failed,
|
|
22946
|
+
inputTokens: this.tokens,
|
|
22947
|
+
costUsd: Math.round(this.tokens * this.model.pricePerMillionInputTokens / 1e6 * 1e4) / 1e4
|
|
22731
22948
|
};
|
|
22732
|
-
}
|
|
22733
|
-
|
|
22734
|
-
|
|
22735
|
-
|
|
22736
|
-
|
|
22737
|
-
|
|
22738
|
-
|
|
22739
|
-
|
|
22740
|
-
|
|
22741
|
-
|
|
22742
|
-
|
|
22743
|
-
|
|
22744
|
-
|
|
22745
|
-
|
|
22746
|
-
|
|
22747
|
-
|
|
22748
|
-
|
|
22749
|
-
|
|
22750
|
-
|
|
22751
|
-
|
|
22752
|
-
|
|
22949
|
+
}
|
|
22950
|
+
async slot() {
|
|
22951
|
+
while (this.active >= this.limit) await new Promise((resolve) => this.waiting.push(resolve));
|
|
22952
|
+
this.active += 1;
|
|
22953
|
+
}
|
|
22954
|
+
release() {
|
|
22955
|
+
this.active -= 1;
|
|
22956
|
+
this.waiting.shift()?.();
|
|
22957
|
+
}
|
|
22958
|
+
async ask(request) {
|
|
22959
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
|
|
22960
|
+
if (this.authError) break;
|
|
22961
|
+
let delay = 0;
|
|
22962
|
+
await this.slot();
|
|
22963
|
+
try {
|
|
22964
|
+
const response = await this.model.ask(request);
|
|
22965
|
+
this.requests += 1;
|
|
22966
|
+
this.tokens += response.inputTokens;
|
|
22967
|
+
this.served = response.model;
|
|
22968
|
+
this.streak += 1;
|
|
22969
|
+
if (this.streak >= 50 && this.limit < MAX_CONCURRENCY) {
|
|
22970
|
+
this.limit += 1;
|
|
22971
|
+
this.streak = 0;
|
|
22972
|
+
this.waiting.shift()?.();
|
|
22973
|
+
}
|
|
22974
|
+
return response;
|
|
22975
|
+
} catch (error) {
|
|
22976
|
+
const failure = error instanceof DecisionModelError ? error : new DecisionModelError("unavailable", String(error));
|
|
22977
|
+
if (failure.kind === "auth") {
|
|
22978
|
+
this.authError = failure;
|
|
22979
|
+
break;
|
|
22980
|
+
}
|
|
22981
|
+
if (!failure.retryable || attempt === MAX_ATTEMPTS) break;
|
|
22982
|
+
if (failure.kind === "rate-limit") {
|
|
22983
|
+
this.limit = Math.max(MIN_CONCURRENCY, Math.floor(this.limit / 2));
|
|
22984
|
+
this.streak = 0;
|
|
22985
|
+
}
|
|
22986
|
+
delay = failure.retryAfterMs ?? 2 ** attempt * 250 + Math.floor(Math.random() * 250);
|
|
22987
|
+
} finally {
|
|
22988
|
+
this.release();
|
|
22989
|
+
}
|
|
22990
|
+
await sleep(delay);
|
|
22753
22991
|
}
|
|
22754
|
-
|
|
22992
|
+
this.failed += 1;
|
|
22993
|
+
return null;
|
|
22755
22994
|
}
|
|
22756
|
-
|
|
22757
|
-
|
|
22758
|
-
|
|
22759
|
-
|
|
22760
|
-
|
|
22761
|
-
|
|
22762
|
-
|
|
22995
|
+
};
|
|
22996
|
+
//#endregion
|
|
22997
|
+
//#region packages/core/src/analyze/internal-links-index.ts
|
|
22998
|
+
const PAGE_TERMS = 300;
|
|
22999
|
+
const VECTOR_TERMS = 100;
|
|
23000
|
+
var SiteIndex = class {
|
|
23001
|
+
ids = /* @__PURE__ */ new Map();
|
|
23002
|
+
df = [];
|
|
23003
|
+
counts = [];
|
|
23004
|
+
vectors = [];
|
|
23005
|
+
postings = [];
|
|
23006
|
+
scratch = /* @__PURE__ */ new Float64Array(0);
|
|
23007
|
+
get size() {
|
|
23008
|
+
return this.counts.length;
|
|
23009
|
+
}
|
|
23010
|
+
addPage(termCounts) {
|
|
23011
|
+
const kept = Object.entries(termCounts).sort((left, right) => right[1] - left[1] || (left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0)).slice(0, PAGE_TERMS);
|
|
23012
|
+
const ids = new Int32Array(kept.length);
|
|
23013
|
+
const counts = new Float32Array(kept.length);
|
|
23014
|
+
kept.forEach(([term, count], index) => {
|
|
23015
|
+
let id = this.ids.get(term);
|
|
23016
|
+
if (id === void 0) {
|
|
23017
|
+
id = this.df.length;
|
|
23018
|
+
this.ids.set(term, id);
|
|
23019
|
+
this.df.push(0);
|
|
23020
|
+
}
|
|
23021
|
+
this.df[id] = (this.df[id] ?? 0) + 1;
|
|
23022
|
+
ids[index] = id;
|
|
23023
|
+
counts[index] = count;
|
|
22763
23024
|
});
|
|
22764
|
-
|
|
22765
|
-
|
|
22766
|
-
|
|
22767
|
-
|
|
22768
|
-
|
|
22769
|
-
const lexicalTargets = allTargets.slice(0, LEXICAL_TARGET_LIMIT);
|
|
22770
|
-
const exactTargets = new Map(allTargets.map((target) => [normalizeInternalLinkQuery(target.query), target]));
|
|
22771
|
-
const sourceMap = /* @__PURE__ */ new Map();
|
|
22772
|
-
let sourceValidRows = 0;
|
|
22773
|
-
for (const row of input.sourceRows) {
|
|
22774
|
-
if (!validInternalLinkRow(row)) {
|
|
22775
|
-
increment("sourceInvalidRows");
|
|
22776
|
-
continue;
|
|
22777
|
-
}
|
|
22778
|
-
sourceValidRows += 1;
|
|
22779
|
-
const query = row.keys[0] ?? "";
|
|
22780
|
-
const page = normalizeInternalLinkUrl(row.keys[1] ?? "");
|
|
22781
|
-
if (!page) continue;
|
|
22782
|
-
if (aliases.has(page)) {
|
|
22783
|
-
increment("sourceTargetAliasRows");
|
|
22784
|
-
continue;
|
|
22785
|
-
}
|
|
22786
|
-
const key = `${page}\u0000${normalizeInternalLinkQuery(query)}`;
|
|
22787
|
-
const current = sourceMap.get(key) ?? {
|
|
22788
|
-
page,
|
|
22789
|
-
query,
|
|
22790
|
-
clicks: 0,
|
|
22791
|
-
impressions: 0
|
|
22792
|
-
};
|
|
22793
|
-
current.clicks += row.clicks;
|
|
22794
|
-
current.impressions += row.impressions;
|
|
22795
|
-
if (compareInternalLinkText(query, current.query) < 0) current.query = query;
|
|
22796
|
-
sourceMap.set(key, current);
|
|
23025
|
+
this.counts.push({
|
|
23026
|
+
ids,
|
|
23027
|
+
counts
|
|
23028
|
+
});
|
|
23029
|
+
return this.counts.length - 1;
|
|
22797
23030
|
}
|
|
22798
|
-
|
|
22799
|
-
|
|
22800
|
-
|
|
22801
|
-
|
|
22802
|
-
|
|
22803
|
-
|
|
22804
|
-
|
|
23031
|
+
idfOf(id) {
|
|
23032
|
+
return Math.log((this.size + 1) / ((this.df[id] ?? 1) + 1)) + 1;
|
|
23033
|
+
}
|
|
23034
|
+
idf(term) {
|
|
23035
|
+
const id = this.ids.get(term);
|
|
23036
|
+
return id === void 0 ? Math.log((this.size + 1) / 2) + 1 : this.idfOf(id);
|
|
23037
|
+
}
|
|
23038
|
+
finish() {
|
|
23039
|
+
this.vectors = this.counts.map(({ ids, counts }) => {
|
|
23040
|
+
let norm = 0;
|
|
23041
|
+
const shared = [];
|
|
23042
|
+
for (let index = 0; index < ids.length; index += 1) {
|
|
23043
|
+
const id = ids[index] ?? 0;
|
|
23044
|
+
const weight = (1 + Math.log(counts[index] ?? 1)) * this.idfOf(id);
|
|
23045
|
+
norm += weight * weight;
|
|
23046
|
+
if ((this.df[id] ?? 0) >= 2) shared.push([id, weight]);
|
|
23047
|
+
}
|
|
23048
|
+
norm = Math.sqrt(norm) || 1;
|
|
23049
|
+
shared.sort((left, right) => right[1] - left[1] || left[0] - right[0]);
|
|
23050
|
+
const top = shared.slice(0, VECTOR_TERMS);
|
|
23051
|
+
return {
|
|
23052
|
+
ids: Int32Array.from(top.map(([id]) => id)),
|
|
23053
|
+
weights: Float32Array.from(top.map(([, weight]) => weight / norm))
|
|
23054
|
+
};
|
|
22805
23055
|
});
|
|
22806
|
-
|
|
22807
|
-
|
|
22808
|
-
|
|
22809
|
-
|
|
22810
|
-
|
|
22811
|
-
|
|
22812
|
-
|
|
22813
|
-
|
|
22814
|
-
|
|
22815
|
-
|
|
22816
|
-
|
|
22817
|
-
exactTargets,
|
|
22818
|
-
lexicalTargets,
|
|
22819
|
-
scoreLexical
|
|
23056
|
+
const lists = this.df.map(() => ({
|
|
23057
|
+
pages: [],
|
|
23058
|
+
weights: []
|
|
23059
|
+
}));
|
|
23060
|
+
this.vectors.forEach((vector, page) => {
|
|
23061
|
+
vector.ids.forEach((id, index) => {
|
|
23062
|
+
const list = lists[id];
|
|
23063
|
+
if (!list) return;
|
|
23064
|
+
list.pages.push(page);
|
|
23065
|
+
list.weights.push(vector.weights[index] ?? 0);
|
|
23066
|
+
});
|
|
22820
23067
|
});
|
|
22821
|
-
|
|
22822
|
-
|
|
22823
|
-
|
|
22824
|
-
}
|
|
22825
|
-
|
|
22826
|
-
const matches = matchesByUrl.get(source.page) ?? [];
|
|
22827
|
-
matches.push(match);
|
|
22828
|
-
matchesByUrl.set(source.page, matches);
|
|
23068
|
+
this.postings = lists.map((list) => ({
|
|
23069
|
+
pages: Int32Array.from(list.pages),
|
|
23070
|
+
weights: Float32Array.from(list.weights)
|
|
23071
|
+
}));
|
|
23072
|
+
this.scratch = new Float64Array(this.size);
|
|
22829
23073
|
}
|
|
22830
|
-
|
|
22831
|
-
|
|
22832
|
-
|
|
22833
|
-
|
|
22834
|
-
|
|
22835
|
-
|
|
22836
|
-
|
|
22837
|
-
|
|
22838
|
-
|
|
22839
|
-
|
|
22840
|
-
|
|
22841
|
-
|
|
22842
|
-
|
|
22843
|
-
|
|
22844
|
-
|
|
22845
|
-
|
|
22846
|
-
|
|
22847
|
-
|
|
22848
|
-
|
|
22849
|
-
|
|
22850
|
-
|
|
22851
|
-
|
|
22852
|
-
|
|
22853
|
-
|
|
22854
|
-
|
|
22855
|
-
|
|
22856
|
-
|
|
22857
|
-
|
|
22858
|
-
|
|
22859
|
-
|
|
22860
|
-
const
|
|
22861
|
-
const
|
|
22862
|
-
const
|
|
23074
|
+
similarity(left, right) {
|
|
23075
|
+
const a = this.vectors[left];
|
|
23076
|
+
const b = this.vectors[right];
|
|
23077
|
+
if (!a || !b) return 0;
|
|
23078
|
+
const weights = /* @__PURE__ */ new Map();
|
|
23079
|
+
a.ids.forEach((id, index) => {
|
|
23080
|
+
weights.set(id, a.weights[index] ?? 0);
|
|
23081
|
+
});
|
|
23082
|
+
let sum = 0;
|
|
23083
|
+
b.ids.forEach((id, index) => {
|
|
23084
|
+
sum += (weights.get(id) ?? 0) * (b.weights[index] ?? 0);
|
|
23085
|
+
});
|
|
23086
|
+
return sum;
|
|
23087
|
+
}
|
|
23088
|
+
neighbours(page, min) {
|
|
23089
|
+
const vector = this.vectors[page];
|
|
23090
|
+
if (!vector) return [];
|
|
23091
|
+
const scores = this.scratch;
|
|
23092
|
+
const touched = [];
|
|
23093
|
+
vector.ids.forEach((id, index) => {
|
|
23094
|
+
const weight = vector.weights[index] ?? 0;
|
|
23095
|
+
const list = this.postings[id];
|
|
23096
|
+
if (!list) return;
|
|
23097
|
+
for (let at = 0; at < list.pages.length; at += 1) {
|
|
23098
|
+
const other = list.pages[at] ?? 0;
|
|
23099
|
+
if (other === page) continue;
|
|
23100
|
+
if (scores[other] === 0) touched.push(other);
|
|
23101
|
+
scores[other] = (scores[other] ?? 0) + weight * (list.weights[at] ?? 0);
|
|
23102
|
+
}
|
|
23103
|
+
});
|
|
23104
|
+
const out = [];
|
|
23105
|
+
for (const other of touched) {
|
|
23106
|
+
const score = scores[other] ?? 0;
|
|
23107
|
+
if (score >= min) out.push({
|
|
23108
|
+
page: other,
|
|
23109
|
+
score
|
|
23110
|
+
});
|
|
23111
|
+
scores[other] = 0;
|
|
23112
|
+
}
|
|
23113
|
+
return out.sort((left, right) => right.score - left.score || left.page - right.page);
|
|
23114
|
+
}
|
|
23115
|
+
};
|
|
23116
|
+
const PAIR_SENTENCES = 4;
|
|
23117
|
+
const PAIR_ANCHORS = 8;
|
|
23118
|
+
const MIN_SENTENCE_WORDS = 6;
|
|
23119
|
+
const NAME_WEIGHT = 2;
|
|
22863
23120
|
const GENERIC_ANCHORS = /^(click here|here|read more|learn more|more|this|this post|this article|this page|link|website|check it out|see more|view|continue reading)$/i;
|
|
22864
23121
|
function normalise(value) {
|
|
22865
23122
|
return value.normalize("NFKD").replace(/\p{Diacritic}/gu, "").normalize("NFC").replace(/([\p{L}\p{N}])[’'`]([\p{L}\p{N}])/gu, "$1$2").toLowerCase().replace(/&/g, " and ").replace(/[^\p{L}\p{N}]+/gu, " ").replace(/\s+/g, " ").trim();
|
|
@@ -23127,65 +23384,49 @@ function pageSentences(html) {
|
|
|
23127
23384
|
flushRaw("");
|
|
23128
23385
|
return sentences;
|
|
23129
23386
|
}
|
|
23130
|
-
|
|
23131
|
-
|
|
23132
|
-
|
|
23133
|
-
|
|
23134
|
-
|
|
23135
|
-
|
|
23136
|
-
|
|
23137
|
-
|
|
23138
|
-
return Math.log((this.size + 1) / ((this.df.get(term) ?? 1) + 1)) + 1;
|
|
23139
|
-
}
|
|
23140
|
-
vector(counts) {
|
|
23141
|
-
let norm = 0;
|
|
23142
|
-
const shared = [];
|
|
23143
|
-
for (const term in counts) {
|
|
23144
|
-
const weight = (1 + Math.log(counts[term] ?? 1)) * this.idf(term);
|
|
23145
|
-
norm += weight * weight;
|
|
23146
|
-
if ((this.df.get(term) ?? 0) >= 2) shared.push([term, weight]);
|
|
23147
|
-
}
|
|
23148
|
-
norm = Math.sqrt(norm) || 1;
|
|
23149
|
-
shared.sort((left, right) => right[1] - left[1] || (left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0));
|
|
23150
|
-
return new Map(shared.slice(0, MAX_TERMS).map(([term, weight]) => [term, weight / norm]));
|
|
23151
|
-
}
|
|
23152
|
-
similarity(left, right) {
|
|
23153
|
-
let sum = 0;
|
|
23154
|
-
for (const [term, weight] of left) sum += weight * (right.get(term) ?? 0);
|
|
23155
|
-
return sum;
|
|
23387
|
+
function linkableSentences(sentences) {
|
|
23388
|
+
const seen = /* @__PURE__ */ new Set();
|
|
23389
|
+
const out = [];
|
|
23390
|
+
for (const sentence of sentences) {
|
|
23391
|
+
if (out.length >= 150) break;
|
|
23392
|
+
if (seen.has(sentence.text)) continue;
|
|
23393
|
+
seen.add(sentence.text);
|
|
23394
|
+
if (!sentence.hasLink && sentence.text.split(/\s+/).length >= MIN_SENTENCE_WORDS) out.push(sentence);
|
|
23156
23395
|
}
|
|
23157
|
-
|
|
23158
|
-
|
|
23159
|
-
|
|
23160
|
-
|
|
23161
|
-
|
|
23162
|
-
|
|
23163
|
-
|
|
23164
|
-
|
|
23165
|
-
|
|
23166
|
-
|
|
23167
|
-
|
|
23396
|
+
return out;
|
|
23397
|
+
}
|
|
23398
|
+
function sentenceMatch(sentence, key, weights) {
|
|
23399
|
+
if (!key.size) return 0;
|
|
23400
|
+
const seen = new Set(terms(sentence));
|
|
23401
|
+
let hit = 0;
|
|
23402
|
+
let total = 0;
|
|
23403
|
+
for (const [term, place] of key) {
|
|
23404
|
+
const weight = weights.idf(term) * place;
|
|
23405
|
+
total += weight;
|
|
23406
|
+
if (seen.has(term)) hit += weight;
|
|
23168
23407
|
}
|
|
23169
|
-
|
|
23170
|
-
|
|
23408
|
+
return total ? hit / total : 0;
|
|
23409
|
+
}
|
|
23410
|
+
function anchorCandidates(sentence, key, weights, limit) {
|
|
23171
23411
|
const scored = /* @__PURE__ */ new Map();
|
|
23172
23412
|
const parts = sentence.split(/\s+/).map((word) => word.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "")).filter(Boolean);
|
|
23173
|
-
for (let size =
|
|
23413
|
+
for (let size = 1; size <= 4; size += 1) for (let start = 0; start + size <= parts.length; start += 1) {
|
|
23174
23414
|
const gram = parts.slice(start, start + size);
|
|
23175
23415
|
const first = (gram[0] ?? "").toLowerCase();
|
|
23176
23416
|
const last = (gram[size - 1] ?? "").toLowerCase();
|
|
23177
23417
|
if (FUNCTION_WORDS.has(first) || FUNCTION_WORDS.has(last) || /^\d+$/.test(first) || /^\d+$/.test(last)) continue;
|
|
23178
23418
|
const stems = terms(gram.join(" "));
|
|
23179
|
-
if (stems.length < 2) continue;
|
|
23419
|
+
if (stems.length < size || size > 1 && stems.length < 2) continue;
|
|
23420
|
+
if (size === 1 && key.get(stems[0] ?? "") !== NAME_WEIGHT) continue;
|
|
23180
23421
|
const hits = stems.filter((term) => key.has(term)).length;
|
|
23181
|
-
const named = stems.reduce((sum, term) => sum + (key.get(term) ?? 0) *
|
|
23422
|
+
const named = stems.reduce((sum, term) => sum + (key.get(term) ?? 0) * weights.idf(term), 0);
|
|
23182
23423
|
if (!stems.some((term) => key.get(term) === NAME_WEIGHT)) continue;
|
|
23183
23424
|
if (Number(key.has(terms(first)[0] ?? "")) + Number(key.has(terms(last)[0] ?? "")) < 2) continue;
|
|
23184
23425
|
const text = gram.join(" ");
|
|
23185
23426
|
const lower = text.toLowerCase();
|
|
23186
23427
|
if (GENERIC_ANCHORS.test(lower)) continue;
|
|
23187
|
-
const rarity = stems.reduce((sum, term) => sum +
|
|
23188
|
-
const score = named * (1 + hits / key.size) + rarity * .05 - (size === 4 ? .2 : 0);
|
|
23428
|
+
const rarity = stems.reduce((sum, term) => sum + weights.idf(term), 0);
|
|
23429
|
+
const score = named * (1 + hits / key.size) + rarity * .05 - (size === 1 ? .5 : size === 4 ? .2 : 0);
|
|
23189
23430
|
const existing = scored.get(lower);
|
|
23190
23431
|
if (!existing || existing.score < score) scored.set(lower, {
|
|
23191
23432
|
text,
|
|
@@ -23194,27 +23435,27 @@ function anchorCandidates(sentence, key, corpus, limit) {
|
|
|
23194
23435
|
}
|
|
23195
23436
|
return [...scored.values()].sort((left, right) => right.score - left.score || (left.text < right.text ? -1 : left.text > right.text ? 1 : 0)).slice(0, limit).map((item) => item.text);
|
|
23196
23437
|
}
|
|
23197
|
-
function
|
|
23438
|
+
function pairPlacement(sentences, target, weights) {
|
|
23198
23439
|
const key = keyTerms(target);
|
|
23199
|
-
const
|
|
23200
|
-
const seen = /* @__PURE__ */ new Set();
|
|
23201
|
-
const sentences = source.sentences.filter((sentence) => {
|
|
23202
|
-
if (seen.has(sentence.text)) return false;
|
|
23203
|
-
seen.add(sentence.text);
|
|
23204
|
-
return !sentence.hasLink && sentence.text.split(/\s+/).length >= MIN_SENTENCE_WORDS;
|
|
23205
|
-
}).map((sentence, index) => ({
|
|
23440
|
+
const chosen = sentences.map((sentence, index) => ({
|
|
23206
23441
|
sentence,
|
|
23207
23442
|
index,
|
|
23208
|
-
match:
|
|
23209
|
-
})).filter((item) => item.match > 0).sort((left, right) => right.match - left.match || left.index - right.index).slice(0,
|
|
23210
|
-
|
|
23211
|
-
|
|
23212
|
-
|
|
23213
|
-
|
|
23214
|
-
|
|
23443
|
+
match: sentenceMatch(sentence.text, key, weights)
|
|
23444
|
+
})).filter((item) => item.match > 0).sort((left, right) => right.match - left.match || left.index - right.index).slice(0, PAIR_SENTENCES);
|
|
23445
|
+
const perSentence = Math.max(1, Math.floor(PAIR_ANCHORS / Math.max(1, chosen.length)));
|
|
23446
|
+
const anchors = [];
|
|
23447
|
+
for (const { sentence, index } of chosen) for (const text of anchorCandidates(sentence.text, key, weights, perSentence)) anchors.push({
|
|
23448
|
+
text,
|
|
23449
|
+
sentence: `S${index + 1}`
|
|
23450
|
+
});
|
|
23215
23451
|
return {
|
|
23216
|
-
|
|
23217
|
-
|
|
23452
|
+
sentences: chosen.map(({ sentence, index, match }) => ({
|
|
23453
|
+
id: `S${index + 1}`,
|
|
23454
|
+
text: sentence.text,
|
|
23455
|
+
section: sentence.section,
|
|
23456
|
+
match: Math.round(match * 100) / 100
|
|
23457
|
+
})),
|
|
23458
|
+
anchors: anchors.slice(0, PAIR_ANCHORS)
|
|
23218
23459
|
};
|
|
23219
23460
|
}
|
|
23220
23461
|
function placementPage(input) {
|
|
@@ -23228,542 +23469,1141 @@ function placementPage(input) {
|
|
|
23228
23469
|
};
|
|
23229
23470
|
}
|
|
23230
23471
|
//#endregion
|
|
23231
|
-
//#region packages/core/src/analyze/internal-links-
|
|
23232
|
-
const
|
|
23233
|
-
const
|
|
23234
|
-
|
|
23235
|
-
|
|
23236
|
-
|
|
23237
|
-
|
|
23238
|
-
|
|
23239
|
-
|
|
23240
|
-
|
|
23241
|
-
|
|
23242
|
-
|
|
23243
|
-
|
|
23244
|
-
|
|
23245
|
-
|
|
23246
|
-
|
|
23247
|
-
|
|
23248
|
-
|
|
23249
|
-
|
|
23250
|
-
|
|
23251
|
-
|
|
23252
|
-
|
|
23253
|
-
|
|
23254
|
-
|
|
23255
|
-
|
|
23256
|
-
|
|
23257
|
-
|
|
23258
|
-
|
|
23259
|
-
|
|
23260
|
-
|
|
23261
|
-
|
|
23472
|
+
//#region packages/core/src/analyze/internal-links-crawl.ts
|
|
23473
|
+
const DEFAULT_MAX_PAGES = 500;
|
|
23474
|
+
const MAX_MAX_PAGES = 5e3;
|
|
23475
|
+
const FETCH_CONCURRENCY = 6;
|
|
23476
|
+
const MAX_CRAWL_DELAY_SECONDS = 10;
|
|
23477
|
+
const CRAWL_DELAY_BUDGET_SECONDS = 900;
|
|
23478
|
+
const REDIRECT_CHECKS = 150;
|
|
23479
|
+
const GENERIC_ANCHOR_LIMIT = 100;
|
|
23480
|
+
const EXCERPT_CHARS = 500;
|
|
23481
|
+
const ASSET = /\.(avif|bmp|css|csv|dmg|docx?|eot|gif|gz|ico|jpe?g|js|json|map|mov|mp3|mp4|ogg|otf|pdf|png|pptx?|rss|svg|tar|ttf|txt|wav|webm|webp|woff2?|xlsx?|xml|zip)$/i;
|
|
23482
|
+
const TRACKING = /^(utm_[a-z]+|gclid|fbclid|msclkid|mc_cid|mc_eid|ref)$/i;
|
|
23483
|
+
var SentenceStore = class SentenceStore {
|
|
23484
|
+
dir;
|
|
23485
|
+
handle;
|
|
23486
|
+
size = 0;
|
|
23487
|
+
constructor(dir, handle) {
|
|
23488
|
+
this.dir = dir;
|
|
23489
|
+
this.handle = handle;
|
|
23490
|
+
}
|
|
23491
|
+
static async open() {
|
|
23492
|
+
const dir = await mkdtemp(join(tmpdir(), "seo-internal-links-"));
|
|
23493
|
+
return new SentenceStore(dir, await open$1(join(dir, "sentences"), "w+"));
|
|
23494
|
+
}
|
|
23495
|
+
async write(sentences) {
|
|
23496
|
+
const bytes = Buffer.from(JSON.stringify(sentences.map((s) => [s.text, s.section])));
|
|
23497
|
+
const offset = this.size;
|
|
23498
|
+
this.size += bytes.length;
|
|
23499
|
+
await this.handle.write(bytes, 0, bytes.length, offset);
|
|
23500
|
+
return {
|
|
23501
|
+
offset,
|
|
23502
|
+
length: bytes.length
|
|
23503
|
+
};
|
|
23262
23504
|
}
|
|
23263
|
-
|
|
23264
|
-
|
|
23265
|
-
|
|
23266
|
-
|
|
23267
|
-
|
|
23268
|
-
|
|
23269
|
-
|
|
23270
|
-
|
|
23271
|
-
|
|
23272
|
-
if (!value) return void 0;
|
|
23273
|
-
try {
|
|
23274
|
-
return assertUrlMatchesGscProperty(site, value);
|
|
23275
|
-
} catch {
|
|
23276
|
-
return;
|
|
23505
|
+
async read(ref) {
|
|
23506
|
+
if (!ref.length) return [];
|
|
23507
|
+
const bytes = Buffer.alloc(ref.length);
|
|
23508
|
+
await this.handle.read(bytes, 0, ref.length, ref.offset);
|
|
23509
|
+
return JSON.parse(bytes.toString("utf8")).map(([text, section]) => ({
|
|
23510
|
+
text,
|
|
23511
|
+
section,
|
|
23512
|
+
hasLink: false
|
|
23513
|
+
}));
|
|
23277
23514
|
}
|
|
23278
|
-
|
|
23279
|
-
|
|
23280
|
-
|
|
23281
|
-
|
|
23282
|
-
|
|
23283
|
-
}
|
|
23284
|
-
function sameObservedUrl(left, right) {
|
|
23285
|
-
try {
|
|
23286
|
-
return withoutFragment(left) === withoutFragment(right);
|
|
23287
|
-
} catch {
|
|
23288
|
-
return false;
|
|
23515
|
+
async close() {
|
|
23516
|
+
await this.handle.close().catch(() => void 0);
|
|
23517
|
+
await rm(this.dir, {
|
|
23518
|
+
recursive: true,
|
|
23519
|
+
force: true
|
|
23520
|
+
});
|
|
23289
23521
|
}
|
|
23290
|
-
}
|
|
23291
|
-
|
|
23292
|
-
|
|
23522
|
+
};
|
|
23523
|
+
const bareHost = (host) => host.toLowerCase().replace(/^www\./, "");
|
|
23524
|
+
function crawlUrl(value, base) {
|
|
23293
23525
|
try {
|
|
23294
|
-
|
|
23526
|
+
const url = base ? new URL(value, base) : new URL(value);
|
|
23527
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
|
|
23528
|
+
url.hash = "";
|
|
23529
|
+
for (const key of [...url.searchParams.keys()]) if (TRACKING.test(key)) url.searchParams.delete(key);
|
|
23530
|
+
return url.toString();
|
|
23295
23531
|
} catch {
|
|
23296
23532
|
return;
|
|
23297
23533
|
}
|
|
23298
23534
|
}
|
|
23299
|
-
|
|
23300
|
-
let
|
|
23301
|
-
|
|
23302
|
-
|
|
23303
|
-
|
|
23304
|
-
|
|
23305
|
-
|
|
23306
|
-
|
|
23535
|
+
function excerptOf(sentences) {
|
|
23536
|
+
let out = "";
|
|
23537
|
+
for (const sentence of sentences) {
|
|
23538
|
+
const next = out ? `${out} ${sentence.text}` : sentence.text;
|
|
23539
|
+
if (next.length > EXCERPT_CHARS) break;
|
|
23540
|
+
out = next;
|
|
23541
|
+
}
|
|
23542
|
+
return out || (sentences[0]?.text.slice(0, EXCERPT_CHARS) ?? "");
|
|
23543
|
+
}
|
|
23544
|
+
function readPage(fetched, finalUrl) {
|
|
23545
|
+
const $ = load(fetched.html);
|
|
23546
|
+
const meta = (name) => $(`meta[name="${name}" i]`).map((_, element) => $(element).attr("content") ?? "").get();
|
|
23547
|
+
const headings = $("h1, h2, h3").map((_, element) => ({
|
|
23548
|
+
level: Number(element.tagName.slice(1)),
|
|
23549
|
+
text: $(element).text().replace(/\s+/g, " ").trim()
|
|
23550
|
+
})).get().filter((heading) => heading.text);
|
|
23551
|
+
const links = $("a[href]").map((_, element) => {
|
|
23552
|
+
const anchor = $(element);
|
|
23307
23553
|
return {
|
|
23308
|
-
|
|
23309
|
-
|
|
23310
|
-
|
|
23311
|
-
verification: "failed",
|
|
23312
|
-
technicalSignals: [],
|
|
23313
|
-
warnings: [{
|
|
23314
|
-
stage: "target-fetch",
|
|
23315
|
-
url: input.targetUrl,
|
|
23316
|
-
code: "fetch-failed",
|
|
23317
|
-
message: errorMessage(error)
|
|
23318
|
-
}]
|
|
23554
|
+
href: anchor.attr("href") ?? "",
|
|
23555
|
+
text: anchor.text().replace(/\s+/g, " ").trim(),
|
|
23556
|
+
location: anchor.closest("nav, header").length ? "navigation" : anchor.closest("footer").length ? "footer" : anchor.closest("main, article, [role=\"main\"]").length ? "main-content" : "other"
|
|
23319
23557
|
};
|
|
23320
|
-
}
|
|
23321
|
-
|
|
23558
|
+
}).get();
|
|
23559
|
+
const robots = [...meta("robots"), ...meta("googlebot")];
|
|
23560
|
+
return {
|
|
23561
|
+
title: $("title").first().text().replace(/\s+/g, " ").trim(),
|
|
23562
|
+
metaDescription: meta("description")[0]?.trim() ?? "",
|
|
23563
|
+
canonical: $("link[rel~=\"canonical\" i]").first().attr("href"),
|
|
23564
|
+
lang: $("html").attr("lang"),
|
|
23565
|
+
headings,
|
|
23566
|
+
links,
|
|
23567
|
+
nofollow: robots.some((value) => /\bnofollow\b/i.test(value)),
|
|
23568
|
+
noindex: hasMetaRobotsDirective(robots, "noindex") || hasXRobotsDirective(fetched.headers["x-robots-tag"], "noindex"),
|
|
23569
|
+
url: finalUrl
|
|
23570
|
+
};
|
|
23571
|
+
}
|
|
23572
|
+
async function crawlForLinks(input) {
|
|
23573
|
+
const warnings = [];
|
|
23574
|
+
const robots = createRobotsSession({ refresh: input.refresh });
|
|
23575
|
+
const sentences = await SentenceStore.open();
|
|
23576
|
+
const renderer = createPageRenderer({ concurrency: 2 });
|
|
23322
23577
|
try {
|
|
23323
|
-
|
|
23578
|
+
return await crawl(input, robots, sentences, renderer, warnings);
|
|
23324
23579
|
} catch (error) {
|
|
23325
|
-
|
|
23326
|
-
|
|
23327
|
-
|
|
23328
|
-
|
|
23329
|
-
status: fetched.status,
|
|
23330
|
-
aliases: [input.targetUrl],
|
|
23331
|
-
verification: "failed",
|
|
23332
|
-
technicalSignals: [],
|
|
23333
|
-
fetchDiagnostics: fetched.diagnostics,
|
|
23334
|
-
warnings: [{
|
|
23335
|
-
stage: "target-extract",
|
|
23336
|
-
url: input.targetUrl,
|
|
23337
|
-
code: "extract-failed",
|
|
23338
|
-
message: errorMessage(error)
|
|
23339
|
-
}]
|
|
23340
|
-
};
|
|
23580
|
+
await sentences.close();
|
|
23581
|
+
throw error;
|
|
23582
|
+
} finally {
|
|
23583
|
+
await renderer.close();
|
|
23341
23584
|
}
|
|
23342
|
-
|
|
23343
|
-
|
|
23344
|
-
const
|
|
23345
|
-
|
|
23346
|
-
|
|
23347
|
-
|
|
23348
|
-
|
|
23349
|
-
|
|
23350
|
-
|
|
23351
|
-
|
|
23352
|
-
|
|
23353
|
-
|
|
23354
|
-
|
|
23355
|
-
|
|
23356
|
-
|
|
23357
|
-
|
|
23358
|
-
|
|
23359
|
-
|
|
23360
|
-
|
|
23361
|
-
|
|
23362
|
-
|
|
23363
|
-
|
|
23364
|
-
|
|
23365
|
-
|
|
23366
|
-
|
|
23367
|
-
|
|
23368
|
-
|
|
23369
|
-
|
|
23370
|
-
|
|
23371
|
-
|
|
23372
|
-
|
|
23373
|
-
|
|
23585
|
+
}
|
|
23586
|
+
async function crawl(input, robots, sentences, renderer, warnings) {
|
|
23587
|
+
const start = crawlUrl(input.startUrl) ?? input.startUrl;
|
|
23588
|
+
let origin = new URL(start).origin;
|
|
23589
|
+
const site = bareHost(new URL(start).host);
|
|
23590
|
+
const sameSite = (url) => {
|
|
23591
|
+
try {
|
|
23592
|
+
return bareHost(new URL(url).host) === site;
|
|
23593
|
+
} catch {
|
|
23594
|
+
return false;
|
|
23595
|
+
}
|
|
23596
|
+
};
|
|
23597
|
+
const urls = [];
|
|
23598
|
+
const urlIds = /* @__PURE__ */ new Map();
|
|
23599
|
+
const idOf = (url) => {
|
|
23600
|
+
let id = urlIds.get(url);
|
|
23601
|
+
if (id === void 0) {
|
|
23602
|
+
id = urls.length;
|
|
23603
|
+
urls.push(url);
|
|
23604
|
+
urlIds.set(url, id);
|
|
23605
|
+
}
|
|
23606
|
+
return id;
|
|
23607
|
+
};
|
|
23608
|
+
const aliases = /* @__PURE__ */ new Map();
|
|
23609
|
+
const pageOfUrl = /* @__PURE__ */ new Map();
|
|
23610
|
+
const pages = [];
|
|
23611
|
+
const index = new SiteIndex();
|
|
23612
|
+
const genericAnchors = [];
|
|
23613
|
+
const skipped = {};
|
|
23614
|
+
const skip = (reason) => {
|
|
23615
|
+
skipped[reason] = (skipped[reason] ?? 0) + 1;
|
|
23616
|
+
};
|
|
23617
|
+
const rootRobots = await robots.resolve(origin, start);
|
|
23618
|
+
const delay = await robots.crawlDelay(origin);
|
|
23619
|
+
const crawlDelaySeconds = delay === void 0 ? null : Math.min(delay, MAX_CRAWL_DELAY_SECONDS);
|
|
23620
|
+
const pageLimit = crawlDelaySeconds === null ? input.maxPages : Math.min(input.maxPages, Math.max(20, Math.floor(CRAWL_DELAY_BUDGET_SECONDS / crawlDelaySeconds)));
|
|
23621
|
+
if (pageLimit < input.maxPages) warnings.push(`robots.txt asks for ${crawlDelaySeconds} seconds between requests, so the crawl stopped at ${pageLimit} pages to finish in about ${CRAWL_DELAY_BUDGET_SECONDS / 60} minutes.`);
|
|
23622
|
+
const rate = crawlDelaySeconds === null ? {
|
|
23623
|
+
concurrency: FETCH_CONCURRENCY,
|
|
23624
|
+
intervalCap: FETCH_CONCURRENCY,
|
|
23625
|
+
intervalMs: 1e3
|
|
23626
|
+
} : {
|
|
23627
|
+
concurrency: 1,
|
|
23628
|
+
intervalCap: 1,
|
|
23629
|
+
intervalMs: Math.ceil(crawlDelaySeconds * 1e3)
|
|
23630
|
+
};
|
|
23631
|
+
const options = {
|
|
23632
|
+
js: "auto",
|
|
23633
|
+
refresh: input.refresh,
|
|
23634
|
+
respectRobots: true,
|
|
23635
|
+
robotsResolver: robots.resolve,
|
|
23636
|
+
renderer,
|
|
23637
|
+
rate
|
|
23638
|
+
};
|
|
23639
|
+
const stats = (extra = {}) => ({
|
|
23640
|
+
maxPages: input.maxPages,
|
|
23641
|
+
pageLimit,
|
|
23642
|
+
urlsFound: 0,
|
|
23643
|
+
pagesCrawled: pages.length,
|
|
23644
|
+
capReached: false,
|
|
23645
|
+
skipped,
|
|
23646
|
+
crawlDelaySeconds,
|
|
23647
|
+
blockedByRobots: false,
|
|
23648
|
+
redirectsChecked: 0,
|
|
23649
|
+
...extra
|
|
23650
|
+
});
|
|
23651
|
+
if (rootRobots.allowed === false) return {
|
|
23652
|
+
origin,
|
|
23653
|
+
method: "links",
|
|
23654
|
+
pages,
|
|
23655
|
+
urls,
|
|
23656
|
+
pageOfUrl,
|
|
23657
|
+
index,
|
|
23658
|
+
sentences,
|
|
23659
|
+
genericAnchors,
|
|
23660
|
+
stats: stats({ blockedByRobots: true }),
|
|
23661
|
+
warnings: [`robots.txt does not allow this crawler to read ${start}.`]
|
|
23662
|
+
};
|
|
23663
|
+
const declared = await robots.sitemapUrls(origin);
|
|
23664
|
+
const sitemap = await sitemapSeeds({
|
|
23665
|
+
url: origin,
|
|
23666
|
+
maxPages: pageLimit * 2,
|
|
23667
|
+
declaredSitemapUrls: declared,
|
|
23374
23668
|
warnings,
|
|
23375
|
-
|
|
23669
|
+
fetchSitemapUrls
|
|
23670
|
+
});
|
|
23671
|
+
const seeds = sitemap.urls.map((url) => crawlUrl(url)).filter((url) => Boolean(url) && sameSite(url) && !ASSET.test(new URL(url).pathname));
|
|
23672
|
+
const method = seeds.length ? "sitemap" : "links";
|
|
23673
|
+
const queue = [];
|
|
23674
|
+
const queued = /* @__PURE__ */ new Set();
|
|
23675
|
+
const maxQueue = pageLimit * 5;
|
|
23676
|
+
const enqueue = (url) => {
|
|
23677
|
+
if (!url || queued.has(url) || queued.size >= maxQueue) return;
|
|
23678
|
+
if (!sameSite(url) || ASSET.test(new URL(url).pathname)) return;
|
|
23679
|
+
queued.add(url);
|
|
23680
|
+
idOf(url);
|
|
23681
|
+
queue.push(url);
|
|
23682
|
+
};
|
|
23683
|
+
enqueue(input.firstUrl ? crawlUrl(input.firstUrl) : void 0);
|
|
23684
|
+
if (method === "sitemap") for (const url of seeds) enqueue(url);
|
|
23685
|
+
else enqueue(start);
|
|
23686
|
+
let overflow = 0;
|
|
23687
|
+
const crawlOne = async (url) => {
|
|
23688
|
+
let fetched;
|
|
23689
|
+
try {
|
|
23690
|
+
fetched = await fetchPage(url, options);
|
|
23691
|
+
} catch (error) {
|
|
23692
|
+
skip(error instanceof Error && error.name === "RobotsAccessError" ? "robots" : "fetch-failed");
|
|
23693
|
+
return;
|
|
23694
|
+
}
|
|
23695
|
+
const finalUrl = crawlUrl(fetched.finalUrl) ?? url;
|
|
23696
|
+
if (finalUrl !== url) aliases.set(idOf(url), idOf(finalUrl));
|
|
23697
|
+
if (!sameSite(finalUrl)) return skip("off-site");
|
|
23698
|
+
if (fetched.status < 200 || fetched.status > 299) return skip("http-error");
|
|
23699
|
+
if (!/html/i.test(fetched.headers["content-type"] ?? "text/html")) return skip("not-html");
|
|
23700
|
+
let page;
|
|
23701
|
+
try {
|
|
23702
|
+
page = readPage(fetched, finalUrl);
|
|
23703
|
+
} catch {
|
|
23704
|
+
return skip("fetch-failed");
|
|
23705
|
+
}
|
|
23706
|
+
const canonical = page.canonical ? crawlUrl(page.canonical, finalUrl) : void 0;
|
|
23707
|
+
if (canonical && sameSite(canonical) && normalizePageUrl(canonical) !== normalizePageUrl(finalUrl)) {
|
|
23708
|
+
aliases.set(idOf(finalUrl), idOf(canonical));
|
|
23709
|
+
enqueue(canonical);
|
|
23710
|
+
return skip("canonicalised");
|
|
23711
|
+
}
|
|
23712
|
+
const finalId = idOf(finalUrl);
|
|
23713
|
+
if (pageOfUrl.has(finalId)) return skip("duplicate");
|
|
23714
|
+
if (pages.length >= pageLimit) {
|
|
23715
|
+
overflow += 1;
|
|
23716
|
+
return;
|
|
23717
|
+
}
|
|
23718
|
+
const text = placementPage({
|
|
23376
23719
|
...page,
|
|
23377
|
-
url:
|
|
23720
|
+
url: finalUrl,
|
|
23378
23721
|
html: fetched.html
|
|
23379
|
-
})
|
|
23722
|
+
});
|
|
23723
|
+
const contextual = /* @__PURE__ */ new Set();
|
|
23724
|
+
const other = /* @__PURE__ */ new Set();
|
|
23725
|
+
const id = pages.length;
|
|
23726
|
+
const hasMain = /<(main|article)[\s>]|role=["']main["']/i.test(fetched.html);
|
|
23727
|
+
for (const link of page.links) {
|
|
23728
|
+
const href = crawlUrl(link.href, finalUrl);
|
|
23729
|
+
if (!href || !sameSite(href) || ASSET.test(new URL(href).pathname)) continue;
|
|
23730
|
+
const hrefId = idOf(href);
|
|
23731
|
+
if (link.location === "main-content" || !hasMain && link.location === "other") {
|
|
23732
|
+
contextual.add(hrefId);
|
|
23733
|
+
if (GENERIC_ANCHORS.test(link.text.trim()) && genericAnchors.length < GENERIC_ANCHOR_LIMIT) genericAnchors.push({
|
|
23734
|
+
source: id,
|
|
23735
|
+
href,
|
|
23736
|
+
text: link.text.trim()
|
|
23737
|
+
});
|
|
23738
|
+
} else other.add(hrefId);
|
|
23739
|
+
}
|
|
23740
|
+
const record = {
|
|
23741
|
+
url: finalUrl,
|
|
23742
|
+
title: text.title,
|
|
23743
|
+
h1: text.h1,
|
|
23744
|
+
description: text.description,
|
|
23745
|
+
headings: text.headings,
|
|
23746
|
+
excerpt: excerptOf(text.sentences),
|
|
23747
|
+
words: text.sentences.reduce((sum, sentence) => sum + sentence.text.split(/\s+/).length, 0),
|
|
23748
|
+
lang: page.lang?.toLowerCase().split(/[-_]/)[0] || null,
|
|
23749
|
+
noindex: page.noindex,
|
|
23750
|
+
contextual: [...contextual],
|
|
23751
|
+
other: [...other],
|
|
23752
|
+
text: {
|
|
23753
|
+
offset: 0,
|
|
23754
|
+
length: 0
|
|
23755
|
+
}
|
|
23756
|
+
};
|
|
23757
|
+
pageOfUrl.set(finalId, id);
|
|
23758
|
+
pages.push(record);
|
|
23759
|
+
index.addPage(termCounts(text));
|
|
23760
|
+
record.text = await sentences.write(linkableSentences(text.sentences));
|
|
23761
|
+
if (method === "links" && !page.nofollow) for (const link of page.links) enqueue(crawlUrl(link.href, finalUrl));
|
|
23762
|
+
if (pages.length % 50 === 0) input.progress?.(`Crawled ${pages.length} pages of up to ${pageLimit}.`);
|
|
23763
|
+
};
|
|
23764
|
+
const worker = async () => {
|
|
23765
|
+
while (pages.length < pageLimit) {
|
|
23766
|
+
const url = queue.shift();
|
|
23767
|
+
if (!url) {
|
|
23768
|
+
if (active === 0) return;
|
|
23769
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
23770
|
+
continue;
|
|
23771
|
+
}
|
|
23772
|
+
active += 1;
|
|
23773
|
+
try {
|
|
23774
|
+
await crawlOne(url);
|
|
23775
|
+
} finally {
|
|
23776
|
+
active -= 1;
|
|
23777
|
+
}
|
|
23778
|
+
}
|
|
23380
23779
|
};
|
|
23381
|
-
|
|
23382
|
-
|
|
23383
|
-
|
|
23384
|
-
|
|
23385
|
-
|
|
23386
|
-
|
|
23387
|
-
|
|
23388
|
-
|
|
23389
|
-
|
|
23390
|
-
|
|
23391
|
-
|
|
23392
|
-
|
|
23780
|
+
let active = 0;
|
|
23781
|
+
if (method === "links" && queue[0] === start) {
|
|
23782
|
+
queue.shift();
|
|
23783
|
+
active += 1;
|
|
23784
|
+
await crawlOne(start).finally(() => {
|
|
23785
|
+
active -= 1;
|
|
23786
|
+
});
|
|
23787
|
+
const first = pages[0];
|
|
23788
|
+
if (first) origin = new URL(first.url).origin;
|
|
23789
|
+
}
|
|
23790
|
+
await Promise.all(Array.from({ length: rate.concurrency ?? 1 }, () => worker()));
|
|
23791
|
+
const capReached = pages.length >= pageLimit && (queue.length > 0 || overflow > 0);
|
|
23792
|
+
const linkedCounts = /* @__PURE__ */ new Map();
|
|
23793
|
+
for (const page of pages) for (const id of [...page.contextual, ...page.other]) if (!pageOfUrl.has(id) && !aliases.has(id)) linkedCounts.set(id, (linkedCounts.get(id) ?? 0) + 1);
|
|
23794
|
+
const toCheck = [...linkedCounts].sort((left, right) => right[1] - left[1] || left[0] - right[0]).slice(0, REDIRECT_CHECKS);
|
|
23795
|
+
let redirectsChecked = 0;
|
|
23796
|
+
await Promise.all(Array.from({ length: rate.concurrency ?? 1 }, async () => {
|
|
23797
|
+
while (toCheck.length) {
|
|
23798
|
+
const next = toCheck.shift();
|
|
23799
|
+
if (!next) break;
|
|
23800
|
+
const url = urls[next[0]];
|
|
23801
|
+
try {
|
|
23802
|
+
const fetched = await fetchPage(url, {
|
|
23803
|
+
...options,
|
|
23804
|
+
js: "off"
|
|
23805
|
+
});
|
|
23806
|
+
redirectsChecked += 1;
|
|
23807
|
+
const finalUrl = crawlUrl(fetched.finalUrl);
|
|
23808
|
+
if (finalUrl && finalUrl !== url) aliases.set(next[0], idOf(finalUrl));
|
|
23809
|
+
} catch {}
|
|
23393
23810
|
}
|
|
23811
|
+
}));
|
|
23812
|
+
for (let id = 0; id < urls.length; id += 1) {
|
|
23813
|
+
if (pageOfUrl.has(id)) continue;
|
|
23814
|
+
const seen = /* @__PURE__ */ new Set();
|
|
23815
|
+
let current = id;
|
|
23816
|
+
while (current !== void 0 && !seen.has(current)) {
|
|
23817
|
+
seen.add(current);
|
|
23818
|
+
const page = pageOfUrl.get(current);
|
|
23819
|
+
if (page !== void 0) {
|
|
23820
|
+
pageOfUrl.set(id, page);
|
|
23821
|
+
break;
|
|
23822
|
+
}
|
|
23823
|
+
current = aliases.get(current);
|
|
23824
|
+
}
|
|
23825
|
+
}
|
|
23826
|
+
index.finish();
|
|
23827
|
+
return {
|
|
23828
|
+
origin,
|
|
23829
|
+
method,
|
|
23830
|
+
pages,
|
|
23831
|
+
urls,
|
|
23832
|
+
pageOfUrl,
|
|
23833
|
+
index,
|
|
23834
|
+
sentences,
|
|
23835
|
+
genericAnchors,
|
|
23836
|
+
stats: stats({
|
|
23837
|
+
urlsFound: Math.max(queued.size, sitemap.discovery.urlsReturned),
|
|
23838
|
+
capReached,
|
|
23839
|
+
redirectsChecked
|
|
23840
|
+
}),
|
|
23841
|
+
warnings
|
|
23394
23842
|
};
|
|
23395
23843
|
}
|
|
23396
|
-
|
|
23397
|
-
|
|
23398
|
-
|
|
23399
|
-
|
|
23400
|
-
|
|
23401
|
-
|
|
23402
|
-
|
|
23403
|
-
|
|
23404
|
-
|
|
23405
|
-
|
|
23406
|
-
|
|
23407
|
-
|
|
23844
|
+
//#endregion
|
|
23845
|
+
//#region packages/core/src/analyze/internal-links-judge.ts
|
|
23846
|
+
const PAGE_TYPES = {
|
|
23847
|
+
pillar_guide: "A long, broad guide that covers a whole topic and would naturally link out to many narrower pages",
|
|
23848
|
+
support_article: "A focused article, post, tutorial, or explainer about one narrower subject",
|
|
23849
|
+
product_or_service: "Sells or describes a product, service, plan, pricing, demo, or signup: a page that makes money",
|
|
23850
|
+
comparison_or_review: "Compares options, reviews one, or lists the best of something to help a reader choose",
|
|
23851
|
+
glossary_or_definition: "A short page that defines one term or answers one 'what is' question",
|
|
23852
|
+
tool_or_calculator: "An interactive tool, calculator, generator, or checker the reader uses on the page",
|
|
23853
|
+
news_or_update: "A dated announcement, release note, changelog entry, or news item that will go stale",
|
|
23854
|
+
category_hub: "A listing, archive, tag, category, or index page that mostly lists other pages",
|
|
23855
|
+
home: "The site's home page",
|
|
23856
|
+
about_or_company: "About, team, contact, press, careers, or other company information",
|
|
23857
|
+
legal_or_utility: "Privacy, terms, cookies, login, search, 404, thank-you, or other utility pages readers do not seek out",
|
|
23858
|
+
other: null
|
|
23859
|
+
};
|
|
23860
|
+
const SEARCH_INTENTS = {
|
|
23861
|
+
informational: "The reader wants to learn or understand something",
|
|
23862
|
+
commercial: "The reader is comparing or evaluating options before buying",
|
|
23863
|
+
transactional: "The reader wants to buy, sign up, download, or complete an action now",
|
|
23864
|
+
navigational: "The reader wants to reach a specific page or part of the site"
|
|
23865
|
+
};
|
|
23866
|
+
const LINK_ROLES = {
|
|
23867
|
+
explains_concept: "The target explains a term or idea the source mentions",
|
|
23868
|
+
deeper_detail: "The target goes deeper into a sub-topic the source only touches on",
|
|
23869
|
+
broader_guide: "The target is the broader guide or hub the source belongs under",
|
|
23870
|
+
next_step: "The target is what a reader should read or do after this part",
|
|
23871
|
+
product_or_service: "The target is the product or service that solves the problem discussed",
|
|
23872
|
+
comparison_or_alternative: "The target compares or offers an alternative to what the source discusses",
|
|
23873
|
+
weak_or_none: "No clear reason a reader would want the target here"
|
|
23874
|
+
};
|
|
23875
|
+
const THRESHOLDS = {
|
|
23876
|
+
possible: .5,
|
|
23877
|
+
strong: .75
|
|
23878
|
+
};
|
|
23879
|
+
const FLOORS = {
|
|
23880
|
+
minSentenceMatch: .1,
|
|
23881
|
+
minSimilarityAlone: .15
|
|
23882
|
+
};
|
|
23883
|
+
function targetWeight(type) {
|
|
23884
|
+
if (type === "home" || type === "legal_or_utility" || type === "about_or_company" || type === "category_hub") return 0;
|
|
23885
|
+
return type === "news_or_update" ? .6 : 1;
|
|
23886
|
+
}
|
|
23887
|
+
function canBeSource(type) {
|
|
23888
|
+
return type !== "home" && type !== "category_hub" && type !== "legal_or_utility";
|
|
23889
|
+
}
|
|
23890
|
+
function classifyRequest(pages) {
|
|
23891
|
+
const questions = {};
|
|
23892
|
+
pages.forEach((_, index) => {
|
|
23893
|
+
questions[`type_${index}`] = {
|
|
23894
|
+
type: "choice",
|
|
23895
|
+
instructions: `What kind of page is \`pages[${index}]\`?`,
|
|
23896
|
+
criteria: PAGE_TYPES
|
|
23408
23897
|
};
|
|
23409
|
-
|
|
23410
|
-
|
|
23411
|
-
|
|
23412
|
-
|
|
23413
|
-
js: "auto",
|
|
23414
|
-
refresh: input.refresh
|
|
23415
|
-
});
|
|
23416
|
-
} catch (error) {
|
|
23417
|
-
return {
|
|
23418
|
-
exclusion: "failed",
|
|
23419
|
-
warnings: [{
|
|
23420
|
-
stage: "source-fetch",
|
|
23421
|
-
url: input.candidate.sourceUrl,
|
|
23422
|
-
code: "fetch-failed",
|
|
23423
|
-
message: errorMessage(error)
|
|
23424
|
-
}]
|
|
23898
|
+
questions[`intent_${index}`] = {
|
|
23899
|
+
type: "choice",
|
|
23900
|
+
instructions: `What does a visitor arriving at \`pages[${index}]\` from a search engine most likely want?`,
|
|
23901
|
+
criteria: SEARCH_INTENTS
|
|
23425
23902
|
};
|
|
23426
|
-
}
|
|
23427
|
-
|
|
23428
|
-
|
|
23429
|
-
|
|
23430
|
-
|
|
23431
|
-
|
|
23432
|
-
|
|
23433
|
-
|
|
23434
|
-
|
|
23435
|
-
|
|
23436
|
-
|
|
23437
|
-
|
|
23438
|
-
|
|
23903
|
+
});
|
|
23904
|
+
return {
|
|
23905
|
+
state: { pages: pages.map((page) => ({
|
|
23906
|
+
url: page.url,
|
|
23907
|
+
title: page.title,
|
|
23908
|
+
description: page.description,
|
|
23909
|
+
h1: page.h1,
|
|
23910
|
+
headings: page.headings,
|
|
23911
|
+
excerpt: page.excerpt,
|
|
23912
|
+
word_count: page.words
|
|
23913
|
+
})) },
|
|
23914
|
+
questions
|
|
23915
|
+
};
|
|
23916
|
+
}
|
|
23917
|
+
function choiceOf(answer) {
|
|
23918
|
+
return answer?.type === "choice" ? answer.choice : null;
|
|
23919
|
+
}
|
|
23920
|
+
function readClassification(answers, index) {
|
|
23921
|
+
const type = choiceOf(answers[`type_${index}`]);
|
|
23922
|
+
const intent = choiceOf(answers[`intent_${index}`]);
|
|
23923
|
+
return {
|
|
23924
|
+
type: type && Object.hasOwn(PAGE_TYPES, type) ? type : null,
|
|
23925
|
+
intent: intent && Object.hasOwn(SEARCH_INTENTS, intent) ? intent : null
|
|
23926
|
+
};
|
|
23927
|
+
}
|
|
23928
|
+
function judgeRequest(source, targets) {
|
|
23929
|
+
const sentences = /* @__PURE__ */ new Map();
|
|
23930
|
+
for (const target of targets) for (const sentence of target.placement.sentences) sentences.set(sentence.id, {
|
|
23931
|
+
id: sentence.id,
|
|
23932
|
+
under_heading: sentence.section,
|
|
23933
|
+
text: sentence.text
|
|
23934
|
+
});
|
|
23935
|
+
const questions = {};
|
|
23936
|
+
targets.forEach((target, index) => {
|
|
23937
|
+
const ref = `\`targets[${index}]\``;
|
|
23938
|
+
questions[`link_${index}`] = {
|
|
23939
|
+
type: "noul",
|
|
23940
|
+
instructions: `Should the page \`source\` add a link from within its body text to the page ${ref}?`,
|
|
23941
|
+
criteria: {
|
|
23942
|
+
true: "A reader of the source would gain something by reading the target here; the two pages cover related subjects and the link would feel natural and helpful",
|
|
23943
|
+
false: "The pages are only loosely related, the target adds nothing for this reader, or the link would feel forced or spammy"
|
|
23944
|
+
}
|
|
23945
|
+
};
|
|
23946
|
+
questions[`where_${index}`] = {
|
|
23947
|
+
type: "choice",
|
|
23948
|
+
instructions: `Which sentence in \`source.sentences\` is the most natural place to add a link to ${ref}? Choose "none" if no listed sentence leads a reader towards that page.`,
|
|
23949
|
+
criteria: {
|
|
23950
|
+
...Object.fromEntries(target.placement.sentences.map((sentence) => [sentence.id, sentence.text])),
|
|
23951
|
+
none: "No listed sentence is a natural place for this link"
|
|
23952
|
+
}
|
|
23953
|
+
};
|
|
23954
|
+
if (target.placement.anchors.length) questions[`anchor_${index}`] = {
|
|
23955
|
+
type: "choice",
|
|
23956
|
+
instructions: `Which phrase should become the clickable link text for a link from \`source\` to ${ref}? Prefer a phrase that tells the reader what the target page is about and reads naturally as a link.`,
|
|
23957
|
+
criteria: {
|
|
23958
|
+
...Object.fromEntries(target.placement.anchors.map((anchor, at) => [`A${at + 1}`, {
|
|
23959
|
+
text: anchor.text,
|
|
23960
|
+
in_sentence: anchor.sentence
|
|
23961
|
+
}])),
|
|
23962
|
+
none: "None of these phrases would make a good link"
|
|
23963
|
+
}
|
|
23964
|
+
};
|
|
23965
|
+
questions[`role_${index}`] = {
|
|
23966
|
+
type: "choice",
|
|
23967
|
+
instructions: `If \`source\` linked to ${ref}, what would the link do for the reader?`,
|
|
23968
|
+
criteria: LINK_ROLES
|
|
23439
23969
|
};
|
|
23440
|
-
}
|
|
23441
|
-
const warnings = page.warnings.map((message) => ({
|
|
23442
|
-
stage: "source-extract",
|
|
23443
|
-
url: input.candidate.sourceUrl,
|
|
23444
|
-
code: "extractor-warning",
|
|
23445
|
-
message
|
|
23446
|
-
}));
|
|
23447
|
-
if (input.target.aliases.some((alias) => sameObservedUrl(page.finalUrl, alias))) return {
|
|
23448
|
-
exclusion: "self-alias",
|
|
23449
|
-
warnings
|
|
23450
|
-
};
|
|
23451
|
-
const technicalSignals = pageTechnicalSignals({
|
|
23452
|
-
url: input.candidate.sourceUrl,
|
|
23453
|
-
page,
|
|
23454
|
-
fetchDiagnostics: fetched.diagnostics,
|
|
23455
|
-
httpStatus: fetched.status
|
|
23456
23970
|
});
|
|
23457
|
-
if (technicalSignals.length) return {
|
|
23458
|
-
exclusion: "technical",
|
|
23459
|
-
warnings
|
|
23460
|
-
};
|
|
23461
|
-
const observed = linksToAliases(page, input.target.aliases).map((link) => ({
|
|
23462
|
-
href: link.href,
|
|
23463
|
-
text: link.text,
|
|
23464
|
-
rel: link.rel,
|
|
23465
|
-
location: link.location
|
|
23466
|
-
}));
|
|
23467
|
-
const contextual = observed.filter((link) => link.location === "main-content");
|
|
23468
|
-
if (contextual.some((link) => sameObservedUrl(link.href, input.target.preferredUrl))) return {
|
|
23469
|
-
exclusion: "existing-link",
|
|
23470
|
-
warnings
|
|
23471
|
-
};
|
|
23472
|
-
const aliasContextual = contextual.length > 0;
|
|
23473
|
-
const linkStatus = aliasContextual ? "alias-contextual" : observed.length ? "non-contextual-only" : "missing";
|
|
23474
|
-
const confidence = input.candidate.bestMatchKind === "exact-query" && !page.contentExtraction.fallback ? "medium" : "low";
|
|
23475
|
-
const actionType = aliasContextual ? "review-alias-link" : "review-contextual-link";
|
|
23476
|
-
const evidenceRef = aliasContextual ? `A main-content link points to a non-preferred target alias; the preferred target is ${input.target.preferredUrl}.` : `${input.candidate.sourceUrl} matched ${input.candidate.matchedQueries} retained GSC query row${input.candidate.matchedQueries === 1 ? "" : "s"}, and no main-content link to the preferred target was observed in the fetched HTML.`;
|
|
23477
|
-
const action = aliasContextual ? `Review the existing contextual link and update it to the preferred target ${input.target.preferredUrl} if the redirect or old alias is not intentional.` : `Review whether this source page and ${input.target.preferredUrl} serve complementary intent. If the link would help a reader, add a natural main-content link without forcing exact-match anchor text.`;
|
|
23478
23971
|
return {
|
|
23479
|
-
|
|
23480
|
-
|
|
23481
|
-
|
|
23482
|
-
|
|
23483
|
-
|
|
23484
|
-
|
|
23485
|
-
pageWarnings: page.warnings,
|
|
23486
|
-
actionType,
|
|
23487
|
-
linkEvidence: {
|
|
23488
|
-
status: linkStatus,
|
|
23489
|
-
observedCount: observed.length,
|
|
23490
|
-
observedLimit: 20,
|
|
23491
|
-
limitedCount: Math.max(0, observed.length - 20),
|
|
23492
|
-
observed: observed.slice(0, 20)
|
|
23972
|
+
state: {
|
|
23973
|
+
source: {
|
|
23974
|
+
url: source.url,
|
|
23975
|
+
title: source.title,
|
|
23976
|
+
page_type: source.type ?? "unknown",
|
|
23977
|
+
sentences: [...sentences.values()].sort((left, right) => Number(left.id.slice(1)) - Number(right.id.slice(1)))
|
|
23493
23978
|
},
|
|
23494
|
-
|
|
23495
|
-
|
|
23496
|
-
|
|
23497
|
-
|
|
23498
|
-
|
|
23499
|
-
|
|
23500
|
-
|
|
23501
|
-
|
|
23502
|
-
}
|
|
23979
|
+
targets: targets.map((target) => ({
|
|
23980
|
+
url: target.url,
|
|
23981
|
+
title: target.title,
|
|
23982
|
+
description: target.description,
|
|
23983
|
+
h1: target.h1,
|
|
23984
|
+
page_type: target.type ?? "unknown",
|
|
23985
|
+
search_intent: target.intent ?? "unknown",
|
|
23986
|
+
opening: target.opening
|
|
23987
|
+
}))
|
|
23503
23988
|
},
|
|
23504
|
-
|
|
23505
|
-
|
|
23506
|
-
|
|
23507
|
-
|
|
23508
|
-
|
|
23509
|
-
|
|
23989
|
+
questions
|
|
23990
|
+
};
|
|
23991
|
+
}
|
|
23992
|
+
function floor(sentence, similarity, probability) {
|
|
23993
|
+
if (sentence && sentence.match < FLOORS.minSentenceMatch) return {
|
|
23994
|
+
state: "held-back",
|
|
23995
|
+
reason: "weak-sentence",
|
|
23996
|
+
detail: `the sentence has ${Math.round(sentence.match * 100)}% of the words that name the page; a link needs ${Math.round(FLOORS.minSentenceMatch * 100)}%`,
|
|
23997
|
+
probability
|
|
23998
|
+
};
|
|
23999
|
+
if (!sentence && similarity < FLOORS.minSimilarityAlone) return {
|
|
24000
|
+
state: "held-back",
|
|
24001
|
+
reason: "no-sentence",
|
|
24002
|
+
detail: `no sentence to place it in, and the pages are ${similarity.toFixed(2)} similar; a link without a sentence needs ${FLOORS.minSimilarityAlone}`,
|
|
24003
|
+
probability
|
|
24004
|
+
};
|
|
24005
|
+
return null;
|
|
24006
|
+
}
|
|
24007
|
+
function decideWithModel(placement, similarity, answers, index) {
|
|
24008
|
+
const link = answers[`link_${index}`];
|
|
24009
|
+
const role = choiceOf(answers[`role_${index}`]);
|
|
24010
|
+
if (link?.type !== "noul" || !role) return {
|
|
24011
|
+
state: "declined",
|
|
24012
|
+
reason: "no-answer",
|
|
24013
|
+
probability: null
|
|
24014
|
+
};
|
|
24015
|
+
const probability = Math.round(link.noul * 1e3) / 1e3;
|
|
24016
|
+
if (probability < THRESHOLDS.possible) return {
|
|
24017
|
+
state: "declined",
|
|
24018
|
+
reason: "below-threshold",
|
|
24019
|
+
probability
|
|
24020
|
+
};
|
|
24021
|
+
if (role === "weak_or_none" || !Object.hasOwn(LINK_ROLES, role)) return {
|
|
24022
|
+
state: "declined",
|
|
24023
|
+
reason: "weak-role",
|
|
24024
|
+
probability
|
|
24025
|
+
};
|
|
24026
|
+
const where = choiceOf(answers[`where_${index}`]);
|
|
24027
|
+
const sentence = placement.sentences.find((option) => option.id === where) ?? null;
|
|
24028
|
+
const held = floor(sentence, similarity, probability);
|
|
24029
|
+
if (held) return held;
|
|
24030
|
+
let anchor = null;
|
|
24031
|
+
const picked = answers[`anchor_${index}`];
|
|
24032
|
+
if (sentence && picked?.type === "choice" && picked.choice !== "none") {
|
|
24033
|
+
const inSentence = placement.anchors.map((option, at) => ({
|
|
24034
|
+
...option,
|
|
24035
|
+
key: `A${at + 1}`
|
|
24036
|
+
})).filter((option) => option.sentence === sentence.id);
|
|
24037
|
+
anchor = (inSentence.find((option) => option.key === picked.choice) ?? [...inSentence].sort((left, right) => (picked.probabilities[right.key] ?? 0) - (picked.probabilities[left.key] ?? 0) || left.key.localeCompare(right.key))[0])?.text ?? null;
|
|
24038
|
+
}
|
|
24039
|
+
return {
|
|
24040
|
+
state: "suggest",
|
|
24041
|
+
tier: probability >= THRESHOLDS.strong ? "strong" : "possible",
|
|
24042
|
+
probability,
|
|
24043
|
+
role,
|
|
24044
|
+
sentence: sentence ? {
|
|
24045
|
+
text: sentence.text,
|
|
24046
|
+
section: sentence.section,
|
|
24047
|
+
match: sentence.match
|
|
24048
|
+
} : null,
|
|
24049
|
+
anchor
|
|
24050
|
+
};
|
|
24051
|
+
}
|
|
24052
|
+
function decideWithoutModel(placement, similarity) {
|
|
24053
|
+
const sentence = placement.sentences[0] ?? null;
|
|
24054
|
+
const held = floor(sentence, similarity, null);
|
|
24055
|
+
if (held) return held;
|
|
24056
|
+
return {
|
|
24057
|
+
state: "suggest",
|
|
24058
|
+
tier: "heuristic",
|
|
24059
|
+
probability: null,
|
|
24060
|
+
role: null,
|
|
24061
|
+
sentence: sentence ? {
|
|
24062
|
+
text: sentence.text,
|
|
24063
|
+
section: sentence.section,
|
|
24064
|
+
match: sentence.match
|
|
24065
|
+
} : null,
|
|
24066
|
+
anchor: placement.anchors.find((option) => option.sentence === sentence?.id)?.text ?? null
|
|
23510
24067
|
};
|
|
23511
24068
|
}
|
|
23512
24069
|
//#endregion
|
|
24070
|
+
//#region packages/core/src/analyze/internal-links-report.ts
|
|
24071
|
+
const DAY$1 = 864e5;
|
|
24072
|
+
const ISO_DAY = /^\d{4}-\d{2}-\d{2}$/;
|
|
24073
|
+
function internalLinksReportRange(input, fallbackDays, now) {
|
|
24074
|
+
if (!input.startDate && !input.endDate) return {
|
|
24075
|
+
...finalGscDateRange(fallbackDays, now),
|
|
24076
|
+
days: fallbackDays
|
|
24077
|
+
};
|
|
24078
|
+
const { startDate, endDate } = input;
|
|
24079
|
+
if (!startDate || !endDate || !ISO_DAY.test(startDate) || !ISO_DAY.test(endDate) || startDate > endDate) throw new SeoError("INVALID_INPUT", "Send a start date and an end date together, in order, as YYYY-MM-DD.");
|
|
24080
|
+
return {
|
|
24081
|
+
startDate,
|
|
24082
|
+
endDate,
|
|
24083
|
+
days: Math.round((Date.parse(`${endDate}T00:00:00Z`) - Date.parse(`${startDate}T00:00:00Z`)) / DAY$1) + 1
|
|
24084
|
+
};
|
|
24085
|
+
}
|
|
24086
|
+
const plural$3 = (count, word) => `${count} ${word}${count === 1 ? "" : "s"}`;
|
|
24087
|
+
function internalLinksVerdict(report) {
|
|
24088
|
+
const { counts } = report;
|
|
24089
|
+
if (report.dataStatus === "blocked") return "robots.txt does not allow this crawler to read the site, so nothing was checked.";
|
|
24090
|
+
if (report.dataStatus === "empty") return "No HTML page could be crawled, so there is nothing to link.";
|
|
24091
|
+
if (report.dataStatus === "target-not-crawled") return "The target page could not be crawled as its own page (it failed, left the site, or names another page as canonical). Check it with seo page first.";
|
|
24092
|
+
if (report.dataStatus === "target-not-linkable") return `The target is ${report.target?.noindex ? "noindex" : `a ${report.target?.type?.replaceAll("_", " ")} page`}, so no links to it are suggested.`;
|
|
24093
|
+
const judged = report.judge.status === "used" || report.judge.status === "partial" ? `${counts.strong} strong and ${counts.possible} possible link${counts.strong + counts.possible === 1 ? "" : "s"} judged by ${report.judge.model}${counts.heuristic ? `, plus ${plural$3(counts.heuristic, "heuristic link")} where a request failed` : ""}` : `${plural$3(counts.heuristic, "heuristic link")} (no decision model)`;
|
|
24094
|
+
const scope = report.mode === "url" ? `to ${report.target?.url}` : `across ${plural$3(counts.pagesCrawled, "page")}`;
|
|
24095
|
+
const capped = report.crawl.capReached ? ` The crawl stopped at its limit of ${report.crawl.pageLimit} pages, so pages beyond it were not checked.` : "";
|
|
24096
|
+
return `${judged} ${scope}${counts.heldBack ? `; ${counts.heldBack} held back by the evidence floors` : ""}.${capped}`;
|
|
24097
|
+
}
|
|
24098
|
+
//#endregion
|
|
23513
24099
|
//#region packages/core/src/analyze/internal-links.ts
|
|
24100
|
+
const GSC_DAYS = 90;
|
|
23514
24101
|
const MAX_GSC_ROWS = 1e5;
|
|
23515
|
-
const
|
|
23516
|
-
const
|
|
23517
|
-
const
|
|
23518
|
-
const
|
|
23519
|
-
const
|
|
23520
|
-
|
|
23521
|
-
|
|
23522
|
-
|
|
23523
|
-
|
|
23524
|
-
|
|
23525
|
-
|
|
23526
|
-
|
|
23527
|
-
|
|
23528
|
-
const
|
|
23529
|
-
|
|
23530
|
-
|
|
23531
|
-
|
|
23532
|
-
|
|
23533
|
-
const
|
|
23534
|
-
|
|
23535
|
-
|
|
23536
|
-
|
|
23537
|
-
|
|
23538
|
-
|
|
24102
|
+
const MAX_PAGES_PER_QUERY = 25;
|
|
24103
|
+
const MAX_GSC_PAIRS = 5e5;
|
|
24104
|
+
const MIN_SIMILARITY = .08;
|
|
24105
|
+
const TARGETS_PER_SOURCE = 8;
|
|
24106
|
+
const EXTRA_GSC_TARGETS = 2;
|
|
24107
|
+
const URL_SOURCES_PLACED = 60;
|
|
24108
|
+
const URL_SOURCES_JUDGED = 24;
|
|
24109
|
+
const SOURCE_WINDOW = 64;
|
|
24110
|
+
const WEAK_INBOUND = 3;
|
|
24111
|
+
const EXAMPLE_LIMIT = 20;
|
|
24112
|
+
const LIST_LIMIT = 100;
|
|
24113
|
+
const OPENING_CHARS = 300;
|
|
24114
|
+
function siteStartUrl(site) {
|
|
24115
|
+
const value = site.trim();
|
|
24116
|
+
if (value.startsWith("sc-domain:")) return `https://${value.slice(10)}/`;
|
|
24117
|
+
return `${new URL(/^https?:\/\//i.test(value) ? value : `https://${value}`).origin}/`;
|
|
24118
|
+
}
|
|
24119
|
+
function round$1(value, places = 3) {
|
|
24120
|
+
const factor = 10 ** places;
|
|
24121
|
+
return Math.round(value * factor) / factor;
|
|
24122
|
+
}
|
|
24123
|
+
async function internalLinksReport(input, dependencies) {
|
|
24124
|
+
const deps = {
|
|
24125
|
+
crawl: crawlForLinks,
|
|
24126
|
+
now: () => /* @__PURE__ */ new Date(),
|
|
24127
|
+
...dependencies
|
|
24128
|
+
};
|
|
24129
|
+
const mode = input.targetUrl ? "url" : "site";
|
|
24130
|
+
if (mode === "url" && input.maxPages !== void 0) throw new SeoError("INVALID_INPUT", `--max-pages is for a site audit only. A page run crawls up to 500 pages.`);
|
|
24131
|
+
const maxPages = input.maxPages ?? 500;
|
|
24132
|
+
if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > 5e3) throw new SeoError("INVALID_INPUT", `--max-pages is a whole number from 1 to ${MAX_MAX_PAGES.toLocaleString("en-US")}.`);
|
|
24133
|
+
const targetUrl = input.targetUrl ? crawlUrl(input.targetUrl) : void 0;
|
|
24134
|
+
if (input.targetUrl && !targetUrl) throw new SeoError("INVALID_INPUT", "The target is an http or https URL.");
|
|
24135
|
+
const now = deps.now();
|
|
24136
|
+
const startUrl = targetUrl ? `${new URL(targetUrl).origin}/` : siteStartUrl(input.site);
|
|
24137
|
+
deps.progress?.(`Crawling ${startUrl} (up to ${maxPages} pages).`);
|
|
24138
|
+
const crawl = await deps.crawl({
|
|
24139
|
+
startUrl,
|
|
24140
|
+
maxPages,
|
|
24141
|
+
firstUrl: targetUrl,
|
|
23539
24142
|
refresh: input.refresh,
|
|
23540
|
-
|
|
23541
|
-
});
|
|
23542
|
-
const targetRequests = await Promise.all(target.aliases.map(async (alias) => {
|
|
23543
|
-
const result = await dependencies.searchAnalytics(input.site, {
|
|
23544
|
-
...range,
|
|
23545
|
-
dimensions: ["query", "page"],
|
|
23546
|
-
type: "web",
|
|
23547
|
-
dataState: "final",
|
|
23548
|
-
maxRows: MAX_GSC_ROWS,
|
|
23549
|
-
dimensionFilterGroups: [{
|
|
23550
|
-
groupType: "and",
|
|
23551
|
-
filters: [{
|
|
23552
|
-
dimension: "page",
|
|
23553
|
-
operator: "equals",
|
|
23554
|
-
expression: alias
|
|
23555
|
-
}]
|
|
23556
|
-
}]
|
|
23557
|
-
}, { refresh: input.refresh });
|
|
23558
|
-
ledger.addGsc(result.calls, result.rowsFetched);
|
|
23559
|
-
return {
|
|
23560
|
-
alias,
|
|
23561
|
-
result
|
|
23562
|
-
};
|
|
23563
|
-
}));
|
|
23564
|
-
const targetRows = targetRequests.flatMap(({ result }) => result.rows);
|
|
23565
|
-
const preliminary = analyzeInternalLinksFromRows({
|
|
23566
|
-
targetRows,
|
|
23567
|
-
sourceRows: [],
|
|
23568
|
-
site: input.site,
|
|
23569
|
-
targetAliases: target.aliases,
|
|
23570
|
-
minImpressions,
|
|
23571
|
-
brandTerms
|
|
24143
|
+
progress: deps.progress
|
|
23572
24144
|
});
|
|
23573
|
-
|
|
23574
|
-
|
|
23575
|
-
|
|
23576
|
-
|
|
23577
|
-
|
|
23578
|
-
|
|
23579
|
-
|
|
23580
|
-
}
|
|
23581
|
-
|
|
23582
|
-
|
|
23583
|
-
|
|
24145
|
+
try {
|
|
24146
|
+
return await analyse(input, deps, crawl, {
|
|
24147
|
+
mode,
|
|
24148
|
+
targetUrl,
|
|
24149
|
+
startUrl,
|
|
24150
|
+
now
|
|
24151
|
+
});
|
|
24152
|
+
} finally {
|
|
24153
|
+
await crawl.sentences.close();
|
|
24154
|
+
}
|
|
24155
|
+
}
|
|
24156
|
+
async function analyse(input, deps, crawl, context) {
|
|
24157
|
+
const { pages, index } = crawl;
|
|
24158
|
+
const count = pages.length;
|
|
24159
|
+
const warnings = [...crawl.warnings];
|
|
24160
|
+
const lookup = /* @__PURE__ */ new Map();
|
|
24161
|
+
crawl.urls.forEach((url, id) => {
|
|
24162
|
+
const page = crawl.pageOfUrl.get(id);
|
|
24163
|
+
if (page === void 0) return;
|
|
24164
|
+
lookup.set(url, page);
|
|
24165
|
+
if (!lookup.has(normalizePageUrl(url))) lookup.set(normalizePageUrl(url), page);
|
|
24166
|
+
});
|
|
24167
|
+
const pageAt = (url) => {
|
|
24168
|
+
const clean = crawlUrl(url);
|
|
24169
|
+
if (!clean) return void 0;
|
|
24170
|
+
return lookup.get(clean) ?? lookup.get(normalizePageUrl(clean));
|
|
24171
|
+
};
|
|
24172
|
+
const target = context.targetUrl !== void 0 ? pageAt(context.targetUrl) : void 0;
|
|
24173
|
+
const contextual = pages.map(() => /* @__PURE__ */ new Set());
|
|
24174
|
+
const anywhere = pages.map(() => /* @__PURE__ */ new Set());
|
|
24175
|
+
const redirectedLinks = [];
|
|
24176
|
+
pages.forEach((page, source) => {
|
|
24177
|
+
for (const id of page.contextual) {
|
|
24178
|
+
const to = crawl.pageOfUrl.get(id);
|
|
24179
|
+
if (to === void 0 || to === source) continue;
|
|
24180
|
+
contextual[source]?.add(to);
|
|
24181
|
+
anywhere[source]?.add(to);
|
|
24182
|
+
const via = crawl.urls[id];
|
|
24183
|
+
const final = pages[to]?.url;
|
|
24184
|
+
if (normalizePageUrl(via) !== normalizePageUrl(final) && redirectedLinks.length < LIST_LIMIT) redirectedLinks.push({
|
|
24185
|
+
source: page.url,
|
|
24186
|
+
via,
|
|
24187
|
+
target: final
|
|
24188
|
+
});
|
|
24189
|
+
}
|
|
24190
|
+
for (const id of page.other) {
|
|
24191
|
+
const to = crawl.pageOfUrl.get(id);
|
|
24192
|
+
if (to !== void 0 && to !== source) anywhere[source]?.add(to);
|
|
24193
|
+
}
|
|
24194
|
+
});
|
|
24195
|
+
const inbound = new Array(count).fill(0);
|
|
24196
|
+
const inboundAnywhere = new Array(count).fill(0);
|
|
24197
|
+
contextual.forEach((set) => {
|
|
24198
|
+
for (const to of set) inbound[to] = (inbound[to] ?? 0) + 1;
|
|
24199
|
+
});
|
|
24200
|
+
anywhere.forEach((set) => {
|
|
24201
|
+
for (const to of set) inboundAnywhere[to] = (inboundAnywhere[to] ?? 0) + 1;
|
|
24202
|
+
});
|
|
24203
|
+
const existingContextualLinks = contextual.reduce((sum, set) => sum + set.size, 0);
|
|
24204
|
+
const range = internalLinksReportRange(input, GSC_DAYS, context.now);
|
|
24205
|
+
const sharedQueries = /* @__PURE__ */ new Map();
|
|
24206
|
+
const pairKey = (source, target) => source * count + target;
|
|
24207
|
+
let searchConsole = {
|
|
24208
|
+
status: "not-available",
|
|
24209
|
+
property: null,
|
|
24210
|
+
range: null,
|
|
24211
|
+
rows: 0,
|
|
24212
|
+
possiblyTruncated: false,
|
|
24213
|
+
pairsWithSharedQueries: 0,
|
|
24214
|
+
note: "unavailable" in deps.searchConsole ? deps.searchConsole.unavailable : null
|
|
24215
|
+
};
|
|
24216
|
+
if ("property" in deps.searchConsole && count > 1) {
|
|
24217
|
+
const { startDate, endDate } = range;
|
|
24218
|
+
try {
|
|
24219
|
+
deps.progress?.("Reading Search Console queries.");
|
|
24220
|
+
const result = await deps.searchConsole.query({
|
|
24221
|
+
startDate,
|
|
24222
|
+
endDate,
|
|
24223
|
+
dimensions: ["query", "page"],
|
|
24224
|
+
type: "web",
|
|
24225
|
+
dataState: "final",
|
|
24226
|
+
maxRows: MAX_GSC_ROWS
|
|
24227
|
+
});
|
|
24228
|
+
const brandTerms = deriveBrandTerms({ siteUrl: crawl.origin });
|
|
24229
|
+
const byQuery = /* @__PURE__ */ new Map();
|
|
24230
|
+
for (const row of result.rows) {
|
|
24231
|
+
const [query, url] = row.keys;
|
|
24232
|
+
if (!query || !url || row.impressions < 1) continue;
|
|
24233
|
+
if (isBrandQuery(query, brandTerms)) continue;
|
|
24234
|
+
const page = pageAt(url);
|
|
24235
|
+
if (page === void 0) continue;
|
|
24236
|
+
let set = byQuery.get(query);
|
|
24237
|
+
if (!set) {
|
|
24238
|
+
set = /* @__PURE__ */ new Set();
|
|
24239
|
+
byQuery.set(query, set);
|
|
24240
|
+
}
|
|
24241
|
+
set.add(page);
|
|
24242
|
+
}
|
|
24243
|
+
let capped = false;
|
|
24244
|
+
for (const set of byQuery.values()) {
|
|
24245
|
+
if (set.size < 2 || set.size > MAX_PAGES_PER_QUERY) continue;
|
|
24246
|
+
const list = [...set];
|
|
24247
|
+
for (const source of list) for (const to of list) {
|
|
24248
|
+
if (source === to) continue;
|
|
24249
|
+
if (target !== void 0 && to !== target) continue;
|
|
24250
|
+
const key = pairKey(source, to);
|
|
24251
|
+
if (!sharedQueries.has(key) && sharedQueries.size >= MAX_GSC_PAIRS) {
|
|
24252
|
+
capped = true;
|
|
24253
|
+
continue;
|
|
24254
|
+
}
|
|
24255
|
+
sharedQueries.set(key, (sharedQueries.get(key) ?? 0) + 1);
|
|
24256
|
+
}
|
|
24257
|
+
}
|
|
24258
|
+
searchConsole = {
|
|
24259
|
+
status: "used",
|
|
24260
|
+
property: deps.searchConsole.property,
|
|
24261
|
+
range: {
|
|
24262
|
+
startDate,
|
|
24263
|
+
endDate
|
|
24264
|
+
},
|
|
24265
|
+
rows: result.rowsFetched,
|
|
24266
|
+
possiblyTruncated: result.rowsFetched >= MAX_GSC_ROWS,
|
|
24267
|
+
pairsWithSharedQueries: sharedQueries.size,
|
|
24268
|
+
note: capped ? `Shared queries were counted for the first ${MAX_GSC_PAIRS.toLocaleString("en-US")} page pairs.` : null
|
|
24269
|
+
};
|
|
24270
|
+
} catch (error) {
|
|
24271
|
+
searchConsole = {
|
|
24272
|
+
...searchConsole,
|
|
24273
|
+
status: "failed",
|
|
24274
|
+
property: deps.searchConsole.property,
|
|
24275
|
+
note: error instanceof Error ? error.message : String(error)
|
|
24276
|
+
};
|
|
24277
|
+
}
|
|
24278
|
+
}
|
|
24279
|
+
const runner = "model" in deps.decisionModel ? new DecisionRunner(deps.decisionModel.model) : null;
|
|
24280
|
+
const types = pages.map((page) => new URL(page.url).pathname === "/" ? "home" : null);
|
|
24281
|
+
const intents = pages.map(() => null);
|
|
24282
|
+
let pagesClassified = 0;
|
|
24283
|
+
const classify = async (ids) => {
|
|
24284
|
+
if (!runner) return;
|
|
24285
|
+
const batches = [];
|
|
24286
|
+
for (let at = 0; at < ids.length; at += 5) batches.push(ids.slice(at, at + 5));
|
|
24287
|
+
deps.progress?.(`Classifying ${ids.length} pages in ${batches.length} requests.`);
|
|
24288
|
+
await Promise.all(batches.map(async (batch) => {
|
|
24289
|
+
const response = await runner.ask(classifyRequest(batch.map((id) => pages[id])));
|
|
24290
|
+
if (!response) return;
|
|
24291
|
+
batch.forEach((id, at) => {
|
|
24292
|
+
const read = readClassification(response.answers, at);
|
|
24293
|
+
if (read.type) types[id] = read.type;
|
|
24294
|
+
intents[id] = read.intent;
|
|
24295
|
+
pagesClassified += 1;
|
|
24296
|
+
});
|
|
24297
|
+
}));
|
|
23584
24298
|
};
|
|
23585
|
-
|
|
23586
|
-
|
|
23587
|
-
|
|
23588
|
-
|
|
23589
|
-
|
|
23590
|
-
|
|
23591
|
-
|
|
23592
|
-
|
|
23593
|
-
}
|
|
23594
|
-
const
|
|
23595
|
-
|
|
23596
|
-
|
|
23597
|
-
|
|
23598
|
-
|
|
23599
|
-
|
|
23600
|
-
|
|
23601
|
-
|
|
23602
|
-
|
|
23603
|
-
|
|
23604
|
-
|
|
23605
|
-
|
|
23606
|
-
|
|
23607
|
-
|
|
24299
|
+
const linkable = (source, to) => {
|
|
24300
|
+
const from = pages[source];
|
|
24301
|
+
const page = pages[to];
|
|
24302
|
+
if (!from || !page || source === to) return false;
|
|
24303
|
+
if (page.noindex || targetWeight(types[to] ?? null) === 0) return false;
|
|
24304
|
+
if (contextual[source]?.has(to)) return false;
|
|
24305
|
+
if (from.lang && page.lang && from.lang !== page.lang) return false;
|
|
24306
|
+
return canBeSource(types[source] ?? null);
|
|
24307
|
+
};
|
|
24308
|
+
const adjusted = (to, raw) => raw * ((inbound[to] ?? 0) < WEAK_INBOUND ? 1.25 : 1) * targetWeight(types[to] ?? null);
|
|
24309
|
+
let status = "complete";
|
|
24310
|
+
const candidatesBySource = /* @__PURE__ */ new Map();
|
|
24311
|
+
if (crawl.stats.blockedByRobots) status = "blocked";
|
|
24312
|
+
else if (count === 0) status = "empty";
|
|
24313
|
+
else if (context.mode === "url" && target === void 0) status = "target-not-crawled";
|
|
24314
|
+
if (status === "complete" && context.mode === "site") {
|
|
24315
|
+
await classify(pages.map((_, id) => id));
|
|
24316
|
+
const extra = /* @__PURE__ */ new Map();
|
|
24317
|
+
for (const key of sharedQueries.keys()) {
|
|
24318
|
+
const source = Math.floor(key / count);
|
|
24319
|
+
const list = extra.get(source) ?? [];
|
|
24320
|
+
list.push(key % count);
|
|
24321
|
+
extra.set(source, list);
|
|
24322
|
+
}
|
|
24323
|
+
for (let source = 0; source < count; source += 1) {
|
|
24324
|
+
if (!canBeSource(types[source] ?? null)) continue;
|
|
24325
|
+
const chosen = index.neighbours(source, MIN_SIMILARITY).filter(({ page }) => linkable(source, page)).map(({ page, score }) => ({
|
|
24326
|
+
source,
|
|
24327
|
+
target: page,
|
|
24328
|
+
raw: score,
|
|
24329
|
+
similarity: adjusted(page, score),
|
|
24330
|
+
sharedQueries: sharedQueries.get(pairKey(source, page)) ?? 0
|
|
24331
|
+
})).sort((left, right) => right.similarity - left.similarity || left.target - right.target).slice(0, TARGETS_PER_SOURCE);
|
|
24332
|
+
const taken = new Set(chosen.map((candidate) => candidate.target));
|
|
24333
|
+
const fromQueries = (extra.get(source) ?? []).filter((to) => !taken.has(to) && linkable(source, to)).map((to) => {
|
|
24334
|
+
const raw = index.similarity(source, to);
|
|
24335
|
+
return {
|
|
24336
|
+
source,
|
|
24337
|
+
target: to,
|
|
24338
|
+
raw,
|
|
24339
|
+
similarity: adjusted(to, raw),
|
|
24340
|
+
sharedQueries: sharedQueries.get(pairKey(source, to)) ?? 0
|
|
24341
|
+
};
|
|
24342
|
+
}).sort((left, right) => right.sharedQueries - left.sharedQueries || right.similarity - left.similarity || left.target - right.target).slice(0, EXTRA_GSC_TARGETS);
|
|
24343
|
+
const all = [...chosen, ...fromQueries];
|
|
24344
|
+
if (all.length) candidatesBySource.set(source, all);
|
|
24345
|
+
}
|
|
24346
|
+
}
|
|
24347
|
+
if (status === "complete" && context.mode === "url" && target !== void 0) {
|
|
24348
|
+
const near = /* @__PURE__ */ new Map();
|
|
24349
|
+
for (const { page, score } of index.neighbours(target, MIN_SIMILARITY)) near.set(page, score);
|
|
24350
|
+
for (const key of sharedQueries.keys()) {
|
|
24351
|
+
const source = Math.floor(key / count);
|
|
24352
|
+
if (!near.has(source)) near.set(source, index.similarity(source, target));
|
|
24353
|
+
}
|
|
24354
|
+
const ranked = [...near].map(([source, raw]) => ({
|
|
24355
|
+
source,
|
|
23608
24356
|
target,
|
|
23609
|
-
|
|
23610
|
-
|
|
23611
|
-
|
|
23612
|
-
|
|
23613
|
-
|
|
23614
|
-
|
|
23615
|
-
|
|
23616
|
-
|
|
23617
|
-
|
|
23618
|
-
|
|
23619
|
-
|
|
23620
|
-
|
|
23621
|
-
|
|
23622
|
-
|
|
23623
|
-
|
|
23624
|
-
const
|
|
23625
|
-
|
|
23626
|
-
|
|
23627
|
-
|
|
23628
|
-
const
|
|
23629
|
-
|
|
23630
|
-
|
|
23631
|
-
|
|
23632
|
-
|
|
23633
|
-
|
|
23634
|
-
|
|
23635
|
-
|
|
23636
|
-
|
|
23637
|
-
|
|
23638
|
-
|
|
23639
|
-
|
|
23640
|
-
|
|
23641
|
-
|
|
23642
|
-
|
|
23643
|
-
|
|
23644
|
-
|
|
23645
|
-
|
|
23646
|
-
|
|
23647
|
-
|
|
23648
|
-
|
|
23649
|
-
|
|
24357
|
+
raw,
|
|
24358
|
+
similarity: adjusted(target, raw),
|
|
24359
|
+
sharedQueries: sharedQueries.get(pairKey(source, target)) ?? 0
|
|
24360
|
+
})).sort((left, right) => Number(right.sharedQueries > 0) - Number(left.sharedQueries > 0) || right.similarity - left.similarity || left.source - right.source);
|
|
24361
|
+
await classify([target, ...ranked.slice(0, URL_SOURCES_PLACED).map((c) => c.source)]);
|
|
24362
|
+
if (targetWeight(types[target] ?? null) === 0 || pages[target]?.noindex) status = "target-not-linkable";
|
|
24363
|
+
else for (const candidate of ranked) {
|
|
24364
|
+
if (candidatesBySource.size >= URL_SOURCES_PLACED) break;
|
|
24365
|
+
if (!linkable(candidate.source, target)) continue;
|
|
24366
|
+
candidatesBySource.set(candidate.source, [{
|
|
24367
|
+
...candidate,
|
|
24368
|
+
similarity: adjusted(target, candidate.raw)
|
|
24369
|
+
}]);
|
|
24370
|
+
}
|
|
24371
|
+
}
|
|
24372
|
+
const judged = [];
|
|
24373
|
+
let pairsConsidered = 0;
|
|
24374
|
+
let pairsJudged = 0;
|
|
24375
|
+
let fellBack = 0;
|
|
24376
|
+
const sources = [...candidatesBySource.keys()].sort((left, right) => left - right);
|
|
24377
|
+
let sourcesWithPlacement = 0;
|
|
24378
|
+
for (let at = 0; at < sources.length; at += SOURCE_WINDOW) {
|
|
24379
|
+
if (context.mode === "url" && sourcesWithPlacement >= URL_SOURCES_JUDGED) break;
|
|
24380
|
+
const window = sources.slice(at, at + SOURCE_WINDOW);
|
|
24381
|
+
deps.progress?.(`Placing and judging links from pages ${at + 1} to ${at + window.length} of ${sources.length}.`);
|
|
24382
|
+
let pairs = (await Promise.all(window.map(async (source) => {
|
|
24383
|
+
const page = pages[source];
|
|
24384
|
+
if (!page) return [];
|
|
24385
|
+
const sentences = await crawl.sentences.read(page.text);
|
|
24386
|
+
const placed = [];
|
|
24387
|
+
for (const candidate of candidatesBySource.get(source) ?? []) {
|
|
24388
|
+
pairsConsidered += 1;
|
|
24389
|
+
const to = pages[candidate.target];
|
|
24390
|
+
if (!to) continue;
|
|
24391
|
+
const placement = pairPlacement(sentences, to, index);
|
|
24392
|
+
if (!placement.sentences.length) continue;
|
|
24393
|
+
placed.push({
|
|
24394
|
+
candidate,
|
|
24395
|
+
placement
|
|
24396
|
+
});
|
|
24397
|
+
}
|
|
24398
|
+
return placed;
|
|
24399
|
+
}))).flat();
|
|
24400
|
+
if (context.mode === "url") {
|
|
24401
|
+
pairs = pairs.sort((left, right) => Number(right.candidate.sharedQueries > 0) - Number(left.candidate.sharedQueries > 0) || right.candidate.similarity - left.candidate.similarity || left.candidate.source - right.candidate.source).slice(0, URL_SOURCES_JUDGED - sourcesWithPlacement);
|
|
24402
|
+
sourcesWithPlacement += pairs.length;
|
|
24403
|
+
}
|
|
24404
|
+
const bySource = /* @__PURE__ */ new Map();
|
|
24405
|
+
for (const pair of pairs) {
|
|
24406
|
+
const list = bySource.get(pair.candidate.source) ?? [];
|
|
24407
|
+
list.push(pair);
|
|
24408
|
+
bySource.set(pair.candidate.source, list);
|
|
24409
|
+
}
|
|
24410
|
+
const groups = [];
|
|
24411
|
+
for (const list of bySource.values()) for (let start = 0; start < list.length; start += 2) groups.push(list.slice(start, start + 2));
|
|
24412
|
+
await Promise.all(groups.map(async (group) => {
|
|
24413
|
+
const first = group[0];
|
|
24414
|
+
if (!first) return;
|
|
24415
|
+
const source = pages[first.candidate.source];
|
|
24416
|
+
if (!source) return;
|
|
24417
|
+
const response = runner ? await runner.ask(judgeRequest({
|
|
24418
|
+
url: source.url,
|
|
24419
|
+
title: source.title,
|
|
24420
|
+
type: types[first.candidate.source] ?? null
|
|
24421
|
+
}, group.map(({ candidate, placement }) => {
|
|
24422
|
+
const to = pages[candidate.target];
|
|
24423
|
+
return {
|
|
24424
|
+
url: to.url,
|
|
24425
|
+
title: to.title,
|
|
24426
|
+
description: to.description,
|
|
24427
|
+
h1: to.h1,
|
|
24428
|
+
type: types[candidate.target] ?? null,
|
|
24429
|
+
intent: intents[candidate.target] ?? null,
|
|
24430
|
+
opening: to.excerpt.slice(0, OPENING_CHARS),
|
|
24431
|
+
placement
|
|
24432
|
+
};
|
|
24433
|
+
}))) : null;
|
|
24434
|
+
group.forEach(({ candidate, placement }, position) => {
|
|
24435
|
+
let decision;
|
|
24436
|
+
if (response) {
|
|
24437
|
+
pairsJudged += 1;
|
|
24438
|
+
decision = decideWithModel(placement, candidate.similarity, response.answers, position);
|
|
24439
|
+
} else {
|
|
24440
|
+
if (runner) fellBack += 1;
|
|
24441
|
+
decision = decideWithoutModel(placement, candidate.similarity);
|
|
24442
|
+
}
|
|
24443
|
+
judged.push({
|
|
24444
|
+
candidate,
|
|
24445
|
+
placement,
|
|
24446
|
+
decision
|
|
24447
|
+
});
|
|
24448
|
+
});
|
|
24449
|
+
}));
|
|
24450
|
+
}
|
|
24451
|
+
const heuristicScore = (item) => item.candidate.similarity * (.5 + (item.decision.state === "suggest" ? item.decision.sentence?.match ?? 0 : 0));
|
|
24452
|
+
const suggested = judged.filter((item) => item.decision.state === "suggest").sort((left, right) => (right.decision.probability ?? -1) - (left.decision.probability ?? -1) || heuristicScore(right) - heuristicScore(left) || (pages[left.candidate.source]?.url ?? "").localeCompare(pages[right.candidate.source]?.url ?? "") || (pages[left.candidate.target]?.url ?? "").localeCompare(pages[right.candidate.target]?.url ?? ""));
|
|
24453
|
+
const perTarget = /* @__PURE__ */ new Map();
|
|
24454
|
+
const usedSentences = /* @__PURE__ */ new Set();
|
|
24455
|
+
const kept = suggested.filter((item) => {
|
|
24456
|
+
if (item.decision.tier === "heuristic" && item.decision.sentence) {
|
|
24457
|
+
const key = `${item.candidate.source}\u0000${item.decision.sentence.text}`;
|
|
24458
|
+
if (usedSentences.has(key)) return false;
|
|
24459
|
+
usedSentences.add(key);
|
|
24460
|
+
}
|
|
24461
|
+
const seen = (perTarget.get(item.candidate.target) ?? 0) + 1;
|
|
24462
|
+
perTarget.set(item.candidate.target, seen);
|
|
24463
|
+
return seen <= 8;
|
|
24464
|
+
});
|
|
24465
|
+
const items = kept.map(({ candidate, decision }) => ({
|
|
24466
|
+
source: pages[candidate.source]?.url ?? "",
|
|
24467
|
+
target: pages[candidate.target]?.url ?? "",
|
|
24468
|
+
tier: decision.tier,
|
|
24469
|
+
probability: decision.probability,
|
|
24470
|
+
role: decision.role,
|
|
24471
|
+
anchor: decision.anchor,
|
|
24472
|
+
sentence: decision.sentence?.text ?? null,
|
|
24473
|
+
section: decision.sentence?.section || null,
|
|
24474
|
+
sentenceMatch: decision.sentence?.match ?? null,
|
|
24475
|
+
similarity: round$1(candidate.similarity),
|
|
24476
|
+
sharedQueries: candidate.sharedQueries,
|
|
24477
|
+
targetInboundLinks: inbound[candidate.target] ?? 0,
|
|
24478
|
+
sourceType: types[candidate.source] ?? null,
|
|
24479
|
+
targetType: types[candidate.target] ?? null
|
|
24480
|
+
}));
|
|
24481
|
+
const suggestedInbound = new Array(count).fill(0);
|
|
24482
|
+
const suggestedOutbound = new Array(count).fill(0);
|
|
24483
|
+
for (const { candidate } of kept) {
|
|
24484
|
+
suggestedInbound[candidate.target] = (suggestedInbound[candidate.target] ?? 0) + 1;
|
|
24485
|
+
suggestedOutbound[candidate.source] = (suggestedOutbound[candidate.source] ?? 0) + 1;
|
|
24486
|
+
}
|
|
24487
|
+
const held = judged.filter((item) => item.decision.state === "held-back");
|
|
24488
|
+
const reasons = {};
|
|
24489
|
+
for (const item of held) reasons[item.decision.reason] = (reasons[item.decision.reason] ?? 0) + 1;
|
|
24490
|
+
const declined = judged.filter((item) => item.decision.state === "declined").length;
|
|
24491
|
+
const usage = runner ? runner.usage() : null;
|
|
24492
|
+
const judgeStatus = !runner ? "not-used" : runner.authError || usage && usage.requests === 0 && usage.failedRequests > 0 ? "failed" : (usage?.failedRequests ?? 0) > 0 ? "partial" : "used";
|
|
24493
|
+
const modelLabel = usage ? `${usage.model}` : null;
|
|
24494
|
+
const judgeNote = !runner ? `Jev was not used: ${"unavailable" in deps.decisionModel ? deps.decisionModel.unavailable : "no TypeSafe key"}. Links come from TF-IDF similarity and sentence matching (heuristics). Add a TypeSafe API key at https://seoskill.dev/account#keys and run seo login, or set TYPESAFE_API_KEY on this machine, to have Jev classify pages and judge each link.` : runner.authError ? `${runner.authError.message} ${fellBack} link${fellBack === 1 ? "" : "s"} fell back to heuristics.` : judgeStatus === "partial" ? `${usage?.failedRequests} request${usage?.failedRequests === 1 ? "" : "s"} to ${modelLabel} failed after retries; ${fellBack} link${fellBack === 1 ? "" : "s"} fell back to heuristics.` : `${modelLabel} classified ${pagesClassified} page${pagesClassified === 1 ? "" : "s"} and judged ${pairsJudged} link${pairsJudged === 1 ? "" : "s"} in ${usage?.requests ?? 0} requests, for $${(usage?.costUsd ?? 0).toFixed(4)}.`;
|
|
24495
|
+
if (status === "complete" && (crawl.stats.capReached || judgeStatus === "partial" || judgeStatus === "failed")) status = "partial";
|
|
24496
|
+
const counts = {
|
|
24497
|
+
verdict: "",
|
|
24498
|
+
pagesCrawled: count,
|
|
24499
|
+
existingContextualLinks,
|
|
24500
|
+
orphanPages: inboundAnywhere.filter((value, id) => value === 0 && types[id] !== "home").length,
|
|
24501
|
+
weaklyLinkedPages: inbound.filter((value, id) => value < WEAK_INBOUND && types[id] !== "home").length,
|
|
24502
|
+
pairsConsidered,
|
|
24503
|
+
suggestions: items.length,
|
|
24504
|
+
strong: items.filter((item) => item.tier === "strong").length,
|
|
24505
|
+
possible: items.filter((item) => item.tier === "possible").length,
|
|
24506
|
+
heuristic: items.filter((item) => item.tier === "heuristic").length,
|
|
24507
|
+
heldBack: held.length,
|
|
24508
|
+
declined
|
|
24509
|
+
};
|
|
24510
|
+
const targetPage = target !== void 0 ? pages[target] : void 0;
|
|
24511
|
+
const report = {
|
|
23650
24512
|
site: input.site,
|
|
23651
|
-
|
|
23652
|
-
|
|
23653
|
-
|
|
23654
|
-
|
|
23655
|
-
|
|
23656
|
-
|
|
23657
|
-
|
|
23658
|
-
|
|
23659
|
-
|
|
23660
|
-
|
|
23661
|
-
|
|
23662
|
-
|
|
23663
|
-
|
|
23664
|
-
|
|
23665
|
-
|
|
23666
|
-
calls: result.calls,
|
|
23667
|
-
possiblyTruncated: result.rowsFetched >= MAX_GSC_ROWS
|
|
23668
|
-
})),
|
|
23669
|
-
rowsFetched: targetRequests.reduce((sum, { result }) => sum + result.rowsFetched, 0),
|
|
23670
|
-
calls: targetRequests.reduce((sum, { result }) => sum + result.calls, 0),
|
|
23671
|
-
maxRowsPerRequest: MAX_GSC_ROWS,
|
|
23672
|
-
possiblyTruncated: targetTruncated
|
|
23673
|
-
},
|
|
23674
|
-
candidates: {
|
|
23675
|
-
queried: queryCandidates,
|
|
23676
|
-
rowsFetched: sourceResult.rowsFetched,
|
|
23677
|
-
calls: sourceResult.calls,
|
|
23678
|
-
maxRows: MAX_GSC_ROWS,
|
|
23679
|
-
possiblyTruncated: sourceTruncated
|
|
23680
|
-
},
|
|
23681
|
-
completeness: !queryCandidates ? "not-queried" : targetTruncated || sourceTruncated ? "possibly-truncated" : "retained-rows-only"
|
|
23682
|
-
},
|
|
23683
|
-
methodology: {
|
|
23684
|
-
id: "gsc_internal_link_candidates",
|
|
23685
|
-
version: 4,
|
|
23686
|
-
lexicalTargetLimit: 50,
|
|
23687
|
-
matchedQueryEvidenceLimit: 10,
|
|
23688
|
-
observedLinkEvidenceLimit: 20,
|
|
23689
|
-
matching: "pairwise_exact_then_precision_lexical",
|
|
23690
|
-
ranking: "exact_matches_then_matched_query_impressions_then_relevance",
|
|
23691
|
-
contextualPlacementVerified: true,
|
|
23692
|
-
placement: "tfidf_sentence_match_then_phrase_anchors"
|
|
23693
|
-
},
|
|
23694
|
-
filters: {
|
|
23695
|
-
minImpressions,
|
|
23696
|
-
resultLimit: limit,
|
|
23697
|
-
checkLimit,
|
|
23698
|
-
maxGscRowsPerRequest: MAX_GSC_ROWS
|
|
23699
|
-
},
|
|
23700
|
-
target: {
|
|
23701
|
-
requestedUrl: target.requestedUrl,
|
|
23702
|
-
preferredUrl: target.preferredUrl,
|
|
23703
|
-
finalUrl: target.finalUrl,
|
|
23704
|
-
canonical: target.canonical,
|
|
23705
|
-
status: target.status,
|
|
23706
|
-
aliases: target.aliases,
|
|
23707
|
-
verification: target.verification,
|
|
23708
|
-
technicalSignals: target.technicalSignals,
|
|
23709
|
-
fetchDiagnostics: target.fetchDiagnostics,
|
|
23710
|
-
queries: analysis.targetQueries
|
|
24513
|
+
mode: context.mode,
|
|
24514
|
+
targetUrl: context.targetUrl ?? null,
|
|
24515
|
+
generatedAt: context.now.toISOString(),
|
|
24516
|
+
dataStatus: status,
|
|
24517
|
+
crawl: {
|
|
24518
|
+
startUrl: context.startUrl,
|
|
24519
|
+
method: crawl.method,
|
|
24520
|
+
maxPages: crawl.stats.maxPages,
|
|
24521
|
+
pageLimit: crawl.stats.pageLimit,
|
|
24522
|
+
urlsFound: crawl.stats.urlsFound,
|
|
24523
|
+
pagesCrawled: count,
|
|
24524
|
+
capReached: crawl.stats.capReached,
|
|
24525
|
+
crawlDelaySeconds: crawl.stats.crawlDelaySeconds,
|
|
24526
|
+
skipped: crawl.stats.skipped,
|
|
24527
|
+
redirectsChecked: crawl.stats.redirectsChecked
|
|
23711
24528
|
},
|
|
23712
|
-
|
|
23713
|
-
|
|
23714
|
-
|
|
23715
|
-
|
|
23716
|
-
|
|
23717
|
-
|
|
23718
|
-
|
|
23719
|
-
|
|
23720
|
-
|
|
23721
|
-
|
|
23722
|
-
|
|
23723
|
-
|
|
23724
|
-
|
|
23725
|
-
|
|
24529
|
+
searchConsole,
|
|
24530
|
+
judge: {
|
|
24531
|
+
status: judgeStatus,
|
|
24532
|
+
note: judgeNote,
|
|
24533
|
+
keySource: "model" in deps.decisionModel ? deps.decisionModel.keySource : null,
|
|
24534
|
+
provider: usage?.provider ?? null,
|
|
24535
|
+
model: usage?.model ?? null,
|
|
24536
|
+
requests: usage?.requests ?? 0,
|
|
24537
|
+
failedRequests: usage?.failedRequests ?? 0,
|
|
24538
|
+
inputTokens: usage?.inputTokens ?? 0,
|
|
24539
|
+
costUsd: usage?.costUsd ?? 0,
|
|
24540
|
+
pagesClassified,
|
|
24541
|
+
pairsJudged,
|
|
24542
|
+
thresholds: { ...THRESHOLDS },
|
|
24543
|
+
floors: { ...FLOORS }
|
|
23726
24544
|
},
|
|
24545
|
+
target: target !== void 0 && targetPage ? {
|
|
24546
|
+
url: targetPage.url,
|
|
24547
|
+
title: targetPage.title,
|
|
24548
|
+
type: types[target] ?? null,
|
|
24549
|
+
intent: intents[target] ?? null,
|
|
24550
|
+
inboundLinks: inbound[target] ?? 0,
|
|
24551
|
+
inboundLinksAnywhere: inboundAnywhere[target] ?? 0,
|
|
24552
|
+
noindex: targetPage.noindex
|
|
24553
|
+
} : null,
|
|
24554
|
+
summary: counts,
|
|
23727
24555
|
items,
|
|
23728
|
-
|
|
23729
|
-
|
|
23730
|
-
|
|
23731
|
-
|
|
23732
|
-
|
|
23733
|
-
|
|
23734
|
-
|
|
23735
|
-
|
|
23736
|
-
|
|
23737
|
-
|
|
23738
|
-
|
|
23739
|
-
|
|
23740
|
-
|
|
23741
|
-
|
|
23742
|
-
|
|
24556
|
+
heldBack: {
|
|
24557
|
+
count: held.length,
|
|
24558
|
+
reasons,
|
|
24559
|
+
examples: held.slice(0, EXAMPLE_LIMIT).map((item) => ({
|
|
24560
|
+
source: pages[item.candidate.source]?.url ?? "",
|
|
24561
|
+
target: pages[item.candidate.target]?.url ?? "",
|
|
24562
|
+
reason: item.decision.reason,
|
|
24563
|
+
detail: item.decision.detail
|
|
24564
|
+
}))
|
|
24565
|
+
},
|
|
24566
|
+
pageInventory: context.mode === "site" ? pages.map((page, id) => ({
|
|
24567
|
+
url: page.url,
|
|
24568
|
+
title: page.title,
|
|
24569
|
+
type: types[id] ?? null,
|
|
24570
|
+
intent: intents[id] ?? null,
|
|
24571
|
+
words: page.words,
|
|
24572
|
+
inboundLinks: inbound[id] ?? 0,
|
|
24573
|
+
inboundLinksAnywhere: inboundAnywhere[id] ?? 0,
|
|
24574
|
+
outboundLinks: contextual[id]?.size ?? 0,
|
|
24575
|
+
suggestedInbound: suggestedInbound[id] ?? 0,
|
|
24576
|
+
suggestedOutbound: suggestedOutbound[id] ?? 0,
|
|
24577
|
+
noindex: page.noindex
|
|
24578
|
+
})).sort((left, right) => left.inboundLinks - right.inboundLinks || left.url.localeCompare(right.url)) : [],
|
|
24579
|
+
redirectedLinks,
|
|
24580
|
+
genericAnchors: crawl.genericAnchors.map((anchor) => ({
|
|
24581
|
+
source: pages[anchor.source]?.url ?? "",
|
|
24582
|
+
href: anchor.href,
|
|
24583
|
+
text: anchor.text
|
|
24584
|
+
})),
|
|
24585
|
+
warnings,
|
|
24586
|
+
caveats: [],
|
|
24587
|
+
next: null
|
|
23743
24588
|
};
|
|
23744
|
-
|
|
23745
|
-
|
|
23746
|
-
|
|
23747
|
-
const placed = found.map(({ item, text }) => {
|
|
23748
|
-
const placement = target && text ? linkPlacement(text, target, corpus) : {
|
|
23749
|
-
contentSimilarity: 0,
|
|
23750
|
-
sentences: []
|
|
23751
|
-
};
|
|
23752
|
-
const best = placement.sentences[0];
|
|
23753
|
-
const recommendation = best ? {
|
|
23754
|
-
...item.recommendation,
|
|
23755
|
-
action: `${item.recommendation.action} It fits in: "${best.text}" Anchor text could be ${best.anchors.map((anchor) => `"${anchor}"`).join(" or ")}.`
|
|
23756
|
-
} : {
|
|
23757
|
-
...item.recommendation,
|
|
23758
|
-
action: `${item.recommendation.action} No sentence on this page names the target, so a link here needs new text.`
|
|
23759
|
-
};
|
|
23760
|
-
return {
|
|
23761
|
-
...item,
|
|
23762
|
-
placement,
|
|
23763
|
-
recommendation
|
|
23764
|
-
};
|
|
24589
|
+
report.summary.verdict = internalLinksVerdict({
|
|
24590
|
+
...report,
|
|
24591
|
+
counts
|
|
23765
24592
|
});
|
|
23766
|
-
|
|
24593
|
+
report.caveats = caveats$1(report);
|
|
24594
|
+
const top = items[0];
|
|
24595
|
+
report.next = context.mode === "site" ? top ? `seo internal-links --url ${top.target}` : null : `seo page --url ${report.target?.url ?? context.targetUrl}`;
|
|
24596
|
+
return report;
|
|
24597
|
+
}
|
|
24598
|
+
function caveats$1(report) {
|
|
24599
|
+
const crawl = report.crawl;
|
|
24600
|
+
return [
|
|
24601
|
+
`Crawl: ${crawl.pagesCrawled} page${crawl.pagesCrawled === 1 ? "" : "s"} from ${crawl.method === "sitemap" ? "the sitemap" : "links from the start page"}, limit ${crawl.pageLimit}${crawl.capReached ? " (reached, so pages beyond it were not checked and orphan and weak-link counts cover crawled pages only)" : " (not reached)"}.${crawl.crawlDelaySeconds ? ` robots.txt crawl-delay of ${crawl.crawlDelaySeconds}s was honoured.` : ""}`,
|
|
24602
|
+
report.judge.note,
|
|
24603
|
+
report.searchConsole.status === "used" ? `Search Console: ${report.searchConsole.rows.toLocaleString("en-US")} query rows from ${report.searchConsole.range?.startDate} to ${report.searchConsole.range?.endDate}, brand queries left out; ${report.searchConsole.pairsWithSharedQueries} page pairs share a query.${report.searchConsole.possiblyTruncated ? " The 100,000-row cap was reached." : ""}` : `Search Console was not used${report.searchConsole.note ? `: ${report.searchConsole.note}` : ""}. Candidates come from page text only.`,
|
|
24604
|
+
`Rules in code: a link needs a model probability of ${THRESHOLDS.possible} (${THRESHOLDS.strong} is strong), a sentence with at least ${FLOORS.minSentenceMatch * 100}% of the words that name the target or pages at least ${FLOORS.minSimilarityAlone} similar, and an anchor from the chosen sentence. At most 8 links per target.`,
|
|
24605
|
+
"Similarity, sentence match and anchors are text heuristics, and a model probability is a judgement of reader value. None of them predicts rankings or traffic."
|
|
24606
|
+
];
|
|
23767
24607
|
}
|
|
23768
24608
|
//#endregion
|
|
23769
24609
|
//#region packages/core/src/analyze/url-templates.ts
|
|
@@ -27559,6 +28399,30 @@ async function detectBrandTerms(input) {
|
|
|
27559
28399
|
candidates: ranked
|
|
27560
28400
|
};
|
|
27561
28401
|
}
|
|
28402
|
+
//#endregion
|
|
28403
|
+
//#region packages/core/src/analyze/ai-queries.ts
|
|
28404
|
+
const RULES = [
|
|
28405
|
+
["agent_prompt", "^(search the web for|browse the web|find sources)|do not invent|return the [0-9]+ most relevant|include source urls|context: location|do not include location|question: |reply only with|the model you are using"],
|
|
28406
|
+
["tracker", "\\. ?my location is |\\?\\. ?my( |$)|^evaluate the .+ on "],
|
|
28407
|
+
["reply", "^(yes|yeah|yep|ok|okay|sure|yes please|go on|continue|keep going|tell me more|why not|how so|really|exactly|show me|and then|thanks|thank you|got it|what else|anything else)[?!.,]*$"],
|
|
28408
|
+
["follow_up", "^(what|how) about .{1,40}$|^(and|so|ok so|okay so|but) (what|which|how|why) "],
|
|
28409
|
+
["operator", "(^|\\s)-?(site|before|after|intitle|inurl|intext|filetype):|\"[^\"]*\" or \""],
|
|
28410
|
+
["chat", "\\b(please|can you|could you|give me|show me|tell me|i want|i need|i have|i am|should i|do i need|is it worth|help me|walk me through)\\b|^(what|how|why|is|does|can|will|would) (it|that|this|they)\\b"],
|
|
28411
|
+
["long", "^(\\S+\\s+){9,}\\S+$"]
|
|
28412
|
+
];
|
|
28413
|
+
RULES.map(([kind, pattern]) => [kind, new RegExp(pattern)]);
|
|
28414
|
+
RULES.map(([, pattern]) => `(?:${pattern})`).join("|");
|
|
28415
|
+
new Set("the and for with that this what which who how why when where are is was can does do did you your our from about into than then them they their there have has had will would should could not but its it’s me my i am of to in on at by or an as be if so".split(" "));
|
|
28416
|
+
new Map(`
|
|
28417
|
+
ae:ar al:sq am:hy ao:pt ar:es at:de au:en az:az ba:bs bd:bn be:nl bf:fr
|
|
28418
|
+
bg:bg bh:ar bo:es br:pt ca:en ch:de ci:fr cl:es cm:fr co:es cr:es cy:el
|
|
28419
|
+
cz:cs de:de dk:da dz:ar ec:es ee:et eg:ar es:es fi:fi fr:fr gb:en gh:en
|
|
28420
|
+
gr:el gt:es hk:en hr:hr hu:hu id:id ie:en il:he in:en it:it jo:ar jp:ja
|
|
28421
|
+
ke:en kh:en kr:ko kz:ru lk:en lt:lt lv:lv ma:ar mc:fr md:ro mk:mk mm:en
|
|
28422
|
+
mt:en mx:es my:en ng:en ni:es nl:nl no:nb nz:en pa:es pe:es ph:en pk:en
|
|
28423
|
+
pl:pl pt:pt py:es ro:ro rs:sr sa:ar se:sv sg:en si:sl sk:sk sn:fr sv:es
|
|
28424
|
+
th:th tn:ar tr:tr tw:zh-TW ua:uk us:en uy:es ve:es vn:vi za:en
|
|
28425
|
+
`.trim().split(/\s+/).map((pair) => pair.split(":")));
|
|
27562
28426
|
const DIMENSIONS = [
|
|
27563
28427
|
"query",
|
|
27564
28428
|
"page",
|
|
@@ -27746,20 +28610,6 @@ function checkPages(urls) {
|
|
|
27746
28610
|
return Promise.all(urls.slice(0, MAX_PAGES).map(facts));
|
|
27747
28611
|
}
|
|
27748
28612
|
//#endregion
|
|
27749
|
-
//#region packages/core/src/analyze/ai-queries.ts
|
|
27750
|
-
const RULES = [
|
|
27751
|
-
["agent_prompt", "^(search the web for|browse the web|find sources)|do not invent|return the [0-9]+ most relevant|include source urls|context: location|do not include location|question: "],
|
|
27752
|
-
["tracker", "\\. ?my location is |^evaluate the .+ on "],
|
|
27753
|
-
["reply", "^(yes|yeah|yep|ok|okay|sure|yes please|go on|continue|keep going|tell me more|why not|how so|really|exactly|show me|and then|thanks|thank you|got it|what else|anything else)[?!.,]*$"],
|
|
27754
|
-
["follow_up", "^(what|how) about .{1,40}$|^(and|so|ok so|okay so|but) (what|which|how|why) "],
|
|
27755
|
-
["operator", "(^|\\s)-?(site|before|after|intitle|inurl|intext|filetype):|\"[^\"]*\" or \""],
|
|
27756
|
-
["chat", "\\b(please|can you|could you|give me|show me|tell me|i want|i need|i have|i am|should i|do i need|is it worth|help me|walk me through)\\b|^(what|how|why|is|does|can|will|would) (it|that|this|they)\\b"],
|
|
27757
|
-
["long", "^(\\S+\\s+){9,}\\S+$"]
|
|
27758
|
-
];
|
|
27759
|
-
RULES.map(([kind, pattern]) => [kind, new RegExp(pattern)]);
|
|
27760
|
-
RULES.map(([, pattern]) => `(?:${pattern})`).join("|");
|
|
27761
|
-
new Set("the and for with that this what which who how why when where are is was can does do did you your our from about into than then them they their there have has had will would should could not but its it’s me my i am of to in on at by or an as be if so".split(" "));
|
|
27762
|
-
//#endregion
|
|
27763
28613
|
//#region packages/core/src/commands/report.ts
|
|
27764
28614
|
const SECTIONS = {
|
|
27765
28615
|
overview: "Site totals (clicks, impressions, CTR, position) against the previous period, by device, with the top pages and queries and their change, and Google Analytics sessions for the same dates from the property whose web stream matches the site.",
|
|
@@ -27774,16 +28624,6 @@ const SECTIONS = {
|
|
|
27774
28624
|
bing: "Bing’s top queries from Bing Webmaster Tools with Google’s clicks, impressions and position for the same query beside them, and the same for pages."
|
|
27775
28625
|
};
|
|
27776
28626
|
const SECTION_IDS = Object.keys(SECTIONS);
|
|
27777
|
-
new Map(`
|
|
27778
|
-
ae:ar al:sq am:hy ao:pt ar:es at:de au:en az:az ba:bs bd:bn be:nl bf:fr
|
|
27779
|
-
bg:bg bh:ar bo:es br:pt ca:en ch:de ci:fr cl:es cm:fr co:es cr:es cy:el
|
|
27780
|
-
cz:cs de:de dk:da dz:ar ec:es ee:et eg:ar es:es fi:fi fr:fr gb:en gh:en
|
|
27781
|
-
gr:el gt:es hk:en hr:hr hu:hu id:id ie:en il:he in:en it:it jo:ar jp:ja
|
|
27782
|
-
ke:en kh:en kr:ko kz:ru lk:en lt:lt lv:lv ma:ar mc:fr md:ro mk:mk mm:en
|
|
27783
|
-
mt:en mx:es my:en ng:en ni:es nl:nl no:nb nz:en pa:es pe:es ph:en pk:en
|
|
27784
|
-
pl:pl pt:pt py:es ro:ro rs:sr sa:ar se:sv sg:en si:sl sk:sk sn:fr sv:es
|
|
27785
|
-
th:th tn:ar tr:tr tw:zh-TW ua:uk us:en uy:es ve:es vn:vi za:en
|
|
27786
|
-
`.trim().split(/\s+/).map((pair) => pair.split(":")));
|
|
27787
28627
|
//#endregion
|
|
27788
28628
|
//#region packages/core/src/commands/research.ts
|
|
27789
28629
|
const RESEARCH_ACTIONS = [
|
|
@@ -27791,9 +28631,11 @@ const RESEARCH_ACTIONS = [
|
|
|
27791
28631
|
"ranked_keywords",
|
|
27792
28632
|
"serp",
|
|
27793
28633
|
"competitors",
|
|
27794
|
-
"backlinks"
|
|
28634
|
+
"backlinks",
|
|
28635
|
+
"ai_answers"
|
|
27795
28636
|
];
|
|
27796
28637
|
const RESEARCH_PROVIDERS = ["dataforseo", "ahrefs"];
|
|
28638
|
+
const RESEARCH_HELP = `keyword_metrics (keywords: volume, difficulty, CPC), ranked_keywords (target: keywords a domain or URL ranks for), serp (keywords, up to 5: the ranking pages), competitors (target: domains that share keywords), backlinks (target: referring domains), ai_answers (DataForSEO only; target: the site. It takes the site’s top 10 chat-style and long questions from Search Console’s AI-shaped queries itself, then asks ChatGPT search, Google AI Mode and Google AI Overviews each one and says whether the answer cites or mentions the site. DataForSEO’s live prices are $0.012 a question, so up to about $0.12 a run: tell the person before running it). Rows with a domain, and the target, get a free Ahrefs domain_rating column. Numbers are provider estimates, not Search Console data.`;
|
|
27797
28639
|
//#endregion
|
|
27798
28640
|
//#region packages/core/src/commands/site.ts
|
|
27799
28641
|
function indexStatusFrom(result) {
|
|
@@ -27846,7 +28688,7 @@ z.object({}), `${Object.entries(SECTIONS).map(([id, text]) => `- ${id}: ${text}`
|
|
|
27846
28688
|
}), `${LOST_PAGES_RULE}`, z.object({
|
|
27847
28689
|
site,
|
|
27848
28690
|
...dates
|
|
27849
|
-
}), z.object({
|
|
28691
|
+
}), `${RESEARCH_HELP}`, z.object({
|
|
27850
28692
|
provider: z.enum(RESEARCH_PROVIDERS),
|
|
27851
28693
|
action: z.enum(RESEARCH_ACTIONS),
|
|
27852
28694
|
keywords: z.array(z.string().min(1)).max(100).default([]),
|
|
@@ -28114,6 +28956,25 @@ async function localResearchKeys() {
|
|
|
28114
28956
|
if (ahrefs?.apiKey) keys.set("x-ahrefs-key", ahrefs.apiKey);
|
|
28115
28957
|
return keys;
|
|
28116
28958
|
}
|
|
28959
|
+
const TYPESAFE_API_KEY_ENV = "TYPESAFE_API_KEY";
|
|
28960
|
+
async function localInternalLinksDependencies(options) {
|
|
28961
|
+
const searchConsole = await resolveLocalSite(options.site, { refresh: options.refresh }).then((property) => ({
|
|
28962
|
+
property,
|
|
28963
|
+
query: (request) => querySearchAnalytics(property, request, { refresh: options.refresh })
|
|
28964
|
+
}), (error) => ({ unavailable: `no Search Console property was found for this site (${error instanceof Error ? error.message : String(error)})` }));
|
|
28965
|
+
const local = process.env[TYPESAFE_API_KEY_ENV]?.trim();
|
|
28966
|
+
const saved = local ? void 0 : (await localResearchKeys().catch(() => new Headers())).get("x-typesafe-key");
|
|
28967
|
+
return {
|
|
28968
|
+
searchConsole,
|
|
28969
|
+
decisionModel: local ? {
|
|
28970
|
+
model: typesafeDecisionModel({ apiKey: local }),
|
|
28971
|
+
keySource: "environment"
|
|
28972
|
+
} : saved ? {
|
|
28973
|
+
model: typesafeDecisionModel({ apiKey: saved }),
|
|
28974
|
+
keySource: "account"
|
|
28975
|
+
} : { unavailable: "no TypeSafe key is set" }
|
|
28976
|
+
};
|
|
28977
|
+
}
|
|
28117
28978
|
async function localRefreshClicky() {
|
|
28118
28979
|
if (!cloudLinked()) return null;
|
|
28119
28980
|
await cloudCredentials({ refreshClicky: true });
|
|
@@ -29737,6 +30598,6 @@ function isoDate(daysAgo = 0) {
|
|
|
29737
30598
|
return date.toISOString().slice(0, 10);
|
|
29738
30599
|
}
|
|
29739
30600
|
//#endregion
|
|
29740
|
-
export { AHREFS_API_KEY_ENV, AHREFS_API_KEY_SECRET, AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, ANALYTICS_PROVIDER_IDS, AUDIT_PAGE_RULE_IDS, AhrefsClient, BING_API_KEY_ENV, BING_API_KEY_SECRET, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CLICKY_SITEKEY_ENV, CLICKY_SITE_KEYS_SECRET, CLOUD_SIGN_IN, CLOUD_URL, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DATAFORSEO_CREDENTIAL_SECRET, DATAFORSEO_LOGIN_ENV, DATAFORSEO_PASSWORD_ENV, DEFAULT_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, DEFAULT_UNREACHED_PAGE_LIMIT, DataForSeoClient, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, GoogleApiError, INCIDENTS_URL,
|
|
30601
|
+
export { AHREFS_API_KEY_ENV, AHREFS_API_KEY_SECRET, AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, ANALYTICS_PROVIDER_IDS, AUDIT_PAGE_RULE_IDS, AhrefsClient, BING_API_KEY_ENV, BING_API_KEY_SECRET, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CLICKY_SITEKEY_ENV, CLICKY_SITE_KEYS_SECRET, CLOUD_SIGN_IN, CLOUD_URL, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DATAFORSEO_CREDENTIAL_SECRET, DATAFORSEO_LOGIN_ENV, DATAFORSEO_PASSWORD_ENV, DEFAULT_MAX_PAGES, DEFAULT_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, DEFAULT_UNREACHED_PAGE_LIMIT, DataForSeoClient, DecisionModelError, DecisionRunner, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, GoogleApiError, INCIDENTS_URL, JsonFileStorageAdapter, LINK_ROLES, LOCAL_BING_CONNECT, LOCAL_SIGN_IN, LOG_MAX_AGE_MS, LOG_MAX_FILE_BYTES, LOG_MAX_TOTAL_BYTES, LocalCredentialsProvider, MANAGED_PROVIDER_SECRET_NAMES, MAX_CRAWL_CONCURRENCY, MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES, MAX_CRAWL_TIMEOUT_MS, MAX_MAX_PAGES, MAX_SEARCH_CONSOLE_EXPORT_FILES, MAX_SEARCH_CONSOLE_EXPORT_ROW_LIMIT, PAGE_TYPES, PROVIDER_SECRET_NAMES, ProviderError, ProviderExtensionRegistry, RESEARCH_IMPORT_COLUMN_KEYS, ResponseSizeLimitError, SEARCH_INTENTS, SEO_CRAWLER_IDENTITY, SEO_CRAWLER_TOKEN, SEO_CRAWLER_USER_AGENT, SEO_PROVIDER_API_VERSION, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, STEP_CHANGE_RULES, SeoError, ServiceAccountAccessTokenClient, SessionLedger, TELEMETRY_AGENTS, TELEMETRY_ENDPOINT, TELEMETRY_ERROR_CATEGORIES, TELEMETRY_EVENTS, TELEMETRY_FAILURE_CONTEXTS, TELEMETRY_FAILURE_REASONS, TELEMETRY_NOTICE, TELEMETRY_OPERATIONS, TELEMETRY_REPORTS, TELEMETRY_SCHEMA_VERSION, TELEMETRY_TIMEOUT_MS, TYPESAFE_API_KEY_ENV, agentActionsView, aiReferralSourceForValue, aiReferralsReport, analyticsConnectionLabel, analyticsConnectionProviderId, analyticsConnectionSchema, analyticsProviderDetails, analyticsProviderSchema, analyzeAiReferralRows, analyzeCannibalImportRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertCrawlConfigLimits, assertUrlMatchesGscProperty, auditCrawlPages, auditCrawlRequests, auditPage, authStatus, bearerFetch, boundedSitemapInventory, buildAgentFindings, buildQueryClusterReportFromRows, buildTelemetryPayload, cannibalImportReport, cannibalReport, cannibalReportWith, changeWindow, checkDatabaseReadiness, classifyTelemetryFailure, clearCache, cloudAccountFor, cloudBing, cloudCredentials, cloudGoogleEmails, cloudGoogleToken, cloudKeys, cloudLinked, cloudLogout, clusterQueryRows, combineRobotsValues, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlStatusOnly, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, ctrUnderperformersReportWith, decayClusterDrafts, decayComparisonRange, decayingReport, decayingReportWith, deleteAhrefsApiKey, deleteBingApiKey, deleteClickySiteKey, deleteDataForSeoCredentials, deleteManagedProviderSecrets, deleteProviderExtensionAccount, deleteProviderExtensionCredentials, deleteProviderSecret, deleteTokens, deriveBrandTerms, describeInstalledProviderExtension, detectBrandTerms, detectTelemetryAgent, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensurePrivateDatabaseFiles, ensureSeoCliDirs, estimateSerpTitleWidth, expectedCtrForPosition, explainRule, fetchCruxFieldData, fetchPage, fetchPageStatus, fetchSearchStatusIncidents, fetchSitemapUrls, fileSize, finalGscDateRange, findOverlappingSearchUpdates, finishCloudLogin, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, getAuthModeStatus, getCacheStats, getClientConfig, getDb, getPrivacySnapshot, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTelemetryStatus, getTokenStorageStatus, googleJson, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoverageSignals, inferTemplateMovement, initializeTelemetry, inspectProviderPackage, inspectSearchConsoleUrl, inspectUrl, installProviderPackage, installedProviderPackageSchema, installedProviderPackagesSchema, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isSkippableReportError, isTelemetryCi, isTelemetryOperation, isTelemetryReport, isoDate, isoWeek, latestFinalGscDate, linkRecover, linkRecoverRecommendation, listAnalyticsAccounts, listAnalyticsDataStreams, listGa4AccountSummaries, listGa4DataStreams, listGoogleAccounts, listInstalledProviderExtensions, listRules, listSearchConsoleSites, listSearchUpdates, listSites, loadInstalledProviderExtensions, loadProviderExtensions, loadProviderPackage, loadSearchConsoleExport, localAccounts, localBing, localCredentialsProvider, localError, localGoogleFetch, localInternalLinksDependencies, localRefreshClicky, localResearchKeys, loginWithLoopback, makeRunId, matchGa4WebStreams, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, nodeCommandDeps, normalizeBrandText, normalizeClickySiteId, normalizeClickySiteKey, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, normalizeProviderExtensionAccount, normalizedResearchColumnName, notSignedIn, noteCacheWrite, openCloudLink, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, pendingCloudLogin, performanceAudit, performanceReportIsCacheable, plural, providerCapabilitySchema, providerConnectionFieldSchema, providerExactVersionSchema, providerExtensionIdSchema, providerIdSchema, providerIntegritySchema, providerKindSchema, providerPackageManifestSchema, providerPackageNameSchema, providerPreferenceSchema, providerRegistrationSchema, providerRequestJson, providerRequestText, pruneLogs, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, quickWinsReportWith, readAhrefsApiKey, readAllTokens, readBingApiKey, readBoundedResponseText, readClickySiteKey, readClickySiteKeys, readConfig, readDataForSeoCredentials, readInstalledProviderPackages, readJson, readJsonOrUndefined, readOauthClient, readProviderExtensionAccount, readProviderExtensionCredentials, readProviderSecret, readTokens, recommendationForRule, reconcileExportPagesWithCrawl, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, registrableDomain, removeInstalledProviderPackage, renderCrawlCsv, renderCrawlHtml, renderCrawlJunit, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderWorkflowMarkdown, reportNarrative, requestHeaders, requirePageVerification, researchImportColumns, resetSeoData, resolveLocalSite, resolveSegmentRanges, resolveSeoCliPaths, reviewObservations, roundedPosition, runAnalyticsProviderLandingPages, runAnalyticsReport, runDoctor, runGa4Report, runProviderExtensionAction, runtimeForProvider, safeProviderAdapterError, saveInstalledProviderPackage, searchAnalyticsRows, searchUpdatesFromIncidents, secondPage, secondPageWith, segmentComparisonRange, segmentImpact, segmentRangeDays, seoErrorEnvelope, sessionSourceInListFilter, setActiveGoogleAccount, setTelemetryEnabled, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteStartUrl, startCloudLogin, strikingDistance, strikingDistanceWith, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, telemetryErrorCategory, toSeoError, tokenSchema, tokenStoreSchema, topFixes, trackTelemetryCommandFailed, trackTelemetryReportComplete, trackTelemetryReportFailed, trackTelemetryReportStart, trackTelemetrySetupComplete, trafficAnomaly, trafficAnomalyWith, typesafeDecisionModel, unavailableSegmentImpactReport, unicodeTokens, uninstallProviderExtension, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateResearchImportColumnSources, verifyAnalyticsProvider, verifyProviderExtension, verifyQueryContent, verifyQueryContentWith, withAgentReportContract, withRankingHistory, workflowPresentation, writeAhrefsApiKey, writeBingApiKey, writeClickySiteKey, writeConfig, writeDataForSeoCredentials, writeInstalledProviderPackages, writeOauthClient, writeProviderExtensionAccount, writeProviderExtensionCredentials, writeProviderSecret, writeTokens, xRobotsDirectives };
|
|
29741
30602
|
|
|
29742
30603
|
//# sourceMappingURL=index.js.map
|