recker 1.0.79 → 1.0.80-next.70ea84d

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/dist/browser/browser/mini.d.ts +2 -2
  2. package/dist/browser/browser/recker-mini.d.ts +8 -8
  3. package/dist/browser/browser/recker.d.ts +11 -8
  4. package/dist/browser/browser/recker.js +54 -6
  5. package/dist/browser/core/client.d.ts +15 -10
  6. package/dist/browser/core/client.js +54 -38
  7. package/dist/browser/index.iife.min.js +129 -130
  8. package/dist/browser/index.min.js +129 -130
  9. package/dist/browser/index.mini.iife.js +5697 -636
  10. package/dist/browser/index.mini.iife.min.js +47 -48
  11. package/dist/browser/index.mini.min.js +47 -48
  12. package/dist/browser/index.mini.umd.js +5697 -636
  13. package/dist/browser/index.mini.umd.min.js +47 -48
  14. package/dist/browser/index.umd.min.js +129 -130
  15. package/dist/browser/mini.d.ts +2 -2
  16. package/dist/browser/plugins/proxy-rotator.d.ts +2 -2
  17. package/dist/browser/plugins/proxy-rotator.js +6 -28
  18. package/dist/browser/recker-mini.d.ts +8 -8
  19. package/dist/browser/recker.d.ts +11 -8
  20. package/dist/browser/recker.js +54 -6
  21. package/dist/browser/scrape/document.js +2 -2
  22. package/dist/browser/scrape/element.js +7 -1
  23. package/dist/browser/scrape/parser/nodes/html.js +1 -1
  24. package/dist/browser/scrape/spider.d.ts +52 -0
  25. package/dist/browser/scrape/spider.js +620 -38
  26. package/dist/browser/scrape/types.d.ts +2 -0
  27. package/dist/browser/search/google.d.ts +26 -1
  28. package/dist/browser/search/google.js +427 -45
  29. package/dist/browser/seo/analyzer.d.ts +1 -0
  30. package/dist/browser/seo/analyzer.js +144 -1
  31. package/dist/browser/seo/index.d.ts +1 -1
  32. package/dist/browser/seo/keyword-campaign-analyzer.d.ts +2 -0
  33. package/dist/browser/seo/keyword-campaign-analyzer.js +538 -0
  34. package/dist/browser/seo/keyword-campaign-seed-advanced.d.ts +17 -0
  35. package/dist/browser/seo/keyword-campaign-seed-advanced.js +269 -0
  36. package/dist/browser/seo/keyword-campaign-seed-core.d.ts +29 -0
  37. package/dist/browser/seo/keyword-campaign-seed-core.js +525 -0
  38. package/dist/browser/seo/keyword-campaign-shared.d.ts +165 -0
  39. package/dist/browser/seo/keyword-campaign-shared.js +59 -0
  40. package/dist/browser/seo/keyword-campaign.d.ts +4 -107
  41. package/dist/browser/seo/keyword-campaign.js +2 -380
  42. package/dist/browser/seo/keywords.js +5 -22
  43. package/dist/browser/seo/types.d.ts +19 -0
  44. package/dist/browser/transport/curl.d.ts +5 -1
  45. package/dist/browser/transport/curl.js +207 -50
  46. package/dist/browser/transport/undici.d.ts +4 -3
  47. package/dist/browser/transport/undici.js +123 -49
  48. package/dist/browser/types/index.d.ts +9 -3
  49. package/dist/browser/utils/binary-manager.js +26 -3
  50. package/dist/browser/utils/block-detector.d.ts +8 -0
  51. package/dist/browser/utils/block-detector.js +542 -7
  52. package/dist/cli/commands/hls-runner.js +5 -4
  53. package/dist/cli/commands/live-runner.js +5 -4
  54. package/dist/cli/commands/loadtest-runner.js +3 -2
  55. package/dist/cli/commands/search.d.ts +2 -0
  56. package/dist/cli/commands/search.js +105 -0
  57. package/dist/cli/commands/seo-runner.js +9 -7
  58. package/dist/cli/commands/seo.js +140 -1
  59. package/dist/cli/commands/serve.js +75 -131
  60. package/dist/cli/commands/server-runner.js +59 -82
  61. package/dist/cli/commands/spider-runner.d.ts +37 -1
  62. package/dist/cli/commands/spider-runner.js +134 -10
  63. package/dist/cli/commands/spider.d.ts +18 -1
  64. package/dist/cli/commands/spider.js +457 -27
  65. package/dist/cli/events/handlers/cli.js +30 -1
  66. package/dist/cli/events/handlers/tui.js +26 -0
  67. package/dist/cli/events/types.d.ts +27 -0
  68. package/dist/cli/handler.d.ts +2 -12
  69. package/dist/cli/handler.js +20 -15
  70. package/dist/cli/handlers/protocols.js +39 -12
  71. package/dist/cli/handlers/search.d.ts +2 -0
  72. package/dist/cli/handlers/search.js +171 -0
  73. package/dist/cli/handlers/seo-analyze.d.ts +1 -0
  74. package/dist/cli/handlers/seo-analyze.js +666 -0
  75. package/dist/cli/handlers/seo-robots.d.ts +1 -0
  76. package/dist/cli/handlers/seo-robots.js +76 -0
  77. package/dist/cli/handlers/seo-serp.d.ts +54 -0
  78. package/dist/cli/handlers/seo-serp.js +243 -0
  79. package/dist/cli/handlers/seo-sitemap.d.ts +1 -0
  80. package/dist/cli/handlers/seo-sitemap.js +55 -0
  81. package/dist/cli/handlers/seo-spider.d.ts +1 -0
  82. package/dist/cli/handlers/seo-spider.js +334 -0
  83. package/dist/cli/handlers/seo.d.ts +8 -4
  84. package/dist/cli/handlers/seo.js +294 -442
  85. package/dist/cli/handlers/spider.js +94 -17
  86. package/dist/cli/handlers/streaming.js +5 -1
  87. package/dist/cli/index.js +11 -2
  88. package/dist/cli/presets.d.ts +1 -1
  89. package/dist/cli/presets.js +15 -4
  90. package/dist/cli/tui/app.js +6 -1
  91. package/dist/cli/tui/components/rich-response.d.ts +72 -0
  92. package/dist/cli/tui/components/rich-response.js +117 -0
  93. package/dist/cli/tui/executor-commands/background.js +30 -24
  94. package/dist/cli/tui/executor-commands/testing.js +3 -2
  95. package/dist/cli/tui/hooks/useDomains.d.ts +17 -0
  96. package/dist/cli/tui/hooks/useHelp.js +15 -2
  97. package/dist/cli/tui/job-manager.d.ts +4 -4
  98. package/dist/cli/tui/job-manager.js +5 -1
  99. package/dist/cli/tui/spider-tui.d.ts +63 -0
  100. package/dist/cli/tui/spider-tui.js +120 -2
  101. package/dist/cli/types.d.ts +12 -0
  102. package/dist/cli/types.js +1 -0
  103. package/dist/cli/utils/option-helpers.d.ts +10 -0
  104. package/dist/cli/utils/option-helpers.js +63 -0
  105. package/dist/cli/utils/score-color.d.ts +11 -0
  106. package/dist/cli/utils/score-color.js +11 -0
  107. package/dist/cli/utils/serp-campaign.d.ts +53 -0
  108. package/dist/cli/utils/serp-campaign.js +53 -0
  109. package/dist/cli/utils/serp-config.d.ts +26 -0
  110. package/dist/cli/utils/serp-config.js +125 -0
  111. package/dist/core/client.d.ts +15 -10
  112. package/dist/core/client.js +54 -38
  113. package/dist/index.d.ts +1 -0
  114. package/dist/index.js +1 -0
  115. package/dist/mcp/cli.js +35 -34
  116. package/dist/mcp/client.d.ts +2 -2
  117. package/dist/mcp/client.js +20 -2
  118. package/dist/mcp/contract.d.ts +1 -1
  119. package/dist/mcp/profiles.js +5 -1
  120. package/dist/mcp/prompts/index.js +8 -4
  121. package/dist/mcp/resources/index.js +46 -23
  122. package/dist/mcp/server.js +9 -6
  123. package/dist/mcp/tools/protocols.js +9 -2
  124. package/dist/mcp/tools/registry.js +13 -3
  125. package/dist/mcp/tools/seo.js +427 -2
  126. package/dist/mcp/types.d.ts +5 -1
  127. package/dist/plugins/proxy-rotator.d.ts +2 -2
  128. package/dist/plugins/proxy-rotator.js +6 -28
  129. package/dist/raffel/client.d.ts +38 -0
  130. package/dist/raffel/client.js +282 -0
  131. package/dist/raffel/index.d.ts +2 -0
  132. package/dist/raffel/index.js +2 -0
  133. package/dist/raffel/types.d.ts +40 -0
  134. package/dist/raffel/types.js +14 -0
  135. package/dist/recker.d.ts +13 -7
  136. package/dist/recker.js +58 -6
  137. package/dist/scrape/document.js +2 -2
  138. package/dist/scrape/element.js +7 -1
  139. package/dist/scrape/parser/nodes/html.js +1 -1
  140. package/dist/scrape/spider.d.ts +52 -0
  141. package/dist/scrape/spider.js +620 -38
  142. package/dist/scrape/types.d.ts +2 -0
  143. package/dist/search/google.d.ts +26 -1
  144. package/dist/search/google.js +427 -45
  145. package/dist/search/index.d.ts +1 -1
  146. package/dist/seo/analyzer.d.ts +1 -0
  147. package/dist/seo/analyzer.js +144 -1
  148. package/dist/seo/index.d.ts +1 -1
  149. package/dist/seo/keyword-campaign-analyzer.d.ts +2 -0
  150. package/dist/seo/keyword-campaign-analyzer.js +538 -0
  151. package/dist/seo/keyword-campaign-seed-advanced.d.ts +17 -0
  152. package/dist/seo/keyword-campaign-seed-advanced.js +269 -0
  153. package/dist/seo/keyword-campaign-seed-core.d.ts +29 -0
  154. package/dist/seo/keyword-campaign-seed-core.js +525 -0
  155. package/dist/seo/keyword-campaign-shared.d.ts +165 -0
  156. package/dist/seo/keyword-campaign-shared.js +59 -0
  157. package/dist/seo/keyword-campaign.d.ts +4 -107
  158. package/dist/seo/keyword-campaign.js +2 -380
  159. package/dist/seo/keywords.js +5 -22
  160. package/dist/seo/types.d.ts +19 -0
  161. package/dist/template/index.d.ts +1 -1
  162. package/dist/template/types.d.ts +0 -2
  163. package/dist/testing/index.d.ts +1 -22
  164. package/dist/testing/index.js +1 -11
  165. package/dist/transport/curl.d.ts +5 -1
  166. package/dist/transport/curl.js +207 -50
  167. package/dist/transport/undici.d.ts +4 -3
  168. package/dist/transport/undici.js +123 -49
  169. package/dist/types/index.d.ts +9 -3
  170. package/dist/utils/binary-manager.js +26 -3
  171. package/dist/utils/block-detector.d.ts +8 -0
  172. package/dist/utils/block-detector.js +542 -7
  173. package/dist/version.js +1 -1
  174. package/package.json +12 -1
  175. package/dist/testing/mock-dns-server.d.ts +0 -69
  176. package/dist/testing/mock-dns-server.js +0 -269
  177. package/dist/testing/mock-ftp-server.d.ts +0 -89
  178. package/dist/testing/mock-ftp-server.js +0 -562
  179. package/dist/testing/mock-hls-server.d.ts +0 -80
  180. package/dist/testing/mock-hls-server.js +0 -381
  181. package/dist/testing/mock-http-server.d.ts +0 -124
  182. package/dist/testing/mock-http-server.js +0 -343
  183. package/dist/testing/mock-proxy-server.d.ts +0 -108
  184. package/dist/testing/mock-proxy-server.js +0 -615
  185. package/dist/testing/mock-sse-server.d.ts +0 -76
  186. package/dist/testing/mock-sse-server.js +0 -291
  187. package/dist/testing/mock-telnet-server.d.ts +0 -59
  188. package/dist/testing/mock-telnet-server.js +0 -274
  189. package/dist/testing/mock-udp-server.d.ts +0 -43
  190. package/dist/testing/mock-udp-server.js +0 -188
  191. package/dist/testing/mock-websocket-server.d.ts +0 -76
  192. package/dist/testing/mock-websocket-server.js +0 -334
  193. package/dist/testing/mock-whois-server.d.ts +0 -56
  194. package/dist/testing/mock-whois-server.js +0 -234
  195. package/dist/testing/proxy-certs.d.ts +0 -19
  196. package/dist/testing/proxy-certs.js +0 -208
@@ -0,0 +1,538 @@
1
+ import { searchGoogleAdvanced } from '../search/google.js';
2
+ import { clampNumber, normalizeHost, normalizeKeyword, normalizePlacement, toSeedArray, DEFAULT_SERP_MIN_KEYWORD_LENGTH, } from './keyword-campaign-shared.js';
3
+ import { isLikelyBlocked, isLikelyCaptcha, sleep, buildSerpBlockReason, DEFAULT_SERP_MAX_CONSECUTIVE_BLOCKS, DEFAULT_SERP_SEARCH_CAPTCHA_COOLDOWN_MS, DEFAULT_SERP_SEARCH_DELAY_MS, DEFAULT_SERP_SEARCH_JITTER_MS, DEFAULT_SERP_SEARCH_RETRY_COUNT, DEFAULT_SERP_SEARCH_RETRY_DELAY_MS, } from './keyword-campaign-seed-core.js';
4
+ import { mergeKeywordSeeds } from './keyword-campaign-seed-advanced.js';
5
+ function normalizePageUrl(value) {
6
+ return value && value.trim().length > 0 ? value.trim() : 'unknown-page';
7
+ }
8
+ function ensureCompetitorTracker(trackers, domain) {
9
+ const normalizedDomain = domain.toLowerCase();
10
+ const existing = trackers.get(normalizedDomain);
11
+ if (existing) {
12
+ return existing;
13
+ }
14
+ const created = {
15
+ organicQueries: 0,
16
+ paidQueries: 0,
17
+ bestOrganicRank: null,
18
+ bestPaidRank: null,
19
+ keywords: new Set(),
20
+ outperformedQueries: 0,
21
+ outperformedGapTotal: 0,
22
+ outperformedTargetPositionTotal: 0,
23
+ outperformedCompetitorPositionTotal: 0,
24
+ outperformedComparableQueries: 0,
25
+ outperformedByPage: new Map(),
26
+ };
27
+ trackers.set(normalizedDomain, created);
28
+ return created;
29
+ }
30
+ function trackOutperformance(tracker, sourcePage, targetPosition, competitorPosition) {
31
+ tracker.outperformedQueries += 1;
32
+ const bucket = tracker.outperformedByPage.get(sourcePage) ?? {
33
+ outperformedQueries: 0,
34
+ gapTotal: 0,
35
+ targetTotal: 0,
36
+ competitorTotal: 0,
37
+ comparableQueries: 0,
38
+ };
39
+ bucket.outperformedQueries += 1;
40
+ if (targetPosition !== null) {
41
+ const gap = targetPosition - competitorPosition;
42
+ bucket.gapTotal += gap;
43
+ bucket.targetTotal += targetPosition;
44
+ bucket.competitorTotal += competitorPosition;
45
+ bucket.comparableQueries += 1;
46
+ tracker.outperformedGapTotal += gap;
47
+ tracker.outperformedTargetPositionTotal += targetPosition;
48
+ tracker.outperformedCompetitorPositionTotal += competitorPosition;
49
+ tracker.outperformedComparableQueries += 1;
50
+ }
51
+ tracker.outperformedByPage.set(sourcePage, bucket);
52
+ }
53
+ function buildEmptyReport(targetUrl, targetDomain) {
54
+ return {
55
+ targetUrl,
56
+ targetDomain,
57
+ results: [],
58
+ summary: {
59
+ queriesRequested: 0,
60
+ queriesExecuted: 0,
61
+ queriesFound: 0,
62
+ queriesBlocked: 0,
63
+ queriesCaptcha: 0,
64
+ campaignStopped: false,
65
+ avgTopPosition: null,
66
+ top3Count: 0,
67
+ top10Count: 0,
68
+ topOrganicCompetitors: [],
69
+ topPaidCompetitors: [],
70
+ competitorCoverage: {
71
+ organicUniqueDomains: 0,
72
+ paidUniqueDomains: 0,
73
+ },
74
+ },
75
+ pageComparison: [],
76
+ campaign: {
77
+ active: false,
78
+ confidence: 'low',
79
+ evidence: [],
80
+ },
81
+ };
82
+ }
83
+ function dedupeDomainResults(results, targetDomain) {
84
+ const seen = new Set();
85
+ const domainResults = [];
86
+ for (const result of results) {
87
+ if (!result.domain || result.domain === targetDomain)
88
+ continue;
89
+ if (seen.has(result.domain))
90
+ continue;
91
+ seen.add(result.domain);
92
+ domainResults.push({
93
+ domain: result.domain,
94
+ rank: result.rank,
95
+ placement: result.placement,
96
+ placementHint: result.placementHint,
97
+ url: result.url,
98
+ title: result.title,
99
+ });
100
+ }
101
+ return domainResults;
102
+ }
103
+ function trackCompetitors(candidates, keyword, targetDomain, trackers) {
104
+ for (const competitor of candidates) {
105
+ if (competitor.domain === targetDomain)
106
+ continue;
107
+ const current = ensureCompetitorTracker(trackers, competitor.domain);
108
+ if (competitor.placement === 'ad') {
109
+ current.paidQueries += 1;
110
+ if (current.bestPaidRank === null || competitor.rank < current.bestPaidRank) {
111
+ current.bestPaidRank = competitor.rank;
112
+ }
113
+ }
114
+ else {
115
+ current.organicQueries += 1;
116
+ if (current.bestOrganicRank === null || competitor.rank < current.bestOrganicRank) {
117
+ current.bestOrganicRank = competitor.rank;
118
+ }
119
+ }
120
+ current.keywords.add(keyword);
121
+ }
122
+ }
123
+ function makeSummaryEntry(domain, tracker) {
124
+ const outperformedPages = [...tracker.outperformedByPage.entries()]
125
+ .map(([pageUrl, pageStats]) => ({
126
+ pageUrl,
127
+ outperformedQueries: pageStats.outperformedQueries,
128
+ avgTargetPosition: pageStats.comparableQueries > 0
129
+ ? Math.round(pageStats.targetTotal / pageStats.comparableQueries)
130
+ : null,
131
+ avgCompetitorPosition: pageStats.comparableQueries > 0
132
+ ? Math.round(pageStats.competitorTotal / pageStats.comparableQueries)
133
+ : null,
134
+ avgGap: pageStats.comparableQueries > 0
135
+ ? Math.round(pageStats.gapTotal / pageStats.comparableQueries)
136
+ : null,
137
+ }))
138
+ .sort((a, b) => b.outperformedQueries - a.outperformedQueries || a.pageUrl.localeCompare(b.pageUrl));
139
+ return {
140
+ domain,
141
+ organicQueries: tracker.organicQueries,
142
+ paidQueries: tracker.paidQueries,
143
+ bestOrganicRank: tracker.bestOrganicRank,
144
+ bestPaidRank: tracker.bestPaidRank,
145
+ matchedKeywords: tracker.keywords.size,
146
+ totalOutperformedQueries: tracker.outperformedQueries,
147
+ avgOutperformedGap: tracker.outperformedComparableQueries > 0
148
+ ? Math.round(tracker.outperformedGapTotal / tracker.outperformedComparableQueries)
149
+ : null,
150
+ avgTargetPositionWhenOutperformed: tracker.outperformedComparableQueries > 0
151
+ ? Math.round(tracker.outperformedTargetPositionTotal / tracker.outperformedComparableQueries)
152
+ : null,
153
+ avgCompetitorPositionWhenOutperformed: tracker.outperformedComparableQueries > 0
154
+ ? Math.round(tracker.outperformedCompetitorPositionTotal / tracker.outperformedComparableQueries)
155
+ : null,
156
+ outperformedPages,
157
+ };
158
+ }
159
+ function finalizeCompetitorSummary(trackers) {
160
+ const ranked = [...trackers.entries()].map(([domain, tracker]) => ({
161
+ domain,
162
+ tracker,
163
+ organicQueries: tracker.organicQueries,
164
+ paidQueries: tracker.paidQueries,
165
+ bestOrganicRank: tracker.bestOrganicRank,
166
+ bestPaidRank: tracker.bestPaidRank,
167
+ matchedKeywords: tracker.keywords.size,
168
+ score: (tracker.organicQueries * 2) + (tracker.paidQueries * 3),
169
+ }));
170
+ const scoreSort = (a, b) => b.score - a.score || a.domain.localeCompare(b.domain);
171
+ const topOrganic = ranked
172
+ .filter((entry) => entry.organicQueries > 0)
173
+ .sort(scoreSort)
174
+ .map(({ domain, tracker }) => makeSummaryEntry(domain, tracker))
175
+ .slice(0, 10);
176
+ const topPaid = ranked
177
+ .filter((entry) => entry.paidQueries > 0)
178
+ .sort(scoreSort)
179
+ .map(({ domain, tracker }) => makeSummaryEntry(domain, tracker))
180
+ .slice(0, 10);
181
+ return {
182
+ topOrganic,
183
+ topPaid,
184
+ coverage: {
185
+ organicUniqueDomains: ranked.filter((entry) => entry.organicQueries > 0).length,
186
+ paidUniqueDomains: ranked.filter((entry) => entry.paidQueries > 0).length,
187
+ },
188
+ };
189
+ }
190
+ export async function analyzeKeywordCampaign(options) {
191
+ const targetUrl = options.targetUrl.trim();
192
+ const targetDomain = normalizeHost(targetUrl);
193
+ const queryLimit = clampNumber(options.maxQueries, 20);
194
+ const defaultCampaignResultLimit = 25;
195
+ const searchResultLimit = Math.max(clampNumber(options.maxResultsPerQuery, defaultCampaignResultLimit), 20);
196
+ const minKeywordLength = options.minKeywordLength ?? DEFAULT_SERP_MIN_KEYWORD_LENGTH;
197
+ const minWeight = Math.max(1, options.minWeight ?? 1);
198
+ const searchConcurrency = clampNumber(options.searchConcurrency, 4);
199
+ const searchDelayMs = Math.max(0, options.searchDelayMs ?? DEFAULT_SERP_SEARCH_DELAY_MS);
200
+ const searchDelayJitterMs = Math.max(0, options.searchDelayJitterMs ?? DEFAULT_SERP_SEARCH_JITTER_MS);
201
+ const searchCaptchaCooldownMs = Math.max(0, options.searchCaptchaCooldownMs ?? DEFAULT_SERP_SEARCH_CAPTCHA_COOLDOWN_MS);
202
+ const searchRetryCount = Math.max(0, options.searchRetryCount ?? DEFAULT_SERP_SEARCH_RETRY_COUNT);
203
+ const searchRetryDelayMs = Math.max(0, options.searchRetryDelayMs ?? DEFAULT_SERP_SEARCH_RETRY_DELAY_MS);
204
+ const searchMaxConsecutiveBlocks = Math.max(0, options.searchMaxConsecutiveBlocks ?? DEFAULT_SERP_MAX_CONSECUTIVE_BLOCKS);
205
+ const discovered = toSeedArray(options.discoveredKeywords ?? []);
206
+ const preset = (options.presetKeywords ?? []).map((keyword) => ({
207
+ keyword,
208
+ normalizedKeyword: normalizeKeyword(keyword),
209
+ source: 'preset',
210
+ weight: 1,
211
+ }));
212
+ const seeds = mergeKeywordSeeds(discovered, preset, minKeywordLength, minWeight).slice(0, queryLimit);
213
+ const report = buildEmptyReport(targetUrl, targetDomain);
214
+ if (seeds.length === 0 || !targetDomain) {
215
+ report.summary.queriesRequested = 0;
216
+ return report;
217
+ }
218
+ const pageBuckets = new Map();
219
+ const competitorTrackers = new Map();
220
+ const campaignEvidence = [];
221
+ const campaignControl = {
222
+ consecutiveBlockSignals: 0,
223
+ limitConsecutiveBlockSignals: searchMaxConsecutiveBlocks,
224
+ };
225
+ const resultsByIndex = new Array(seeds.length);
226
+ const requestTiming = {
227
+ nextRequestAt: 0,
228
+ delayMultiplier: 1,
229
+ backoffMultiplier: 1,
230
+ cooldownBonus: 1,
231
+ lastFailurePenaltyAppliedAt: 0,
232
+ lastFailureAt: 0,
233
+ };
234
+ let requestGate = Promise.resolve();
235
+ const runInGate = async (task) => {
236
+ const previous = requestGate;
237
+ let release;
238
+ requestGate = new Promise((resolve) => {
239
+ release = resolve;
240
+ });
241
+ await previous;
242
+ try {
243
+ await task();
244
+ }
245
+ finally {
246
+ release?.();
247
+ }
248
+ };
249
+ const gate = async () => {
250
+ await runInGate(async () => {
251
+ const now = Date.now();
252
+ const targetDelay = Math.round(Math.max(0, searchDelayMs * requestTiming.delayMultiplier));
253
+ const jitter = searchDelayJitterMs > 0
254
+ ? Math.floor(Math.random() * (searchDelayJitterMs + 1))
255
+ : 0;
256
+ const availableAt = Math.max(requestTiming.nextRequestAt, now);
257
+ const waitMs = Math.max(0, availableAt - now);
258
+ requestTiming.nextRequestAt = availableAt + targetDelay + jitter;
259
+ if (waitMs > 0) {
260
+ await sleep(waitMs);
261
+ }
262
+ });
263
+ };
264
+ const applySignalCooldown = (signalType) => {
265
+ const now = Date.now();
266
+ if (signalType === 'success') {
267
+ requestTiming.delayMultiplier = Math.max(1, requestTiming.delayMultiplier * 0.75);
268
+ requestTiming.backoffMultiplier = Math.max(1, requestTiming.backoffMultiplier * 0.85);
269
+ requestTiming.lastFailureAt = 0;
270
+ return;
271
+ }
272
+ requestTiming.lastFailureAt = now;
273
+ requestTiming.delayMultiplier = Math.min(8, requestTiming.delayMultiplier * 1.8);
274
+ requestTiming.backoffMultiplier = Math.min(4, requestTiming.backoffMultiplier + 1);
275
+ const isConsecutiveCaptcha = signalType === 'captcha'
276
+ && now - requestTiming.lastFailurePenaltyAppliedAt < searchCaptchaCooldownMs;
277
+ const consecutivePenaltyMultiplier = isConsecutiveCaptcha
278
+ ? 1 + Math.min(5, Math.floor((requestTiming.backoffMultiplier - 1) * 2))
279
+ : 1;
280
+ if (signalType === 'captcha') {
281
+ requestTiming.lastFailurePenaltyAppliedAt = now;
282
+ }
283
+ const cooldown = Math.round(searchCaptchaCooldownMs * requestTiming.backoffMultiplier * consecutivePenaltyMultiplier);
284
+ requestTiming.nextRequestAt = Math.max(requestTiming.nextRequestAt, now + cooldown);
285
+ };
286
+ report.summary.queriesRequested = seeds.length;
287
+ const registerSignal = (wasBlocked, keyword, source, sourcePage) => {
288
+ if (report.summary.campaignStopped)
289
+ return;
290
+ if (!wasBlocked) {
291
+ campaignControl.consecutiveBlockSignals = 0;
292
+ return;
293
+ }
294
+ const hasBlockLimit = campaignControl.limitConsecutiveBlockSignals > 0;
295
+ if (!hasBlockLimit) {
296
+ return;
297
+ }
298
+ campaignControl.consecutiveBlockSignals += 1;
299
+ if (campaignControl.consecutiveBlockSignals >= campaignControl.limitConsecutiveBlockSignals) {
300
+ report.summary.campaignStopped = true;
301
+ report.summary.campaignStopReason = `Parada antecipada por ${campaignControl.consecutiveBlockSignals} sinais consecutivos de bloqueio/captcha para evitar bloqueio adicional`;
302
+ campaignEvidence.push(`${source}: "${keyword}" ${sourcePage ? `(${sourcePage}) ` : ''}alcançou limite de bloqueio; pausa de segurança ativa para reduzir WAF/challenge.`);
303
+ }
304
+ };
305
+ const executeSearch = async (seed, pageKey, normalizedKeyword) => {
306
+ const query = seed.keyword;
307
+ let response;
308
+ let lastError;
309
+ let attempts = 0;
310
+ if (report.summary.campaignStopped) {
311
+ return undefined;
312
+ }
313
+ while (attempts <= searchRetryCount) {
314
+ if (report.summary.campaignStopped) {
315
+ return undefined;
316
+ }
317
+ attempts += 1;
318
+ if (report.summary.campaignStopped) {
319
+ return undefined;
320
+ }
321
+ await gate();
322
+ if (report.summary.campaignStopped) {
323
+ return undefined;
324
+ }
325
+ try {
326
+ response = await searchGoogleAdvanced(query, {
327
+ num: searchResultLimit,
328
+ maxResults: searchResultLimit,
329
+ transport: options.transport,
330
+ timeout: options.timeout,
331
+ country: options.country,
332
+ gl: options.gl,
333
+ hl: options.hl,
334
+ ...(options.searchParams ?? {}),
335
+ });
336
+ }
337
+ catch (error) {
338
+ lastError = error;
339
+ }
340
+ if (!response) {
341
+ if (attempts <= searchRetryCount) {
342
+ const retryDelay = Math.min(searchRetryDelayMs * attempts, 6_000) + Math.floor(Math.random() * (searchDelayJitterMs + 1));
343
+ requestTiming.nextRequestAt = Math.max(requestTiming.nextRequestAt, Date.now() + retryDelay);
344
+ continue;
345
+ }
346
+ break;
347
+ }
348
+ const wasCaptcha = isLikelyCaptcha(response.captcha);
349
+ const wasBlocked = isLikelyBlocked(response.block);
350
+ if ((wasCaptcha || wasBlocked) && attempts <= searchRetryCount) {
351
+ const retryDelay = Math.min(searchRetryDelayMs * attempts, 6_000) + Math.floor(Math.random() * (searchDelayJitterMs + 1));
352
+ applySignalCooldown(wasCaptcha ? 'captcha' : 'blocked');
353
+ requestTiming.nextRequestAt = Math.max(requestTiming.nextRequestAt, Date.now() + retryDelay + (wasCaptcha ? searchCaptchaCooldownMs : 0));
354
+ response = undefined;
355
+ continue;
356
+ }
357
+ applySignalCooldown('success');
358
+ break;
359
+ }
360
+ if (!response) {
361
+ return {
362
+ result: {
363
+ keyword: query,
364
+ source: seed.source,
365
+ sourcePage: seed.sourcePage,
366
+ sourceWeight: seed.weight,
367
+ blocked: true,
368
+ blockReason: 'request failed',
369
+ found: false,
370
+ bestPosition: null,
371
+ totalChecked: searchResultLimit,
372
+ placement: 'unknown',
373
+ searchUrl: '',
374
+ searchTransport: options.transport ?? 'undici',
375
+ competitors: [],
376
+ },
377
+ wasBlocked: true,
378
+ wasCaptcha: false,
379
+ pageKey,
380
+ normalizedKeyword,
381
+ };
382
+ }
383
+ const wasBlocked = isLikelyBlocked(response.block);
384
+ const wasCaptcha = isLikelyCaptcha(response.captcha);
385
+ const parsedResults = response.results.map((result) => ({
386
+ ...result,
387
+ domain: normalizeHost(result.url),
388
+ placement: normalizePlacement(result.placement),
389
+ }));
390
+ const matched = parsedResults.find((result) => {
391
+ return result.domain && result.domain === targetDomain;
392
+ });
393
+ const competitorCandidates = dedupeDomainResults(parsedResults.map((result) => ({
394
+ domain: result.domain,
395
+ rank: result.rank,
396
+ placement: result.placement,
397
+ placementHint: result.placementHint,
398
+ url: result.url,
399
+ title: result.title,
400
+ })), targetDomain);
401
+ if (wasCaptcha && searchCaptchaCooldownMs > 0) {
402
+ requestTiming.nextRequestAt = Math.max(requestTiming.nextRequestAt, Date.now() + searchCaptchaCooldownMs);
403
+ }
404
+ return {
405
+ result: {
406
+ keyword: query,
407
+ source: seed.source,
408
+ sourcePage: seed.sourcePage,
409
+ sourceWeight: seed.weight,
410
+ blocked: wasBlocked || wasCaptcha,
411
+ blockReason: buildSerpBlockReason(response.block, response.captcha),
412
+ found: Boolean(matched),
413
+ bestPosition: matched?.rank ?? null,
414
+ totalChecked: parsedResults.length,
415
+ matchedUrl: matched?.url,
416
+ matchedTitle: matched?.title,
417
+ matchedDisplayUrl: matched?.displayedUrl,
418
+ placement: matched ? normalizePlacement(matched.placement) : 'unknown',
419
+ placementHint: matched?.placementHint,
420
+ searchUrl: response.searchUrl,
421
+ searchTransport: response.transport.used,
422
+ competitors: competitorCandidates,
423
+ },
424
+ wasBlocked,
425
+ wasCaptcha,
426
+ pageKey,
427
+ normalizedKeyword,
428
+ };
429
+ };
430
+ const workers = Math.max(1, Math.min(searchConcurrency, seeds.length));
431
+ const workerState = { nextIndex: 0 };
432
+ const finalizeExecution = (execution) => {
433
+ const { result: searchResult, wasBlocked, wasCaptcha, pageKey, normalizedKeyword, } = execution;
434
+ const bucket = pageBuckets.get(pageKey) ?? {
435
+ tracked: 0,
436
+ found: 0,
437
+ totalPosition: 0,
438
+ top3: 0,
439
+ top10: 0,
440
+ };
441
+ bucket.tracked += 1;
442
+ report.summary.queriesExecuted += 1;
443
+ if (wasBlocked)
444
+ report.summary.queriesBlocked += 1;
445
+ if (wasCaptcha)
446
+ report.summary.queriesCaptcha += 1;
447
+ if (searchResult.placement === 'unknown' && searchResult.blocked) {
448
+ campaignEvidence.push(`Falha ao buscar keyword "${searchResult.keyword}"`);
449
+ }
450
+ trackCompetitors(searchResult.competitors, normalizedKeyword, targetDomain, competitorTrackers);
451
+ if (searchResult.found && searchResult.bestPosition !== null) {
452
+ const position = searchResult.bestPosition;
453
+ report.summary.queriesFound += 1;
454
+ if (position <= 3)
455
+ report.summary.top3Count += 1;
456
+ if (position <= 10)
457
+ report.summary.top10Count += 1;
458
+ if (searchResult.placement === 'ad') {
459
+ campaignEvidence.push(`${searchResult.keyword} aparece como anúncio em posição #${position}`);
460
+ }
461
+ else {
462
+ campaignEvidence.push(`${searchResult.keyword} aparece orgânico em posição #${position}`);
463
+ }
464
+ for (const competitor of searchResult.competitors) {
465
+ if (competitor.rank < position) {
466
+ const tracker = ensureCompetitorTracker(competitorTrackers, competitor.domain);
467
+ trackOutperformance(tracker, pageKey, position, competitor.rank);
468
+ }
469
+ }
470
+ bucket.found += 1;
471
+ bucket.totalPosition += position;
472
+ if (position <= 3)
473
+ bucket.top3 += 1;
474
+ if (position <= 10)
475
+ bucket.top10 += 1;
476
+ }
477
+ pageBuckets.set(pageKey, bucket);
478
+ };
479
+ const runWorker = async () => {
480
+ while (true) {
481
+ if (report.summary.campaignStopped)
482
+ break;
483
+ if (workerState.nextIndex >= seeds.length)
484
+ break;
485
+ const currentIndex = workerState.nextIndex++;
486
+ const seed = seeds[currentIndex];
487
+ const pageKey = normalizePageUrl(seed.sourcePage ?? (seed.source === 'preset' ? 'preset-queries' : 'unknown'));
488
+ const execution = await executeSearch(seed, pageKey, seed.normalizedKeyword);
489
+ if (!execution) {
490
+ if (report.summary.campaignStopped)
491
+ break;
492
+ continue;
493
+ }
494
+ resultsByIndex[currentIndex] = execution;
495
+ registerSignal(execution.wasBlocked || execution.wasCaptcha, execution.result.keyword, execution.result.source, execution.result.sourcePage);
496
+ finalizeExecution(execution);
497
+ }
498
+ };
499
+ await Promise.all(new Array(workers).fill(null).map(() => runWorker()));
500
+ report.results = resultsByIndex
501
+ .filter((entry) => Boolean(entry))
502
+ .map(({ result }) => result);
503
+ const foundPositions = report.results
504
+ .filter((entry) => entry.found && entry.bestPosition !== null)
505
+ .map((entry) => entry.bestPosition);
506
+ if (foundPositions.length > 0) {
507
+ report.summary.avgTopPosition = Math.round(foundPositions.reduce((acc, pos) => acc + pos, 0) / foundPositions.length);
508
+ }
509
+ const hasTop3 = foundPositions.some((position) => position <= 3);
510
+ const hasTop10 = foundPositions.some((position) => position <= 10);
511
+ report.campaign = {
512
+ active: foundPositions.some((position) => position <= 10),
513
+ confidence: hasTop3 ? 'high' : hasTop10 ? 'medium' : foundPositions.length > 0 ? 'low' : 'low',
514
+ evidence: campaignEvidence,
515
+ };
516
+ const competitorSummary = finalizeCompetitorSummary(competitorTrackers);
517
+ report.summary.topOrganicCompetitors = competitorSummary.topOrganic;
518
+ report.summary.topPaidCompetitors = competitorSummary.topPaid;
519
+ report.summary.competitorCoverage = competitorSummary.coverage;
520
+ report.pageComparison = [...pageBuckets.entries()].map(([pageUrl, bucket]) => ({
521
+ pageUrl,
522
+ tracked: bucket.tracked,
523
+ found: bucket.found,
524
+ appearanceRate: bucket.tracked > 0
525
+ ? Number(((bucket.found / bucket.tracked) * 100).toFixed(1))
526
+ : 0,
527
+ avgPosition: bucket.found > 0 ? Math.round(bucket.totalPosition / bucket.found) : null,
528
+ top3: bucket.top3,
529
+ top10: bucket.top10,
530
+ })).sort((a, b) => {
531
+ if (a.avgPosition === null)
532
+ return 1;
533
+ if (b.avgPosition === null)
534
+ return -1;
535
+ return a.avgPosition - b.avgPosition;
536
+ });
537
+ return report;
538
+ }
@@ -0,0 +1,17 @@
1
+ import type { SeoReport } from './types.js';
2
+ import type { KeywordCampaignSeed } from './keyword-campaign-shared.js';
3
+ export declare function deduplicateKeywordPermutations(seeds: KeywordCampaignSeed[]): KeywordCampaignSeed[];
4
+ export declare function getSeedWordCount(seed: KeywordCampaignSeed): number;
5
+ export declare function extractSchemaOrgLongTailSeeds(report: SeoReport, options: {
6
+ minKeywordLength: number;
7
+ maxPhrases: number;
8
+ sourcePage?: string;
9
+ }): KeywordCampaignSeed[];
10
+ export declare function extractHeadingPathCompositionSeeds(report: SeoReport, options: {
11
+ minKeywordLength: number;
12
+ maxPhrases: number;
13
+ topKeywordSet: Set<string>;
14
+ sourcePage?: string;
15
+ }): KeywordCampaignSeed[];
16
+ export declare function sortSeedsByWeight(a: KeywordCampaignSeed, b: KeywordCampaignSeed): number;
17
+ export declare function mergeKeywordSeeds(discovered: KeywordCampaignSeed[], preset: KeywordCampaignSeed[], minKeywordLength?: number, minWeight?: number): KeywordCampaignSeed[];